Merge pull request #26852 from taosdata/fix/TD-29991/inNull

fix: in null

source/common/src/tdatablock.c

@@ -87,7 +87,7 @@ int32_t getJsonValueLen(const char* data) {
 }
 
 int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) {
-  if (isNull) {
+  if (isNull || pData == NULL) {
     // There is a placehold for each NULL value of binary or nchar type.
     if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
       pColumnInfoData->varmeta.offset[rowIndex] = -1;  // it is a null value of VAR type.

source/libs/scalar/src/scalar.c

@@ -1176,27 +1176,6 @@ EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
     }
   }
 
-  if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) {
-    SNodeListNode *listNode = (SNodeListNode *)node->pRight;
-    SNode         *tnode = NULL;
-    WHERE_EACH(tnode, listNode->pNodeList) {
-      if (SCL_IS_NULL_VALUE_NODE(tnode)) {
-        if (node->opType == OP_TYPE_IN) {
-          ERASE_NODE(listNode->pNodeList);
-          continue;
-        } else {  // OP_TYPE_NOT_IN
-          return sclRewriteNullInOptr(pNode, ctx, node->opType);
-        }
-      }
-
-      WHERE_NEXT;
-    }
-
-    if (listNode->pNodeList->length <= 0) {
-      return sclRewriteNullInOptr(pNode, ctx, node->opType);
-    }
-  }
-
   return DEAL_RES_CONTINUE;
 }
 
@@ -1334,6 +1313,27 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
     return DEAL_RES_ERROR;
   }
 
+  if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) {
+    SNodeListNode *listNode = (SNodeListNode *)node->pRight;
+    SNode         *tnode = NULL;
+    WHERE_EACH(tnode, listNode->pNodeList) {
+      if (SCL_IS_NULL_VALUE_NODE(tnode)) {
+        if (node->opType == OP_TYPE_IN) {
+          ERASE_NODE(listNode->pNodeList);
+          continue;
+        } else {  // OP_TYPE_NOT_IN
+          return sclRewriteNullInOptr(pNode, ctx, node->opType);
+        }
+      }
+
+      WHERE_NEXT;
+    }
+
+    if (listNode->pNodeList->length <= 0) {
+      return sclRewriteNullInOptr(pNode, ctx, node->opType);
+    }
+  }
+
   if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) {
     return sclRewriteNonConstOperator(pNode, ctx);
   }

source/libs/scalar/src/sclvector.c

@@ -793,7 +793,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
     return vectorConvertFromVarData(&cCtx, overflow);
   }
 
-  if (overflow) {
+  if (overflow && TSDB_DATA_TYPE_NULL != cCtx.inType) {
     if (1 != pIn->numOfRows) {
       sclError("invalid numOfRows %d", pIn->numOfRows);
       return TSDB_CODE_APP_ERROR;

tests/parallel_test/cases.task

@@ -560,6 +560,8 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/normal.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/normal.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py
@@ -739,6 +741,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/normal.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2
@@ -837,6 +840,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/normal.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3
@@ -934,6 +938,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/normal.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4

tests/system-test/2-query/normal.py

@@ -0,0 +1,143 @@
+from wsgiref.headers import tspecials
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        
+        self.dbname = "db"
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def inAndNotinTest(self):
+        dbname = self.dbname
+        
+        tdSql.query(f"select 1 in (1, 2)")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+        
+        tdSql.query(f"select 1 in (2, 3)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)
+        
+        tdSql.query(f"select 1 not in (2, 3)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True) 
+        
+        tdSql.query(f"select 1 not in (1)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)
+        
+        tdSql.query(f"select 1 in (1, null)")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+        
+        tdSql.query(f"select 1 in (2, null)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)   # 1 not in (2, null) is NULL?
+ 
+        tdSql.query(f"select 1 not in (1, null)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False) 
+               
+        tdSql.query(f"select 1 not in (2, null)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)    # 1 not in (2, null) is NULL?
+        
+        tdSql.query(f"select 1 not in (null)")  
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)    # 1 not in (null) is NULL?
+               
+        tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 int, col2 nchar(20)) tags(loc nchar(20))''')
+        tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')")
+        tdSql.execute(f"create table {dbname}.stb_2 using {dbname}.stb tags('shanghai')")
+        
+        for i in range(self.rowNum):
+            tdSql.execute(f"insert into {dbname}.stb_1 values({self.ts + i + 1}, {i+1}, 'taosdata_{i+1}')" )
+        for i in range(self.rowNum):
+            tdSql.execute(f"insert into {dbname}.stb_2 values({self.ts + i + 1}, {i+1}, 'taosdata_{i+1}')" )
+
+        tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 2) order by ts")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3) order by ts")
+        tdSql.checkRows(3)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 3)
+        tdSql.checkData(2, 1, 9)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3, 'xy') order by ts")
+        tdSql.checkRows(3)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 3)
+        tdSql.checkData(2, 1, 9)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, '9', 3) order by ts")
+        tdSql.checkRows(3)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 3)
+        tdSql.checkData(2, 1, 9)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3, null) order by ts")
+        tdSql.checkRows(3)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 3)
+        tdSql.checkData(2, 1, 9)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col2 in (1, 'taosdata_1', 3, null) order by ts")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col2  not in (1, 'taosdata_1', 3, null) order by ts")
+        tdSql.checkRows(0)
+        
+        tdSql.execute(f"insert into {dbname}.stb_1 values({self.ts + self.rowNum + 1}, {self.rowNum+1}, null)" )
+        tdSql.execute(f"insert into {dbname}.stb_2 values({self.ts + self.rowNum + 1}, {self.rowNum+1}, null)" )
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col2 in (1, 'taosdata_1', 3, null) order by ts")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+        
+        tdSql.query(f"select * from {dbname}.stb_1 where col2 not in (1, 'taosdata_1', 3, null) order by ts")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc in ('beijing', null)")
+        tdSql.checkRows(11)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc in ('shanghai', null)")
+        tdSql.checkRows(11)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc in ('shanghai', 'shanghai', null)")
+        tdSql.checkRows(11)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc in ('beijing', 'shanghai', null)")
+        tdSql.checkRows(22)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc not in ('beijing', null)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select * from {dbname}.stb where loc not in ('shanghai', 'shanghai', null)")
+        tdSql.checkRows(0)
+
+
+    def run(self):
+        dbname = "db"
+        tdSql.prepare()
+        
+        self.inAndNotinTest()
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/runAllOne.sh

@@ -243,6 +243,8 @@ python3 ./test.py -f 2-query/max.py -P
 python3 ./test.py -f 2-query/max.py -P -R
 python3 ./test.py -f 2-query/min.py -P
 python3 ./test.py -f 2-query/min.py -P -R
+python3 ./test.py -f 2-query/normal.py -P
+python3 ./test.py -f 2-query/normal.py -P -R
 python3 ./test.py -f 2-query/mode.py -P
 python3 ./test.py -f 2-query/mode.py -P -R
 python3 ./test.py -f 2-query/Now.py -P
@@ -424,6 +426,7 @@ python3 ./test.py -f 2-query/Now.py -P -Q 2
 python3 ./test.py -f 2-query/Today.py -P -Q 2
 python3 ./test.py -f 2-query/max.py -P -Q 2
 python3 ./test.py -f 2-query/min.py -P -Q 2
+python3 ./test.py -f 2-query/normal.py -P -Q 2
 python3 ./test.py -f 2-query/mode.py -P -Q 2
 python3 ./test.py -f 2-query/count.py -P -Q 2
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 2
@@ -522,6 +525,7 @@ python3 ./test.py -f 2-query/Now.py -P -Q 3
 python3 ./test.py -f 2-query/Today.py -P -Q 3
 python3 ./test.py -f 2-query/max.py -P -Q 3
 python3 ./test.py -f 2-query/min.py -P -Q 3
+python3 ./test.py -f 2-query/normal.py -P -Q 3
 python3 ./test.py -f 2-query/mode.py -P -Q 3
 python3 ./test.py -f 2-query/count.py -P -Q 3
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 3
@@ -619,6 +623,7 @@ python3 ./test.py -f 2-query/Now.py -P -Q 4
 python3 ./test.py -f 2-query/Today.py -P -Q 4
 python3 ./test.py -f 2-query/max.py -P -Q 4
 python3 ./test.py -f 2-query/min.py -P -Q 4
+python3 ./test.py -f 2-query/normal.py -P -Q 4
 python3 ./test.py -f 2-query/mode.py -P -Q 4
 python3 ./test.py -f 2-query/count.py -P -Q 4
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 4

tests/system-test/simpletest.bat

@@ -46,6 +46,7 @@ python3 .\test.py -f 2-query\between.py
 @REM python3 .\test.py -f 2-query\Today.py
 @REM python3 .\test.py -f 2-query\max.py
 @REM python3 .\test.py -f 2-query\min.py
+@REM python3 .\test.py -f 2-query\normal.py
 @REM python3 .\test.py -f 2-query\count.py
 @REM python3 .\test.py -f 2-query\last.py
 @REM python3 .\test.py -f 2-query\first.py

tests/system-test/win-test-file

@@ -382,6 +382,8 @@ python3 ./test.py -f 2-query/max.py
 python3 ./test.py -f 2-query/max.py -R
 python3 ./test.py -f 2-query/min.py
 python3 ./test.py -f 2-query/min.py -R
+python3 ./test.py -f 2-query/normal.py
+python3 ./test.py -f 2-query/normal.py -R
 python3 ./test.py -f 2-query/mode.py
 python3 ./test.py -f 2-query/mode.py -R
 python3 ./test.py -f 2-query/Now.py
@@ -550,6 +552,7 @@ python3 ./test.py -f 2-query/Now.py -Q 2
 python3 ./test.py -f 2-query/Today.py -Q 2
 python3 ./test.py -f 2-query/max.py -Q 2
 python3 ./test.py -f 2-query/min.py -Q 2
+python3 ./test.py -f 2-query/normal.py -Q 2
 python3 ./test.py -f 2-query/mode.py -Q 2
 python3 ./test.py -f 2-query/count.py -Q 2
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2
@@ -646,6 +649,7 @@ python3 ./test.py -f 2-query/Now.py -Q 3
 python3 ./test.py -f 2-query/Today.py -Q 3
 python3 ./test.py -f 2-query/max.py -Q 3
 python3 ./test.py -f 2-query/min.py -Q 3
+python3 ./test.py -f 2-query/normal.py -Q 3
 python3 ./test.py -f 2-query/mode.py -Q 3
 python3 ./test.py -f 2-query/count.py -Q 3
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3
@@ -742,6 +746,7 @@ python3 ./test.py -f 2-query/Now.py -Q 4
 python3 ./test.py -f 2-query/Today.py -Q 4
 python3 ./test.py -f 2-query/max.py -Q 4
 python3 ./test.py -f 2-query/min.py -Q 4
+python3 ./test.py -f 2-query/normal.py -Q 4
 python3 ./test.py -f 2-query/mode.py -Q 4
 python3 ./test.py -f 2-query/count.py -Q 4
 python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4