Merge pull request #26292 from taosdata/fix/TD-30620

Add index filtering based on column value type

source/libs/index/src/indexFilter.c

@@ -624,6 +624,23 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ
   }
   return 0;
 }
+
+static int8_t sifShouldUseIndexBasedOnType(SIFParam *left, SIFParam *right) {
+  // not compress
+  if (left->colValType == TSDB_DATA_TYPE_FLOAT) return 0;
+
+  if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY ||
+      left->colValType == TSDB_DATA_TYPE_JSON || right->colValType == TSDB_DATA_TYPE_JSON) {
+    return 0;
+  }
+
+  if (IS_VAR_DATA_TYPE(left->colValType)) {
+    if (!IS_VAR_DATA_TYPE(right->colValType)) return 0;
+  } else if (IS_NUMERIC_TYPE(left->colValType)) {
+    if (left->colValType != right->colValType) return 0;
+  }
+  return 1;
+}
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
   int             ret = 0;
   SIndexMetaArg  *arg = &output->arg;
@@ -641,8 +658,10 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
     ret = indexJsonSearch(arg->ivtIdx, mtm, output->result);
     indexMultiTermQueryDestroy(mtm);
   } else {
-    if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY) {
-      return TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR;
+    int8_t useIndex = sifShouldUseIndexBasedOnType(left, right);
+    if (!useIndex) {
+      output->status = SFLT_NOT_INDEX;
+      return -1;
     }
 
     bool       reverse = false, equal = false;
@@ -660,15 +679,12 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
 
     SDataTypeBuf typedata;
     memset(&typedata, 0, sizeof(typedata));
-    if (IS_VAR_DATA_TYPE(left->colValType)) {
-      if (!IS_VAR_DATA_TYPE(right->colValType)) {
-        NUM_TO_STRING(right->colValType, right->condValue, sizeof(buf) - 2, buf + VARSTR_HEADER_SIZE);
-        varDataSetLen(buf, strlen(buf + VARSTR_HEADER_SIZE));
-        param.val = buf;
-      }
-    } else {
-      if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
+
+    if (sifSetFltParam(left, right, &typedata, &param) != 0) {
+      output->status = SFLT_NOT_INDEX;
+      return -1;
     }
+
     ret = left->api.metaFilterTableIds(arg->metaEx, &param, output->result);
     if (ret == 0) {
       taosArraySort(output->result, uidCompare);

tests/parallel_test/cases.task

@@ -1488,12 +1488,14 @@
 ,,y,script,./test.sh -f tmp/monitor.sim
 ,,y,script,./test.sh -f tsim/tagindex/add_index.sim
 ,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim
+,,y,script,./test.sh -f tsim/tagindex/indexOverflow.sim
 ,,y,script,./test.sh -f tsim/view/view.sim
 ,,y,script,./test.sh -f tsim/query/cache_last.sim 
 ,,y,script,./test.sh -f tsim/query/const.sim 
 ,,y,script,./test.sh -f tsim/query/nestedJoinView.sim
 
 
+
 #develop test
 ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py
 ,,n,develop-test,python3 ./test.py -f 2-query/pseudo_column.py

tests/script/tsim/tagindex/indexOverflow.sim

@@ -0,0 +1,82 @@
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print ======== step0
+$dbPrefix = ta_3_db
+$tbPrefix = ta_3_tb
+$mtPrefix = ta_3_mt
+$lastRowNum = 0
+$tbNum =  100000 
+$rowNum = 20
+$totalNum = 200
+
+print =============== create database 
+sql create database $dbPrefix 
+sql use $dbPrefix 
+
+
+
+sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 tinyint, t1c tinyint)
+$i = 0
+$tinyLimit = 127
+$tinyTable = tinyTable
+while $i < $tinyLimit
+  $tb = $tinyTable . $i
+  sql insert into $tb using $mtPrefix tags( $i , $i ) values( now , $i ) 
+  $i = $i + 1
+endw
+
+$i = 0
+$maxTinyLimit = 200
+
+# 1. compress index and no-index to verify resultset  
+# 2. compress resultset of index filter and scalar filter 
+while $i < $maxTinyLimit
+  sql select * from $mtPrefix where t1 <= $i  
+  $lastRowNum = $rows
+  
+  sql select * from $mtPrefix where t1c <= $i 
+  if $lastRowNum != $rows then 
+    return -1
+  endi
+
+  $i = $i + 1
+endw
+
+
+$tbPrefix = ta_3_tb_c
+$mtPrefix = ta_3_mt_c
+$colPrefix = 'col'
+sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 nchar(18), t1c nchar(18))
+$i = 0
+$tinyLimit = 127
+while $i < $tinyLimit
+  $tb = $tbPrefix . $i
+  sql insert into $tb using $mtPrefix tags( $colPrefix , $colPrefix ) values( now , $i ) 
+  $i = $i + 1
+endw
+
+$i = 0
+$maxTinyLimit = 200
+
+# 1. compress index and no-index to verify resultset  
+# 2. compress resultset of index filter and scalar filter 
+while $i < $maxTinyLimit
+  sql select * from $mtPrefix where t1 <= $i  
+  $lastRowNum = $rows
+  
+  sql select * from $mtPrefix where t1c <= $i 
+  if $lastRowNum != $rows then 
+    return -1
+  endi
+
+  $i = $i + 1
+endw
+
+
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/win-test-file

@@ -401,6 +401,7 @@
 ./test.sh -f tsim/tag/tbNameIn.sim
 ./test.sh -f tmp/monitor.sim
 ./test.sh -f tsim/tagindex/add_index.sim
+./test.sh -f tsim/tagindex/indexOverflow.sim
 ./test.sh -f tsim/tagindex/sma_and_tag_index.sim
 ./test.sh -f tsim/view/view.sim
 ./test.sh -f tsim/query/cache_last.sim