Merge pull request #12545 from taosdata/fix/indexConditionError

fix: index_condition _filter_error

source/libs/index/inc/indexComm.h

@@ -33,11 +33,17 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
 typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
 
-TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b);
+TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType);
+TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
 _cache_range_compare indexGetCompare(RangeType ty);
 
 int32_t indexConvertData(void* src, int8_t type, void** dst);
+int32_t indexConvertDataToStr(void* src, int8_t type, void** dst);
+
+int32_t indexGetDataByteLen(int8_t type);
+
+char* indexInt2str(int64_t val, char* dst, int radix);
 
 #ifdef __cplusplus
 }

source/libs/index/src/index.c

@@ -109,17 +109,15 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
   taosThreadMutexInit(&sIdx->mtx, NULL);
 
   sIdx->refId = indexAddRef(sIdx);
-  taosAcquireRef(indexRefMgt, sIdx->refId);
+  indexAcquireRef(sIdx->refId);
 
   *index = sIdx;
-
   return 0;
 
 END:
   if (sIdx != NULL) {
     indexClose(sIdx);
   }
-
   *index = NULL;
   return -1;
 }
@@ -273,7 +271,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
   tm->nColName = nColName;
 
   char*   buf = NULL;
-  int32_t len = indexConvertData((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
+  int32_t len = indexConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
   assert(len != -1);
 
   tm->colVal = buf;

source/libs/index/src/indexCache.c

@@ -282,8 +282,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
     if (0 != strncmp(c->colVal, pCt->colVal, skip)) {
       break;
     }
+    char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
+    memcpy(p, c->colVal, strlen(c->colVal));
 
-    TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType);
+    TExeCond cond = cmpFn(p + skip, term->colVal, dType);
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
@@ -297,6 +299,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
     } else if (cond == BREAK) {
       break;
     }
+    taosMemoryFree(p);
   }
 
   taosMemoryFree(pCt);
@@ -463,7 +466,6 @@ int indexCacheSchedToMerge(IndexCache* pCache) {
   // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t));
   // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t));
   schedMsg.msg = NULL;
-
   indexAcquireRef(pCache->index->refId);
   taosScheduleTask(indexQhandle, &schedMsg);
 

source/libs/index/src/indexComm.c

@@ -19,10 +19,38 @@
 #include "tcoding.h"
 #include "tcompare.h"
 #include "tdataformat.h"
+#include "ttypes.h"
 
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
+char* indexInt2str(int64_t val, char* dst, int radix) {
+  char     buffer[65];
+  char*    p;
+  int64_t  new_val;
+  uint64_t uval = (uint64_t)val;
+
+  if (radix < 0) {
+    if (val < 0) {
+      *dst++ = '-';
+      uval = (uint64_t)0 - uval; /* Avoid integer overflow in (-val) for LLONG_MIN (BUG#31799). */
+    }
+  }
+  p = &buffer[sizeof(buffer) - 1];
+  *p = '\0';
+  new_val = (int64_t)(uval / 10);
+  *--p = '0' + (char)(uval - (uint64_t)new_val * 10);
+  val = new_val;
+
+  while (val != 0) {
+    new_val = val / 10;
+    *--p = '0' + (char)(val - new_val * 10);
+    val = new_val;
+  }
+  while ((*dst++ = *p++) != 0)
+    ;
+  return dst - 1;
+}
 static __compar_fn_t indexGetCompar(int8_t type) {
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
     return (__compar_fn_t)strcmp;
@@ -31,25 +59,49 @@ static __compar_fn_t indexGetCompar(int8_t type) {
 }
 static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_LESS_THAN, a, b);
+  return tCompare(func, QUERY_LESS_THAN, a, b, type);
 }
 static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
+  return tCompare(func, QUERY_LESS_EQUAL, a, b, type);
 }
 static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
+  return tCompare(func, QUERY_GREATER_THAN, a, b, type);
 }
 static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
-  return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
+  return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
-
+TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
-TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
+  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
+    return tDoCompare(func, cmptype, a, b);
+  }
+#if 1
+  int8_t bytes = tDataTypes[dtype].bytes;
+  if (bytes == 1) {
+    int8_t va = taosStr2int64(a);
+    int8_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (bytes == 2) {
+    int16_t va = taosStr2int64(a);
+    int16_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (bytes == 4) {
+    int32_t va = taosStr2int64(a);
+    int32_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else {
+    int64_t va = taosStr2int64(a);
+    int64_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  }
+#endif
+}
+TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
   // optime later
   int32_t ret = func(a, b);
-  switch (comType) {
+  switch (comparType) {
     case QUERY_LESS_THAN: {
       if (ret < 0) return MATCH;
     } break;
@@ -174,9 +226,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       tlen = taosEncodeFixedU32(dst, *(uint32_t*)src);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src);
+      tlen = taosEncodeFixedI64(NULL, *(int64_t*)src);
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeFixedI64(dst, *(uint32_t*)src);
+      tlen = taosEncodeFixedI64(dst, *(int64_t*)src);
       break;
     case TSDB_DATA_TYPE_DOUBLE:
       tlen = taosEncodeBinary(NULL, src, sizeof(double));
@@ -184,9 +236,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       tlen = taosEncodeBinary(dst, src, sizeof(double));
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src);
+      tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src);
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeFixedU64(dst, *(uint32_t*)src);
+      tlen = taosEncodeFixedU64(dst, *(uint64_t*)src);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
@@ -215,14 +267,94 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       break;
   }
   *dst = *dst - tlen;
-  if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY &&
+  // indexMayFillNumbericData(*dst, tlen);
-      type == TSDB_DATA_TYPE_VARCHAR) {
+  return tlen;
-    uint8_t* p = *dst;
+}
-    for (int i = 0; i < tlen; i++) {
+int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
-      if (p[i] == 0) {
+  int tlen = tDataTypes[type].bytes;
-        p[i] = (uint8_t)'0';
+
+  switch (type) {
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int64_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_UTINYINT:
+      // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
+      //*dst = taosMemoryCalloc(1, tlen + 1);
+      // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint8_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int8_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int16_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint16_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int32_t*)src, *dst, -1);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      tlen = taosEncodeBinary(NULL, src, sizeof(float));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, sizeof(float));
+      *dst = *dst - tlen;
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint32_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(int64_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      tlen = taosEncodeBinary(NULL, src, sizeof(double));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, sizeof(double));
+      *dst = *dst - tlen;
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      assert(0);
+      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      indexInt2str(*(uint64_t*)src, *dst, 1);
+      break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
+      *dst = *dst - tlen;
+
+      break;
     }
+    case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
+#if 1
+      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, strlen(src));
+      *dst = *dst - tlen;
+      break;
+#endif
     }
+    case TSDB_DATA_TYPE_VARBINARY:
+#if 1
+      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      *dst = taosMemoryCalloc(1, tlen + 1);
+      tlen = taosEncodeBinary(dst, src, strlen(src));
+      *dst = *dst - tlen;
+      break;
+#endif
+    default:
+      TASSERT(0);
+      break;
   }
   return tlen;
 }

source/libs/index/src/indexFst.c

@@ -1324,7 +1324,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
         taosArrayPop(sws->inp);
       }
-      streamStateDestroy(p);
+      // streamStateDestroy(p);
       continue;
     }
     FstTransition trn;

source/libs/index/src/indexTfile.c

@@ -410,8 +410,9 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult*
 
     ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
     cost = taosGetTimestampUs() - et;
-    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
+    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, offset: %" PRIu64
-              tem->colName, tem->colVal, cost);
+              ", size: %d, time cost: %" PRIu64 "us",
+              tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost);
   }
   fstSliceDestroy(&key);
   return 0;
@@ -941,7 +942,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
   // TODO(yihao): opt later
   WriterCtx* ctx = reader->ctx;
   // add block cache
-  char    block[1024] = {0};
+  char    block[4096] = {0};
   int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
   assert(nread >= sizeof(uint32_t));
 

source/libs/index/test/jsonUT.cc

@@ -56,6 +56,29 @@ class JsonEnv : public ::testing::Test {
   SIndexJson*     index;
 };
 
+static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
+                      int8_t operType = ADD_VALUE) {
+  SIndexTerm* term =
+      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexMultiTerm* terms = indexMultiTermCreate();
+  indexMultiTermAdd(terms, term);
+  tIndexJsonPut(index, terms, (int64_t)tableId);
+
+  indexMultiTermDestroy(terms);
+}
+static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType,
+                   SArray** result) {
+  std::string colName(colNam);
+
+  SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
+  SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+
+  SArray* res = taosArrayInit(1, sizeof(uint64_t));
+  indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
+  tIndexJsonSearch(index, mq, res);
+  indexMultiTermQueryDestroy(mq);
+  *result = res;
+}
 TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
@@ -204,9 +227,10 @@ TEST_F(JsonEnv, testWriteMillonData) {
 TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test");
-    std::string colVal("10");
+    // std::string colVal("10");
+    int         val = 10;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -217,35 +241,9 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test2");
-    std::string colVal("20");
+    int         val = 20;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
-
-    SIndexMultiTerm* terms = indexMultiTermCreate();
-    indexMultiTermAdd(terms, term);
-    for (size_t i = 0; i < 1000; i++) {
-      tIndexJsonPut(index, terms, i);
-    }
-    indexMultiTermDestroy(terms);
-  }
-  {
-    std::string colName("test2");
-    std::string colVal("15");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
-
-    SIndexMultiTerm* terms = indexMultiTermCreate();
-    indexMultiTermAdd(terms, term);
-    for (size_t i = 0; i < 1000; i++) {
-      tIndexJsonPut(index, terms, i);
-    }
-    indexMultiTermDestroy(terms);
-  }
-  {
-    std::string colName("test2");
-    std::string colVal("15");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -256,11 +254,36 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 15;
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                       (const char*)&val, sizeof(val));
 
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string colName("test2");
+    const char* val = "test";
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+                                       (const char*)val, strlen(val));
+
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string           colName("test");
+    int                   val = 15;
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -270,11 +293,11 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 15;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -284,11 +307,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    ;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -298,11 +322,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -312,11 +337,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
   {
     std::string colName("test");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -326,12 +352,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   }
 }
 
-TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -355,11 +381,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -369,11 +395,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -383,11 +409,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    // std::string colVal("10");
+    int val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -397,11 +424,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -411,11 +438,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -425,9 +452,10 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("other_column");
-    std::string colVal("100");
+    int         val = 100;
+
     SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                       (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -438,11 +466,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
   }
   {
     std::string colName("test1");
-    std::string colVal("10");
+    int         val = 10;
+    // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    colVal.size());
+                                    (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -450,4 +479,102 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
+  {
+    std::string colName("test1");
+    int         val = 15;
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                       (const char*)&val, sizeof(val));
+
+    SIndexMultiTerm* terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+    for (size_t i = 0; i < 1000; i++) {
+      tIndexJsonPut(index, terms, i + 1000);
+    }
+    indexMultiTermDestroy(terms);
+  }
+  {
+    std::string colName("test1");
+    int         val = 8;
+    // std::string colVal("10");
+
+    SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+                                    (const char*)&val, sizeof(val));
+
+    SArray* result = taosArrayInit(1, sizeof(uint64_t));
+    indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
+    tIndexJsonSearch(index, mq, result);
+    EXPECT_EQ(2000, taosArrayGetSize(result));
+    indexMultiTermQueryDestroy(mq);
+  }
+}
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT2) {
+  {
+    int         val = 10;
+    std::string colName("test1");
+    for (int i = 0; i < 10000; i++) {
+      val += 1;
+      WriteData(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), i);
+    }
+  }
+  {
+    int         val = 10;
+    std::string colName("test2xxx");
+    std::string colVal("xxxxxxxxxxxxxxx");
+    for (int i = 0; i < 100000; i++) {
+      val += 1;
+      WriteData(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), i);
+    }
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    int         val = 9;
+    Search(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(10000, taosArrayGetSize(res));
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test2xxx");
+    std::string colVal("xxxxxxxxxxxxxxx");
+    Search(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), QUERY_TERM, &res);
+    EXPECT_EQ(100000, taosArrayGetSize(res));
+  }
+}
+TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
+  {
+    float       val = 10.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+    }
+  }
+  {
+    float       val = 2.0;
+    std::string colName("test1");
+    for (int i = 0; i < 1000; i++) {
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+    }
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    float       val = 1.9;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(2000, taosArrayGetSize(res));
+  }
+  {
+    SArray*     res = NULL;
+    std::string colName("test1");
+    float       val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
+  {
+    std::string colName("test1");
+    SArray*     res = NULL;
+    float       val = 2.1;
+    Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
+    EXPECT_EQ(1000, taosArrayGetSize(res));
+  }
 }

source/libs/index/test/utilUT.cc

@@ -6,12 +6,14 @@
 #include <vector>
 #include "index.h"
 #include "indexCache.h"
+#include "indexComm.h"
 #include "indexFst.h"
 #include "indexFstCountingWriter.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
 #include "indexUtil.h"
+#include "tcoding.h"
 #include "tglobal.h"
 #include "tskiplist.h"
 #include "tutil.h"
@@ -305,3 +307,17 @@ TEST_F(UtilEnv, 01Except) {
   ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1);
   ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100);
 }
+TEST_F(UtilEnv, testFill) {
+  for (int i = 0; i < 10000000; i++) {
+    int64_t val = i;
+    char    buf[65] = {0};
+    indexInt2str(val, buf, 1);
+    EXPECT_EQ(val, taosStr2int64(buf));
+  }
+  for (int i = 0; i < 10000000; i++) {
+    int64_t val = 0 - i;
+    char    buf[65] = {0};
+    indexInt2str(val, buf, -1);
+    EXPECT_EQ(val, taosStr2int64(buf));
+  }
+}