add test case

source/dnode/vnode/src/tq/tqMeta.c

@@ -87,7 +87,7 @@ int32_t tqMetaOpen(STQ* pTq) {
             .reader = handle.execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
             .pMsgCb = &pTq->pVnode->msgCb,
-            .vnode  = pTq->pVnode,
+            .vnode = pTq->pVnode,
         };
         handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
         ASSERT(handle.execHandle.execCol.task[i]);

source/libs/index/inc/indexFstCommon.h

@@ -4,6 +4,7 @@
 #include "tutil.h"
 extern const uint8_t COMMON_INPUTS[];
 extern const char    COMMON_INPUTS_INV[];
+extern const int32_t COMMON_INPUTS_LEN;
 
 #ifdef __cplusplus
 extern "C" {

source/libs/index/inc/indexFstDfa.h

@@ -29,16 +29,16 @@ extern "C" {
 typedef struct FstDfa FstDfa;
 
 typedef struct {
-  SArray * insts;
+  SArray  *insts;
   uint32_t next[256];
   bool     isMatch;
-} State;
+} DfaState;
 
 /*
  * dfa builder related func
  **/
 typedef struct FstDfaBuilder {
-  FstDfa *  dfa;
+  FstDfa   *dfa;
   SHashObj *cache;
 } FstDfaBuilder;
 

source/libs/index/inc/indexFstRegex.h

@@ -65,6 +65,7 @@ typedef struct {
 } FstRegex;
 
 FstRegex *regexCreate(const char *str);
+void      regexDestroy(FstRegex *regex);
 
 uint32_t regexAutomStart(FstRegex *regex);
 bool     regexAutomIsMatch(FstRegex *regex, uint32_t state);

source/libs/index/src/indexCache.c

@@ -22,7 +22,7 @@
 #define MAX_INDEX_KEY_LEN 256  // test only, change later
 
 #define MEM_TERM_LIMIT     10 * 10000
-#define MEM_THRESHOLD      64 * 1024
+#define MEM_THRESHOLD      512 * 1024
 #define MEM_SIGNAL_QUIT    MEM_THRESHOLD * 20
 #define MEM_ESTIMATE_RADIO 1.5
 
@@ -204,7 +204,6 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
     if (0 == strcmp(c->colVal, pCt->colVal)) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
@@ -309,7 +308,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)

source/libs/index/src/indexFst.c

@@ -1307,7 +1307,6 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
     taosArrayPush(sws->inp, &(trn.inp));
 
     if (FST_NODE_IS_FINAL(nextNode)) {
-      // void *eofState = sws->aut->acceptEof(nextState);
       void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
       if (eofState != NULL) {
         isMatch = automFuncs[aut->type].isMatch(aut, eofState);

source/libs/index/src/indexFstCommon.c

@@ -294,3 +294,4 @@ const char COMMON_INPUTS_INV[] = {
     '\xee', '\xef', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb',
     '\xfc', '\xfd', '\xfe', '\xff',
 };
+const int32_t COMMON_INPUTS_LEN = sizeof(COMMON_INPUTS) / sizeof(COMMON_INPUTS[0]);

source/libs/index/src/indexFstDfa.c

@@ -41,7 +41,7 @@ FstDfaBuilder *dfaBuilderCreate(SArray *insts) {
     return NULL;
   }
 
-  SArray *states = taosArrayInit(4, sizeof(State));
+  SArray *states = taosArrayInit(4, sizeof(DfaState));
 
   builder->dfa = dfaCreate(insts, states);
   builder->cache = taosHashInit(
@@ -71,7 +71,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
 
   dfaAdd(builder->dfa, cur, 0);
 
-  SArray * states = taosArrayInit(0, sizeof(uint32_t));
+  SArray  *states = taosArrayInit(0, sizeof(uint32_t));
   uint32_t result;
   if (dfaBuilderCachedState(builder, cur, &result)) {
     taosArrayPush(states, &result);
@@ -98,10 +98,12 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
   return builder->dfa;
 }
 
+FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; }
+
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
                         uint32_t *result) {
   sparSetClear(cur);
-  State *t = taosArrayGet(builder->dfa->states, state);
+  DfaState *t = taosArrayGet(builder->dfa->states, state);
   for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
     uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
     sparSetAdd(cur, ip);
@@ -144,7 +146,7 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *
     *result = *v;
     taosArrayDestroy(tinsts);
   } else {
-    State st;
+    DfaState st;
     st.insts = tinsts;
     st.isMatch = isMatch;
     taosArrayPush(builder->dfa->states, &st);
@@ -169,14 +171,14 @@ bool dfaIsMatch(FstDfa *dfa, uint32_t si) {
   if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
     return false;
   }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
   return st != NULL ? st->isMatch : false;
 }
 bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) {
   if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
     return false;
   }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
   *result = st->next[byte];
   return true;
 }

source/libs/index/src/indexFstRegex.c

@@ -23,7 +23,7 @@ FstRegex *regexCreate(const char *str) {
     return NULL;
   }
   int32_t sz = (int32_t)strlen(str);
-  char *  orig = taosMemoryCalloc(1, sz);
+  char   *orig = taosMemoryCalloc(1, sz);
   memcpy(orig, str, sz);
 
   regex->orig = orig;
@@ -36,6 +36,12 @@ FstRegex *regexCreate(const char *str) {
   return regex;
 }
 
+void regexDestroy(FstRegex *regex) {
+  if (regex == NULL) return;
+  taosMemoryFree(regex->orig);
+  taosMemoryFree(regex);
+}
+
 uint32_t regexAutomStart(FstRegex *regex) {
   ///// no nothing
   return 0;

source/libs/index/test/CMakeLists.txt

@@ -3,6 +3,7 @@ add_executable(idxFstTest "")
 add_executable(idxFstUT "")
 add_executable(idxUtilUT "")
 add_executable(idxJsonUT "")
+add_executable(idxFstUtilUT "")
 
 target_sources(idxTest
   PRIVATE 
@@ -26,6 +27,11 @@ target_sources(idxJsonUT
   PRIVATE 
   "jsonUT.cc" 
 )
+target_sources(idxFstUtilUT
+  PRIVATE 
+  "fstUtilUT.cc" 
+)
+ 
 target_include_directories (idxTest
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
@@ -54,6 +60,12 @@ target_include_directories (idxJsonUT
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
+target_include_directories (idxFstUtilUT
+  PUBLIC
+  "${TD_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+) 
+
 target_link_libraries (idxTest
   os  
   util
@@ -91,6 +103,13 @@ target_link_libraries (idxJsonUT
   gtest_main
   index
 )
+target_link_libraries (idxFstUtilUT
+  os  
+  util
+  common
+  gtest_main
+  index
+)
 
 add_test(
   NAME idxtest
@@ -108,3 +127,8 @@ add_test(
   NAME idxFstUT 
   COMMAND idxFstUT 
 )
+add_test(
+  NAME idxFstUtilUT 
+  COMMAND idxFstUtilUT  
+
+)

source/libs/index/test/fstUtilUT.cc

@@ -0,0 +1,63 @@
+
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstDfa.h"
+#include "indexFstRegex.h"
+#include "indexFstSparse.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "tskiplist.h"
+#include "tutil.h"
+class FstUtilEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    SArray *inst = taosArrayInit(4, sizeof(char));
+    builder = dfaBuilderCreate(inst);
+  }
+  virtual void TearDown() { dfaBuilderDestroy(builder); }
+
+  FstDfaBuilder *builder;
+};
+
+class FstRegexEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { regex = regexCreate("test"); }
+  virtual void TearDown() { regexDestroy(regex); }
+  FstRegex    *regex;
+};
+
+class FstSparseSetEnv : public ::testing::Test {
+ protected:
+  virtual void  SetUp() { set = sparSetCreate(256); }
+  virtual void  TearDown() { sparSetDestroy(set); }
+  FstSparseSet *set;
+};
+
+// test FstDfaBuilder
+TEST_F(FstUtilEnv, test1) {}
+TEST_F(FstUtilEnv, test2) {}
+TEST_F(FstUtilEnv, test3) {}
+TEST_F(FstUtilEnv, test4) {}
+
+// test FstRegex
+
+TEST_F(FstRegexEnv, test1) {}
+TEST_F(FstRegexEnv, test2) {}
+TEST_F(FstRegexEnv, test3) {}
+TEST_F(FstRegexEnv, test4) {}
+
+// test FstSparseSet
+TEST_F(FstSparseSetEnv, test1) {}
+TEST_F(FstSparseSetEnv, test2) {}
+TEST_F(FstSparseSetEnv, test3) {}
+TEST_F(FstSparseSetEnv, test4) {}

source/libs/index/test/utilUT.cc

@@ -8,6 +8,7 @@
 #include "indexCache.h"
 #include "indexComm.h"
 #include "indexFst.h"
+#include "indexFstCommon.h"
 #include "indexFstUtil.h"
 #include "indexInt.h"
 #include "indexTfile.h"
@@ -356,3 +357,11 @@ TEST_F(UtilEnv, TempResultExcept) {
   idxTRsltMergeTo(relt, f);
   EXPECT_EQ(taosArrayGetSize(f), 1);
 }
+
+TEST_F(UtilEnv, testDictComm) {
+  int32_t count = COMMON_INPUTS_LEN;
+  for (int i = 0; i < 256; i++) {
+    uint8_t v = COMMON_INPUTS_INV[i];
+    EXPECT_EQ(COMMON_INPUTS[v], i);
+  }
+}

source/util/src/ttrace.c

@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#include "ttrace.h"
-#include "taos.h"
-#include "thash.h"
-#include "tuuid.h"
-
-// clang-format off
-//static TdThreadOnce init = PTHREAD_ONCE_INIT;
-//static void *              ids = NULL;
-//static TdThreadMutex       mtx;
-//
-//void traceInit() {
-//  ids = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-//  taosThreadMutexInit(&mtx, NULL);
-//}
-//void traceCreateEnv() {
-//  taosThreadOnce(&init, traceInit);
-//}
-//void traceDestroyEnv() {
-//  taosThreadMutexDestroy(&mtx);
-//  taosHashCleanup(ids);
-//}
-//
-//STraceId traceInitId(STraceSubId *h, STraceSubId *l) {
-//  STraceId id = *h; 
-//  id = ((id << 32) & 0xFFFFFFFF) |  ((*l) & 0xFFFFFFFF);  
-//  return id;
-//}
-//void traceId2Str(STraceId *id, char *buf) {
-//  int32_t f = (*id >> 32) & 0xFFFFFFFF;
-//  int32_t s = (*id) & 0xFFFFFFFF;
-//  sprintf(buf, "%d:%d", f, s); 
-//}
-//
-//void traceSetSubId(STraceId *id, STraceSubId *subId) {
-//  int32_t parent = ((*id >> 32) & 0xFFFFFFFF); 
-//  taosThreadMutexLock(&mtx);    
-//  taosHashPut(ids, subId, sizeof(*subId), &parent, sizeof(parent));
-//  taosThreadMutexUnlock(&mtx);    
-//}
-//
-//STraceSubId traceGetParentId(STraceId *id) {
-//  int32_t parent = ((*id >> 32) & 0xFFFFFFFF);   
-//  taosThreadMutexLock(&mtx);    
-//  STraceSubId *p = taosHashGet(ids, (void *)&parent, sizeof(parent));
-//  parent = *p;
-//  taosThreadMutexUnlock(&mtx);    
-//
-//  return parent;  
-//}
-//
-//STraceSubId  traceGenSubId() {
-//  return tGenIdPI32();
-//}
-//void    traceSetRootId(STraceId *traceid, int64_t rootId) {
-//  traceId->rootId = rootId;
-//}
-//int64_t traceGetRootId(STraceId *traceId);
-//
-//void    traceSetMsgId(STraceId *traceid, int64_t msgId);
-//int64_t traceGetMsgId(STraceId *traceid);
-//
-//char *trace2Str(STraceId *id);
-//
-//void traceSetSubId(STraceId *id, int32_t *subId);
-// clang-format on