diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d46f12cf77..a9ebcfd4c3 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -87,7 +87,7 @@ int32_t tqMetaOpen(STQ* pTq) { .reader = handle.execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, + .vnode = pTq->pVnode, }; handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader); ASSERT(handle.execHandle.execCol.task[i]); diff --git a/source/libs/index/inc/indexFstCommon.h b/source/libs/index/inc/indexFstCommon.h index 8335e437fb..e15df5dc34 100644 --- a/source/libs/index/inc/indexFstCommon.h +++ b/source/libs/index/inc/indexFstCommon.h @@ -4,6 +4,7 @@ #include "tutil.h" extern const uint8_t COMMON_INPUTS[]; extern const char COMMON_INPUTS_INV[]; +extern const int32_t COMMON_INPUTS_LEN; #ifdef __cplusplus extern "C" { diff --git a/source/libs/index/inc/indexFstDfa.h b/source/libs/index/inc/indexFstDfa.h index f6c220bcb7..9ca10897fd 100644 --- a/source/libs/index/inc/indexFstDfa.h +++ b/source/libs/index/inc/indexFstDfa.h @@ -29,16 +29,16 @@ extern "C" { typedef struct FstDfa FstDfa; typedef struct { - SArray * insts; + SArray *insts; uint32_t next[256]; bool isMatch; -} State; +} DfaState; /* * dfa builder related func **/ typedef struct FstDfaBuilder { - FstDfa * dfa; + FstDfa *dfa; SHashObj *cache; } FstDfaBuilder; diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 2bf9c9b791..2814b5dc16 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -65,6 +65,7 @@ typedef struct { } FstRegex; FstRegex *regexCreate(const char *str); +void regexDestroy(FstRegex *regex); uint32_t regexAutomStart(FstRegex *regex); bool regexAutomIsMatch(FstRegex *regex, uint32_t state); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 040e8ed830..05ce418037 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -22,7 +22,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 64 * 1024 +#define MEM_THRESHOLD 512 * 1024 #define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 #define MEM_ESTIMATE_RADIO 1.5 @@ -204,7 +204,6 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr if (0 == strcmp(c->colVal, pCt->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) @@ -309,7 +308,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 81ac4c9d40..c4b83f8a07 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1307,7 +1307,6 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) { taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - // void *eofState = sws->aut->acceptEof(nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); diff --git a/source/libs/index/src/indexFstCommon.c b/source/libs/index/src/indexFstCommon.c index 902e68ce09..0b20157009 100644 --- a/source/libs/index/src/indexFstCommon.c +++ b/source/libs/index/src/indexFstCommon.c @@ -294,3 +294,4 @@ const char COMMON_INPUTS_INV[] = { '\xee', '\xef', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', }; +const int32_t COMMON_INPUTS_LEN = sizeof(COMMON_INPUTS) / sizeof(COMMON_INPUTS[0]); diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 3011f124c9..b820f16a2a 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -41,7 +41,7 @@ FstDfaBuilder *dfaBuilderCreate(SArray *insts) { return NULL; } - SArray *states = taosArrayInit(4, sizeof(State)); + SArray *states = taosArrayInit(4, sizeof(DfaState)); builder->dfa = dfaCreate(insts, states); builder->cache = taosHashInit( @@ -71,7 +71,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { dfaAdd(builder->dfa, cur, 0); - SArray * states = taosArrayInit(0, sizeof(uint32_t)); + SArray *states = taosArrayInit(0, sizeof(uint32_t)); uint32_t result; if (dfaBuilderCachedState(builder, cur, &result)) { taosArrayPush(states, &result); @@ -98,10 +98,12 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { return builder->dfa; } +FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; } + bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte, uint32_t *result) { sparSetClear(cur); - State *t = taosArrayGet(builder->dfa->states, state); + DfaState *t = taosArrayGet(builder->dfa->states, state); for (int i = 0; i < taosArrayGetSize(t->insts); i++) { uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i); sparSetAdd(cur, ip); @@ -144,7 +146,7 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t * *result = *v; taosArrayDestroy(tinsts); } else { - State st; + DfaState st; st.insts = tinsts; st.isMatch = isMatch; taosArrayPush(builder->dfa->states, &st); @@ -169,14 +171,14 @@ bool dfaIsMatch(FstDfa *dfa, uint32_t si) { if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { return false; } - State *st = taosArrayGet(dfa->states, si); + DfaState *st = taosArrayGet(dfa->states, si); return st != NULL ? st->isMatch : false; } bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) { if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { return false; } - State *st = taosArrayGet(dfa->states, si); + DfaState *st = taosArrayGet(dfa->states, si); *result = st->next[byte]; return true; } diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index 33eeae802e..37cb58996f 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -23,7 +23,7 @@ FstRegex *regexCreate(const char *str) { return NULL; } int32_t sz = (int32_t)strlen(str); - char * orig = taosMemoryCalloc(1, sz); + char *orig = taosMemoryCalloc(1, sz); memcpy(orig, str, sz); regex->orig = orig; @@ -36,6 +36,12 @@ FstRegex *regexCreate(const char *str) { return regex; } +void regexDestroy(FstRegex *regex) { + if (regex == NULL) return; + taosMemoryFree(regex->orig); + taosMemoryFree(regex); +} + uint32_t regexAutomStart(FstRegex *regex) { ///// no nothing return 0; diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 040460ae5c..10becdff40 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(idxFstTest "") add_executable(idxFstUT "") add_executable(idxUtilUT "") add_executable(idxJsonUT "") +add_executable(idxFstUtilUT "") target_sources(idxTest PRIVATE @@ -26,6 +27,11 @@ target_sources(idxJsonUT PRIVATE "jsonUT.cc" ) +target_sources(idxFstUtilUT + PRIVATE + "fstUtilUT.cc" +) + target_include_directories (idxTest PUBLIC "${TD_SOURCE_DIR}/include/libs/index" @@ -54,6 +60,12 @@ target_include_directories (idxJsonUT "${TD_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories (idxFstUtilUT + PUBLIC + "${TD_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + target_link_libraries (idxTest os util @@ -91,6 +103,13 @@ target_link_libraries (idxJsonUT gtest_main index ) +target_link_libraries (idxFstUtilUT + os + util + common + gtest_main + index +) add_test( NAME idxtest @@ -108,3 +127,8 @@ add_test( NAME idxFstUT COMMAND idxFstUT ) +add_test( + NAME idxFstUtilUT + COMMAND idxFstUtilUT + +) diff --git a/source/libs/index/test/fstUtilUT.cc b/source/libs/index/test/fstUtilUT.cc new file mode 100644 index 0000000000..3a20661484 --- /dev/null +++ b/source/libs/index/test/fstUtilUT.cc @@ -0,0 +1,63 @@ + +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexCache.h" +#include "indexFst.h" +#include "indexFstDfa.h" +#include "indexFstRegex.h" +#include "indexFstSparse.h" +#include "indexFstUtil.h" +#include "indexInt.h" +#include "indexTfile.h" +#include "tglobal.h" +#include "tlog.h" +#include "tskiplist.h" +#include "tutil.h" +class FstUtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + SArray *inst = taosArrayInit(4, sizeof(char)); + builder = dfaBuilderCreate(inst); + } + virtual void TearDown() { dfaBuilderDestroy(builder); } + + FstDfaBuilder *builder; +}; + +class FstRegexEnv : public ::testing::Test { + protected: + virtual void SetUp() { regex = regexCreate("test"); } + virtual void TearDown() { regexDestroy(regex); } + FstRegex *regex; +}; + +class FstSparseSetEnv : public ::testing::Test { + protected: + virtual void SetUp() { set = sparSetCreate(256); } + virtual void TearDown() { sparSetDestroy(set); } + FstSparseSet *set; +}; + +// test FstDfaBuilder +TEST_F(FstUtilEnv, test1) {} +TEST_F(FstUtilEnv, test2) {} +TEST_F(FstUtilEnv, test3) {} +TEST_F(FstUtilEnv, test4) {} + +// test FstRegex + +TEST_F(FstRegexEnv, test1) {} +TEST_F(FstRegexEnv, test2) {} +TEST_F(FstRegexEnv, test3) {} +TEST_F(FstRegexEnv, test4) {} + +// test FstSparseSet +TEST_F(FstSparseSetEnv, test1) {} +TEST_F(FstSparseSetEnv, test2) {} +TEST_F(FstSparseSetEnv, test3) {} +TEST_F(FstSparseSetEnv, test4) {} diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index ab5128cd3e..323a6b4afa 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -8,6 +8,7 @@ #include "indexCache.h" #include "indexComm.h" #include "indexFst.h" +#include "indexFstCommon.h" #include "indexFstUtil.h" #include "indexInt.h" #include "indexTfile.h" @@ -356,3 +357,11 @@ TEST_F(UtilEnv, TempResultExcept) { idxTRsltMergeTo(relt, f); EXPECT_EQ(taosArrayGetSize(f), 1); } + +TEST_F(UtilEnv, testDictComm) { + int32_t count = COMMON_INPUTS_LEN; + for (int i = 0; i < 256; i++) { + uint8_t v = COMMON_INPUTS_INV[i]; + EXPECT_EQ(COMMON_INPUTS[v], i); + } +} diff --git a/source/util/src/ttrace.c b/source/util/src/ttrace.c deleted file mode 100644 index f183fd58fd..0000000000 --- a/source/util/src/ttrace.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "ttrace.h" -#include "taos.h" -#include "thash.h" -#include "tuuid.h" - -// clang-format off -//static TdThreadOnce init = PTHREAD_ONCE_INIT; -//static void * ids = NULL; -//static TdThreadMutex mtx; -// -//void traceInit() { -// ids = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); -// taosThreadMutexInit(&mtx, NULL); -//} -//void traceCreateEnv() { -// taosThreadOnce(&init, traceInit); -//} -//void traceDestroyEnv() { -// taosThreadMutexDestroy(&mtx); -// taosHashCleanup(ids); -//} -// -//STraceId traceInitId(STraceSubId *h, STraceSubId *l) { -// STraceId id = *h; -// id = ((id << 32) & 0xFFFFFFFF) | ((*l) & 0xFFFFFFFF); -// return id; -//} -//void traceId2Str(STraceId *id, char *buf) { -// int32_t f = (*id >> 32) & 0xFFFFFFFF; -// int32_t s = (*id) & 0xFFFFFFFF; -// sprintf(buf, "%d:%d", f, s); -//} -// -//void traceSetSubId(STraceId *id, STraceSubId *subId) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// taosHashPut(ids, subId, sizeof(*subId), &parent, sizeof(parent)); -// taosThreadMutexUnlock(&mtx); -//} -// -//STraceSubId traceGetParentId(STraceId *id) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// STraceSubId *p = taosHashGet(ids, (void *)&parent, sizeof(parent)); -// parent = *p; -// taosThreadMutexUnlock(&mtx); -// -// return parent; -//} -// -//STraceSubId traceGenSubId() { -// return tGenIdPI32(); -//} -//void traceSetRootId(STraceId *traceid, int64_t rootId) { -// traceId->rootId = rootId; -//} -//int64_t traceGetRootId(STraceId *traceId); -// -//void traceSetMsgId(STraceId *traceid, int64_t msgId); -//int64_t traceGetMsgId(STraceId *traceid); -// -//char *trace2Str(STraceId *id); -// -//void traceSetSubId(STraceId *id, int32_t *subId); -// clang-format on