From f0110af30ec8537bee226a1dff317122928d3d48 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Dec 2021 22:04:47 +0800 Subject: [PATCH] add index cache --- include/libs/index/index.h | 9 ++++ source/libs/index/inc/indexInt.h | 11 +++++ source/libs/index/src/index.c | 47 +++++++++++++++----- source/libs/index/src/index_fst.c | 2 +- source/libs/index/src/index_fst_automation.c | 11 ++++- 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 1b74928568..0885ce151e 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -28,6 +28,15 @@ typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef enum { + ADD_VALUE, // add index colume value + DEL_VALUE, // delete index column value + UPDATE_VALUE, // update index column value + ADD_INDEX, // add index on specify column + DROP_INDEX, // drop existed index + DROP_SATBLE // drop stable +} SIndexColumnType; + typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; /* diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a6862c05c8..f6ff9bc139 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -17,7 +17,10 @@ #define _TD_INDEX_INT_H_ #include "index.h" +#include "index_fst.h" #include "tlog.h" +#include "thash.h" +#include "taos.h" #ifdef USE_LUCENE #include @@ -32,12 +35,20 @@ struct SIndex { #ifdef USE_LUCENE index_t *index; #endif + void *cache; + void *tindex; + SHashObj *fieldObj; // + uint64_t suid; + int fieldId; + pthread_mutex_t mtx; }; struct SIndexOpts { #ifdef USE_LUCENE void *opts; #endif + int32_t numOfItermLimit; + int8_t mergeInterval; }; struct SIndexMultiTermQuery { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f8f4311a4a..c011411189 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -15,39 +15,58 @@ #include "index.h" #include "indexInt.h" +#include "index_cache.h" #ifdef USE_LUCENE #include "lucene++/Lucene_c.h" #endif -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +typedef struct SIdxFieldInfo { + int id; // generated by index internal + int type; // field type +} SIdxFieldInfo; + +static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); +static int indexMergeCacheIntoTindex(struct SIndex *sIdx) { + if (sIdx == NULL) { + return -1; + } + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + return 0; +} SIndex *indexOpen(SIndexOpts *opts, const char *path) { pthread_once(&isInit, indexInit); + SIndex *sIdx = malloc(sizeof(SIndex)); + #ifdef USE_LUCENE index_t *index = index_open(path); - SIndex *p = malloc(sizeof(SIndex)); - p->index = index; - return p; + sIdx->index = index; #endif - return NULL; + + sIdx->cache = (void*)indexCacheCreate(); + sIdx->tindex = NULL; + sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pthread_mutex_init(&sIdx->mtx, NULL); + return sIdx; } -void indexClose(SIndex *index) { +void indexClose(SIndex *sIdx) { #ifdef USE_LUCENE - index_close(index->index); - index->index = NULL; + index_close(sIdex->index); + sIdx->index = NULL; #endif - free(index); + indexCacheDestroy(sIdx->cache); + taosHashCleanup(sIdx->fieldObj); + pthread_mutex_destroy(&sIdx->mtx); + free(sIdx); return; - } -#ifdef USE_LUCENE -#endif int indexPut(SIndex *index, SArray* field_vals, int uid) { + #ifdef USE_LUCENE index_document_t *doc = index_document_create(); @@ -63,6 +82,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) { index_put(index->index, doc); index_document_destroy(doc); #endif + pthread_mutex_lock(&index->mtx); + pthread_mutex_unlock(&index->mtx); return 1; } @@ -105,7 +126,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result return 1; } + int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { + return 1; } int indexRebuild(SIndex *index, SIndexOpts *opts); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 9cb4ac6836..7aaa498864 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1330,7 +1330,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); - if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { + if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index f70b90041b..07ad45079b 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -87,9 +87,18 @@ static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { if (ssv == NULL || ctx == NULL) {return NULL;} char *data = ctx->data; + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; - return startWithStateValueCreate(Running, FST_INT, &val); + StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + if (prefixIsMatch(ctx, nsv)) { + nsv->kind = Done; + } else { + nsv->kind = Running; + } + return nsv; } return NULL; }