From 76f70feb49a44b51460bd9a24279966fabcbee57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 20 Dec 2021 09:34:28 +0800 Subject: [PATCH] add index TFile --- source/libs/index/inc/indexInt.h | 2 ++ source/libs/index/inc/index_cache.h | 2 +- source/libs/index/inc/index_tfile.h | 46 ++++++++++++++++++++++++++ source/libs/index/src/index.c | 50 ++++++++++++++++++++++++++--- source/libs/index/src/index_cache.c | 6 ++-- source/libs/index/src/index_tfile.c | 31 ++++++++++++++++++ 6 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 source/libs/index/inc/index_tfile.h create mode 100644 source/libs/index/src/index_tfile.c diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a96c8eecf4..a258cb834d 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -31,6 +31,8 @@ extern "C" { #endif +typedef enum {kTypeValue, kTypeDeletion} STermValueType ; + typedef struct SIndexStat { int32_t totalAdded; // int32_t totalDeled; // diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index e3b7ae273a..97a7b835f6 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -44,7 +44,7 @@ void indexCacheDestroy(void *cache); int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); //int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result); +int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h new file mode 100644 index 0000000000..c3f4bd25e5 --- /dev/null +++ b/source/libs/index/inc/index_tfile.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_TFILE_H__ +#define __INDEX_TFILE_H__ + +#include "index.h" +#include "indexInt.h" +#include "tlockfree.h" +#include "tskiplist.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct IndexTFile { + T_REF_DECLARE() +} IndexTFile; + + + +IndexTFile *indexTFileCreate(); + + +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); + +#ifdef __cplusplus +} + + +#endif + + + +#endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e6c655d097..ec83e84a3b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -16,13 +16,19 @@ #include "index.h" #include "indexInt.h" #include "index_cache.h" +#include "index_tfile.h" #include "tdef.h" #ifdef USE_LUCENE #include "lucene++/Lucene_c.h" #endif - +static int uidCompare(const void *a, const void *b) { + uint64_t u1 = *(uint64_t *)a; + uint64_t u2 = *(uint64_t *)b; + if (u1 == u2) { return 0; } + else { return u1 < u2 ? -1 : 1; } +} typedef struct SIdxColInfo { int colId; // generated by index internal int cVersion; @@ -273,9 +279,44 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) { void indexInit() { //do nothing } -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result) { +static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { + int32_t version = -1; + int16_t colId = -1; + SIdxColInfo *colInfo = NULL; + + SIndexTerm *term = query->term; + const char *colName = term->colName; + int32_t nColName = term->nColName; + + pthread_mutex_lock(&sIdx->mtx); + colInfo = taosHashGet(sIdx->colObj, colName, nColName); + if (colInfo == NULL) { + pthread_mutex_unlock(&sIdx->mtx); + return -1; + } + colId = colInfo->colId; + version = colInfo->cVersion; + pthread_mutex_unlock(&sIdx->mtx); - return 0; + *result = taosArrayInit(4, sizeof(uint64_t)); + //TODO: iterator mem and tidex + STermValueType s; + if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { + if (s == kTypeDeletion) { + indexInfo("col: %s already drop by other opera", term->colName); + // coloum already drop by other oper, no need to query tindex + return 0; + } else { + if (0 != indexTFileSearch(sIdx->tindex, query, *result)) { + indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); + return -1; + } + } + } else { + indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal); + return -1; + } + return 0; } static void indexInterResultsDestroy(SArray *results) { if (results == NULL) { return; } @@ -291,8 +332,7 @@ static void indexInterResultsDestroy(SArray *results) { static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { //refactor, merge interResults into fResults by oType SArray *first = taosArrayGetP(interResults, 0); - - //taosArraySort(first, getCom) + taosArraySort(first, uidCompare); if (oType == MUST) { } else if (oType == SHOULD) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 5a23278908..3c52275a4c 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -46,7 +46,7 @@ static int32_t compareKey(const void *l, const void *r) { rp += sizeof(rf); // compare field type - int16_t lft, rft; + int8_t lft, rft; memcpy(&lft, lp, sizeof(lft)); memcpy(&rft, rp, sizeof(rft)); lp += sizeof(lft); @@ -149,7 +149,7 @@ int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t IndexCache *pCache = cache; return 0; } -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result) { +int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { if (cache == NULL) { return -1; } IndexCache *pCache = cache; SIndexTerm *term = query->term; @@ -159,7 +159,7 @@ int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t char *buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { - + } else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_SUFFIX) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c new file mode 100644 index 0000000000..a1bba56391 --- /dev/null +++ b/source/libs/index/src/index_tfile.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index_tfile.h" + +IndexTFile *indexTFileCreate() { + IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); + return tfile; +} +void IndexTFileDestroy(IndexTFile *tfile) { + free(tfile); +} +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { + IndexTFile *ptfile = (IndexTFile *)tfile; + return 0; +} + + +