diff --git a/cmake/cmake.options b/cmake/cmake.options index e44a38b3f5..2384a427e4 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -57,7 +57,7 @@ option( ) option( - USE_INVERTEDINDEX + BUILD_WITH_INVERTEDINDEX "If use invertedIndex" ON ) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 2cc2aeb6a0..3ca8d10603 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -51,7 +51,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm * @param: * @param: */ -SIndex* indexOpen(SIndexOpts *opt, const char *path); +int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); void indexClose(SIndex *index); int indexPut(SIndex *index, SIndexMultiTerm *terms, int uid); int indexDelete(SIndex *index, SIndexMultiTermQuery *query); diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index f68fc5e61e..4805bd3b77 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -22,9 +22,13 @@ if (${BUILD_WITH_LUCENE}) index PUBLIC lucene++ ) - endif(${BUILD_WITH_LUCENE}) +if (${BUILD_WITH_INVERTEDINDEX}) + add_definitions(-DUSE_INVERTED_INDEX) +endif(${BUILD_WITH_INVERTEDINDEX}) + + if (${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index fb5a9e40b5..7e017049e8 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -37,10 +37,10 @@ struct SIndex { #endif void *cache; void *tindex; - SHashObj *fieldObj;// < field name, field id> + SHashObj *colObj;// < field name, field id> int64_t suid; // current super table id, -1 is normal table - int fieldId; // field id allocated to cache + int colId; // field id allocated to cache int32_t cVersion; // current version allocated to cache pthread_mutex_t mtx; }; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ca6c2062f1..08c59d8d43 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -22,11 +22,10 @@ #endif -typedef struct SIdxFieldInfo { - int fieldId; // generated by index internal +typedef struct SIdxColInfo { + int colId; // generated by index internal int cVersion; - int type; // field type -} SIdxFieldInfo; +} SIdxColInfo; static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); @@ -38,9 +37,10 @@ static int indexMergeCacheIntoTindex(struct SIndex *sIdx) { indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); return 0; } -SIndex *indexOpen(SIndexOpts *opts, const char *path) { +int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { pthread_once(&isInit, indexInit); SIndex *sIdx = calloc(1, sizeof(SIndex)); + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t *index = index_open(path); @@ -49,11 +49,13 @@ SIndex *indexOpen(SIndexOpts *opts, const char *path) { sIdx->cache = (void*)indexCacheCreate(); sIdx->tindex = NULL; - sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - sIdx->fieldId = 1; + sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + sIdx->colId = 1; sIdx->cVersion = 1; pthread_mutex_init(&sIdx->mtx, NULL); - return sIdx; + + *index = sIdx; + return 0; } void indexClose(SIndex *sIdx) { @@ -62,16 +64,16 @@ void indexClose(SIndex *sIdx) { sIdx->index = NULL; #endif -#ifdef USE_INVERTEDINDEX +#ifdef USE_INVERTED_INDEX indexCacheDestroy(sIdx->cache); - taosHashCleanup(sIdx->fieldObj); + taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif free(sIdx); return; } -int indexPut(SIndex *index, SArray* fVals, int uid) { +int indexPut(SIndex *index, SIndexMultiTerm * fVals, int uid) { #ifdef USE_LUCENE index_document_t *doc = index_document_create(); @@ -89,38 +91,38 @@ int indexPut(SIndex *index, SArray* fVals, int uid) { index_document_destroy(doc); #endif -#ifdef USE_INVERTEDINDEX +#ifdef USE_INVERTED_INDEX + //TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm *p = taosArrayGetP(fVals, i); - SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); + SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); if (fi == NULL) { - SIdxFieldInfo tfi = {.fieldId = index->fieldId, .type = p->type}; + SIdxColInfo tfi = {.colId = index->colId}; index->cVersion++; - index->fieldId++; - taosHashPut(index->fieldObj, p->key, p->nKey, &tfi, sizeof(tfi)); + index->colId++; + taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); } else { //TODO, del } } + pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm *p = taosArrayGetP(fVals, i); - SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); + SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); assert(fi != NULL); - int32_t fieldId = fi->fieldId; - int32_t fieldType = fi->type; + int32_t colId = fi->colId; int32_t version = index->cVersion; - int res = indexCachePut(index->cache, fieldId, fieldType, p->val, p->nVal, version, uid, p->operType); + int ret = indexCachePut(index->cache, colId, p->colType, p->colVal, p->nColVal, version, uid, p->operType); if (ret != 0) { - return + return ret; } } - pthread_mutex_unlock(&index->mtx); #endif - return 1; + return 0; } int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { #ifdef USE_LUCENE @@ -159,7 +161,7 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result free(types); #endif -#ifdef USE_INVERTEDINDEX +#ifdef USE_INVERTED_INDEX #endif return 1; @@ -167,13 +169,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { -#ifdef USE_INVERTEDINDEX +#ifdef USE_INVERTED_INDEX #endif return 1; } int indexRebuild(SIndex *index, SIndexOpts *opts) { -#ifdef USE_INVERTEDINDEX +#ifdef USE_INVERTED_INDEX #endif } diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index f84f874a23..6eb532b41e 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,7 +1,7 @@ add_executable(indexTest "") target_sources(indexTest PRIVATE - "indexTests.cpp" + "indexTests.cc" ) target_include_directories ( indexTest PUBLIC diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cc similarity index 74% rename from source/libs/index/test/indexTests.cpp rename to source/libs/index/test/indexTests.cc index f582536817..9dff2e9ea0 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cc @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include #include @@ -61,7 +75,7 @@ class FstReadMemory { // add later bool Search(AutomationCtx *ctx, std::vector &result) { FstStreamBuilder *sb = fstSearch(_fst, ctx); - StreamWithState *st = streamBuilderIntoStream(sb); + StreamWithState *st = streamBuilderIntoStream(sb); StreamWithStateResult *rt = NULL; while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { @@ -279,15 +293,71 @@ void validateFst() { delete m; } +class IndexEnv : public ::testing::Test { + protected: + virtual void SetUp() { + taosRemoveDir(path); + opts = indexOptsCreate(); + int ret = indexOpen(opts, path, &index); + assert(ret == 0); + } + virtual void TearDown() { + indexClose(index); + indexOptsDestroy(opts); + } + + const char *path = "/tmp/tindex"; + SIndexOpts *opts; + SIndex *index; +}; -int main(int argc, char** argv) { - checkFstPerf(); - //checkFstPrefixSearch(); - return 1; +TEST_F(IndexEnv, testPut) { + + // single index column + { + + std::string colName("tag1"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + SIndexMultiTerm *terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + + for (size_t i = 0; i < 100; i++) { + int tableId = i; + int ret = indexPut(index, terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + // multi index column + { + + SIndexMultiTerm *terms = indexMultiTermCreate(); + { + std::string colName("tag1"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + } + { + std::string colName("tag2"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + } + + for (int i = 0; i < 100; i++) { + int tableId = i; + int ret = indexPut(index, terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + // +} + +TEST_F(IndexEnv, testDel) { + } -//TEST(IndexFstBuilder, IndexFstInput) { -// -//} + +