diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 47eb97cc3a..9424eae1b7 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -29,6 +29,12 @@ typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef struct SIndex SIndexJson; +typedef struct SIndexTerm SIndexJsonTerm; +typedef struct SIndexOpts SIndexJsonOpts; +typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery; +typedef struct SArray SIndexJsonMultiTerm; + typedef enum { ADD_VALUE, // add index colume value DEL_VALUE, // delete index column value @@ -39,24 +45,108 @@ typedef enum { } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; + /* - * @param: oper - * + * create multi query + * @param oper (input, relation between querys) */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); + /* - * @param: - * @param: + * destroy multi query + * @param pQuery (input, multi-query-object to be destory) + */ + +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +/* + * add query to multi query + * @param pQuery (input, multi-query-object) + * @param term (input, single query term) + * @param type (input, single query type) + * @return error code + */ +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); +/* + * open index + * @param opt (input, index opt) + * @param path (input, index path) + * @param index (output, index object) + * @return error code + */ +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code */ -int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); void indexClose(SIndex* index); -int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); -int indexDelete(SIndex* index, SIndexMultiTermQuery* query); -int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); -int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +/* + * delete terms that meet query condition + * @param index (input, index object) + * @param query (input, condition query to deleted) + * @return error code + */ + +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +/* + * rebuild index + * @param index (input, index object) + * @parma opt (input, rebuild index opts) + * @return error code + */ +int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * open index + * @param opt (input,index json opt) + * @param path (input, index json path) + * @param index (output, index json object) + * @return error code + */ +int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code + */ + +int tIndexJsonClose(SIndexJson* index); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ + +int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result); /* * @param * @param diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 90ad1e15f4..57d587d297 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -120,29 +120,50 @@ int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ + do { \ + uint8_t oldTy = ty; \ + ty = (ty >> 4) | exTy; \ + ty = (ty << 4) | oldTy; \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5147734a85..b518df2ea2 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -2,8 +2,8 @@ * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -30,6 +30,8 @@ void* indexQhandle = NULL; +static char JSON_COLUMN[] = "JSON"; + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -63,6 +65,9 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); static void indexMergeSameKey(SArray* result, TFileValue* tv); +static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); +int32_t indexSerialKey(ICacheKey* key, char* buf); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); @@ -147,9 +152,8 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { @@ -162,9 +166,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + // ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -554,7 +558,24 @@ END: return -1; } -int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +int32_t indexSerialTermKeyOfTag(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + return indexSerialKey(&key, buf); +} +int32_t indexSerilaTermKeyOfJson(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = JSON_COLUMN, .nColName = strlen(JSON_COLUMN)}; + return indexSerialKey(&key, buf); +} + +int32_t indexSerialTermKey(SIndexTerm* itm, char* buf) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(itm->colType, TSDB_DATA_TYPE_JSON); + if (hasJson) { + return indexSerilaTermKeyOfJson(itm, buf); + } else { + return indexSerialTermKeyOfTag(itm, buf); + } +} +int32_t indexSerialKey(ICacheKey* key, char* buf) { char* p = buf; SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_VAR_TO_BUF(buf, '_', char); @@ -563,3 +584,13 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); return buf - p; } + +// int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +// char* p = buf; +// SERIALIZE_MEM_TO_BUF(buf, key, suid); +// SERIALIZE_VAR_TO_BUF(buf, '_', char); +// // SERIALIZE_MEM_TO_BUF(buf, key, colType); +// // SERIALIZE_VAR_TO_BUF(buf, '_', char); +// SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +// return buf - p; +//} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index d6a7141825..bb29a78cb1 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -24,6 +24,8 @@ #define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 +static char JSON_COLUMN[] = "JSON"; + static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -45,7 +47,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in return NULL; }; cache->mem = indexInternalCacheCreate(type); - cache->colName = tstrdup(colName); + cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c new file mode 100644 index 0000000000..f4d28b45b0 --- /dev/null +++ b/source/libs/index/src/index_json.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "indeInt.h" +#include "index.h" + +int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { + // handle + return tIndexOpen(opts, path, index); +} +// k +int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexPut(index, terms, uid); + // handle put +} + +int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *query, SArray *result) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexSearch(index, query, result); + // handle search +} + +int tIndexJsonClose(SIndexJson *index) { + return tIndexClose(index); + // handle close +} diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index e44f8fc1c3..a05884b790 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -205,7 +205,11 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul } else if (qtype == QUERY_PREFIX) { // handle later // - } else { + } else if (qtype == QUERY_SUFFIX) { + // handle later + } else if (qtype == QUERY_REGEX) { + // handle later + } else if (qtype == QUERY_RANGE) { // handle later } tfileReaderUnRef(reader); @@ -586,11 +590,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, + reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d59a3428da..b34fd71e9c 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -216,21 +216,21 @@ class FstEnv : public ::testing::Test { TEST_F(FstEnv, writeNormal) { fst->CreateWriter(); - std::string str("aa"); + std::string str("11"); for (int i = 0; i < 10; i++) { - str[0] = 'a' + i; + str[0] = '1' + i; str.resize(2); assert(fst->Put(str, i) == true); } // order failed - assert(fst->Put("aa", 1) == false); + assert(fst->Put("11", 1) == false); fst->DestroyWriter(); fst->CreateReader(); uint64_t val; - assert(fst->Get("a", &val) == false); - assert(fst->Get("aa", &val) == true); + assert(fst->Get("1", &val) == false); + assert(fst->Get("11", &val) == true); assert(val == 0); std::vector rlt; diff --git a/source/libs/index/test/jsonDemo.cc b/source/libs/index/test/jsonDemo.cc new file mode 100644 index 0000000000..e69de29bb2