xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

adjust index interface

This commit is contained in:
yihaoDeng 2021-12-18 18:16:07 +08:00
parent 653665641d
commit c51780622f
6 changed files with 100 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/cmake.options
View File

@ -37,6 +37,7 @@ option(
off off
) )
option( option(
BUILD_WITH_NURAFT BUILD_WITH_NURAFT
"If build with NuRaft" "If build with NuRaft"
@ -54,3 +55,9 @@ option(
"If use doxygen build documents" "If use doxygen build documents"
OFF OFF
) )
option(
USE_INVERTEDINDEX
"If use invertedIndex"
ON
)

18
include/libs/index/index.h
View File

@ -24,6 +24,7 @@ extern "C" {
#endif #endif
typedef struct SIndex SIndex; typedef struct SIndex SIndex;
typedef struct SIndexTerm SIndexTerm;
typedef struct SIndexOpts SIndexOpts; typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm; typedef struct SArray SIndexMultiTerm;
@ -35,7 +36,7 @@ typedef enum {
ADD_INDEX, // add index on specify column ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable DROP_SATBLE // drop stable
} SIndexColumnType; } SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType;
@ -45,7 +46,7 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX =
*/ */
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type); int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
/* /*
* @param: * @param:
* @param: * @param:
@ -61,7 +62,7 @@ int indexRebuild(SIndex *index, SIndexOpts *opt);
* @param * @param
*/ */
SIndexMultiTerm *indexMultiTermCreate(); SIndexMultiTerm *indexMultiTermCreate();
int indexMultiTermAdd(SIndexMultiTerm *terms, const char *field, int32_t nFields, const char *value, int32_t nValue); int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term);
void indexMultiTermDestroy(SIndexMultiTerm *terms); void indexMultiTermDestroy(SIndexMultiTerm *terms);
/* /*
* @param: * @param:
@ -70,6 +71,17 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms);
SIndexOpts *indexOptsCreate(); SIndexOpts *indexOptsCreate();
void indexOptsDestroy(SIndexOpts *opts); void indexOptsDestroy(SIndexOpts *opts);
/*
* @param:
* @param:
*/
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType,
const char *colName, int32_t nColName, const char *colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm *p);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

19
source/libs/index/inc/indexInt.h
View File

@ -60,22 +60,21 @@ struct SIndexMultiTermQuery {
// field and key; // field and key;
typedef struct SIndexTerm { typedef struct SIndexTerm {
uint8_t type; // term data type, str/interger/json int64_t suid;
char *key; SIndexOperOnColumn operType; // oper type, add/del/update
int32_t nKey; uint8_t colType; // term data type, str/interger/json
char *val; char *colName;
int32_t nVal; int32_t nColName;
char *colVal;
int32_t nColVal;
} SIndexTerm; } SIndexTerm;
typedef struct SIndexTermQuery { typedef struct SIndexTermQuery {
SIndexTerm* field_value; SIndexTerm* term;
EIndexQueryType type; EIndexQueryType qType;
} SIndexTermQuery; } SIndexTermQuery;
SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal);
void indexTermDestroy(SIndexTerm *p);
#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) #define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0)
#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) #define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0)

8
source/libs/index/inc/index_cache.h
View File

@ -38,13 +38,13 @@ typedef struct IndexCache {
// //
IndexCache *indexCacheCreate(); IndexCache *indexCacheCreate();
void indexCacheDestroy(IndexCache *cache); void indexCacheDestroy(void *cache);
int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen,
uint32_t version, uint64_t uid, int8_t operType); uint32_t version, uint64_t uid, int8_t operType);
int indexCacheGet(IndexCache *cache, uint64_t *rst); int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result); int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result);
#ifdef __cplusplus #ifdef __cplusplus
} }

75
source/libs/index/src/index.c
View File

@ -61,9 +61,12 @@ void indexClose(SIndex *sIdx) {
index_close(sIdex->index); index_close(sIdex->index);
sIdx->index = NULL; sIdx->index = NULL;
#endif #endif
#ifdef USE_INVERTEDINDEX
indexCacheDestroy(sIdx->cache); indexCacheDestroy(sIdx->cache);
taosHashCleanup(sIdx->fieldObj); taosHashCleanup(sIdx->fieldObj);
pthread_mutex_destroy(&sIdx->mtx); pthread_mutex_destroy(&sIdx->mtx);
#endif
free(sIdx); free(sIdx);
return; return;
} }
@ -86,6 +89,7 @@ int indexPut(SIndex *index, SArray* fVals, int uid) {
index_document_destroy(doc); index_document_destroy(doc);
#endif #endif
#ifdef USE_INVERTEDINDEX
//TODO(yihao): reduce the lock range //TODO(yihao): reduce the lock range
pthread_mutex_lock(&index->mtx); pthread_mutex_lock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
@ -106,11 +110,16 @@ int indexPut(SIndex *index, SArray* fVals, int uid) {
SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey);
assert(fi != NULL); assert(fi != NULL);
int32_t fieldId = fi->fieldId; int32_t fieldId = fi->fieldId;
int32_t colType = fi->type; int32_t fieldType = fi->type;
int32_t version = index->cVersion; int32_t version = index->cVersion;
int res = indexCachePut(index->cache, fieldId, fieldType, p->val, p->nVal, version, uid, p->operType);
if (ret != 0) {
return
}
} }
pthread_mutex_unlock(&index->mtx); pthread_mutex_unlock(&index->mtx);
#endif
return 1; return 1;
} }
int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) {
@ -148,16 +157,26 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
free(fields); free(fields);
free(keys); free(keys);
free(types); free(types);
#endif
#ifdef USE_INVERTEDINDEX
#endif #endif
return 1; return 1;
} }
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
#ifdef USE_INVERTEDINDEX
#endif
return 1; return 1;
} }
int indexRebuild(SIndex *index, SIndexOpts *opts); int indexRebuild(SIndex *index, SIndexOpts *opts) {
#ifdef USE_INVERTEDINDEX
#endif
}
SIndexOpts *indexOptsCreate() { SIndexOpts *indexOptsCreate() {
@ -184,53 +203,55 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) {
SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i);
indexTermDestroy(p->field_value); indexTermDestroy(p->term);
} }
taosArrayDestroy(pQuery->query); taosArrayDestroy(pQuery->query);
free(pQuery); free(pQuery);
}; };
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type){ int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){
SIndexTerm *t = indexTermCreate(field, nFields, value, nValue); SIndexTermQuery q = {.qType = qType, .term = term};
if (t == NULL) {return -1;}
SIndexTermQuery q = {.type = type, .field_value = t};
taosArrayPush(pQuery->query, &q); taosArrayPush(pQuery->query, &q);
return 0; return 0;
} }
SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal) { SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *t = (SIndexTerm *)malloc(sizeof(SIndexTerm)); SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm)));
t->key = (char *)calloc(nKey + 1, 1); if (t == NULL) { return NULL; }
memcpy(t->key, key, nKey);
t->nKey = nKey;
t->val = (char *)calloc(nVal + 1, 1); t->suid = suid;
memcpy(t->val, val, nVal); t->operType= oper;
t->nVal = nVal; t->colType = colType;
t->colName = (char *)calloc(1, nColName + 1);
memcpy(t->colName, colName, nColName);
t->nColName = nColName;
t->colVal = (char *)calloc(1, nColVal + 1);
memcpy(t->colVal, colVal, nColVal);
t->nColVal = nColVal;
return t; return t;
} }
void indexTermDestroy(SIndexTerm *p) { void indexTermDestroy(SIndexTerm *p) {
free(p->key); free(p->colName);
free(p->val); free(p->colVal);
free(p); free(p);
} }
SArray *indexMultiTermCreate() { SIndexMultiTerm *indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm *)); return taosArrayInit(4, sizeof(SIndexTerm *));
} }
int indexMultiTermAdd(SArray *array, const char *field, int32_t nField, const char *val, int32_t nVal) { int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
SIndexTerm *term = indexTermCreate(field, nField, val, nVal); taosArrayPush(terms, &term);
if (term == NULL) { return -1; }
taosArrayPush(array, &term);
return 0; return 0;
} }
void indexMultiTermDestroy(SArray *array) { void indexMultiTermDestroy(SIndexMultiTerm *terms) {
for (int32_t i = 0; i < taosArrayGetSize(array); i++) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
SIndexTerm *p = taosArrayGetP(array, i); SIndexTerm *p = taosArrayGetP(terms, i);
indexTermDestroy(p); indexTermDestroy(p);
} }
taosArrayDestroy(array); taosArrayDestroy(terms);
} }
void indexInit() { void indexInit() {

30
source/libs/index/src/index_cache.c
View File

@ -16,7 +16,7 @@
#include "index_cache.h" #include "index_cache.h"
#include "tcompare.h" #include "tcompare.h"
#define MAX_INDEX_KEY_LEN 128 // test only, change later #define MAX_INDEX_KEY_LEN 256// test only, change later
static char* getIndexKey(const void *pData) { static char* getIndexKey(const void *pData) {
return NULL; return NULL;
@ -96,16 +96,19 @@ IndexCache *indexCacheCreate() {
} }
void indexCacheDestroy(IndexCache *cache) { void indexCacheDestroy(void *cache) {
if (cache == NULL) { return; } IndexCache *pCache = cache;
tSkipListDestroy(cache->skiplist); if (pCache == NULL) { return; }
free(cache); tSkipListDestroy(pCache->skiplist);
free(pCache);
} }
int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen,
uint32_t version, uint64_t uid, int8_t operType) { uint32_t version, uint64_t uid, int8_t operType) {
if (cache == NULL) { return -1;} if (cache == NULL) { return -1;}
IndexCache *pCache = cache;
// encode data // encode data
int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType); int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType);
@ -135,20 +138,15 @@ int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const c
memcpy(p, &operType, sizeof(operType)); memcpy(p, &operType, sizeof(operType));
p += sizeof(operType); p += sizeof(operType);
tSkipListPut(cache->skiplist, (void *)buf); tSkipListPut(pCache->skiplist, (void *)buf);
// encode end // encode end
} }
int indexCacheDel(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache;
return 0;
} }
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) { int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result) {
return 0; return 0;
} }