xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9264 from taosdata/feature/index_cache 

update index TFile write
This commit is contained in:
Shengliang Guan 2021-12-22 13:24:43 +08:00 committed by GitHub
parent a630d82cf2 ebcb9be39f
commit 1e32e1f86a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1058 additions and 1217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/libs/index/index.h
View File

@ -76,8 +76,13 @@ void indexOptsDestroy(SIndexOpts *opts);
* @param: * @param:
*/ */
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName, SIndexTerm* indexTermCreate(int64_t suid,
int32_t nColName, const char *colVal, int32_t nColVal); SIndexOperOnColumn operType,
uint8_t colType,
const char* colName,
int32_t nColName,
const char* colVal,
int32_t nColVal);
void indexTermDestroy(SIndexTerm* p); void indexTermDestroy(SIndexTerm* p);
#ifdef __cplusplus #ifdef __cplusplus

3
source/libs/index/inc/index_cache.h
View File

@ -45,8 +45,7 @@ void indexCacheDestroy(void *cache);
int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid); int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid);
// int indexCacheGet(void *cache, uint64_t *rst); // int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch( int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s);
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/libs/index/inc/index_fst.h
View File

@ -142,8 +142,7 @@ uint64_t fstStateInputLen(FstState *state);
// end_addr // end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data);
uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes);
uint64_t fstStateEndAddrForAnyTrans( uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans);
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
// input // input
uint8_t fstStateInput(FstState* state, FstNode* node); uint8_t fstStateInput(FstState* state, FstNode* node);
uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i);
@ -311,8 +310,7 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
void swsResultDestroy(StreamWithStateResult* result); void swsResultDestroy(StreamWithStateResult* result);
typedef void* (*StreamCallback)(void*); typedef void* (*StreamCallback)(void*);
StreamWithState *streamWithStateCreate( StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max);
void streamWithStateDestroy(StreamWithState* sws); void streamWithStateDestroy(StreamWithState* sws);

23
source/libs/index/inc/index_tfile.h
View File

@ -26,25 +26,25 @@
extern "C" { extern "C" {
#endif #endif
// tfile header // tfile header content
// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| // |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->|
// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| // |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->|
typedef struct TFileReadHeader { typedef struct TFileHeader {
uint64_t suid; uint64_t suid;
int32_t version; int32_t version;
char colName[128]; // char colName[128]; //
uint8_t colType; uint8_t colType;
} TFileReadHeader; } TFileHeader;
#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); #define TFILE_HEADER_SIZE (sizeof(TFileHeader) + sizeof(uint32_t))
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) #define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileCacheKey { typedef struct TFileCacheKey {
uint64_t suid; uint64_t suid;
uint8_t colType; uint8_t colType;
int32_t version; int32_t version;
const char *colName; char* colName;
int32_t nColName; int32_t nColName;
} TFileCacheKey; } TFileCacheKey;
@ -59,13 +59,15 @@ typedef struct TFileCache {
typedef struct TFileWriter { typedef struct TFileWriter {
FstBuilder* fb; FstBuilder* fb;
WriterCtx* ctx; WriterCtx* ctx;
TFileHeader header;
uint32_t offset;
} TFileWriter; } TFileWriter;
typedef struct TFileReader { typedef struct TFileReader {
T_REF_DECLARE() T_REF_DECLARE()
Fst* fst; Fst* fst;
WriterCtx* ctx; WriterCtx* ctx;
TFileReadHeader header; TFileHeader header;
} TFileReader; } TFileReader;
typedef struct IndexTFile { typedef struct IndexTFile {
@ -94,11 +96,14 @@ void tfileCacheDestroy(TFileCache *tcache);
TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key);
void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader);
TFileReader *tfileReaderCreate(); TFileReader* tfileReaderCreate(WriterCtx* ctx);
void TFileReaderDestroy(TFileReader *reader); void tfileReaderDestroy(TFileReader* reader);
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header);
void tfileWriterDestroy(TFileWriter* tw); void tfileWriterDestroy(TFileWriter* tw);
int tfileWriterPut(TFileWriter* tw, void* data);
int tfileWriterFinish(TFileWriter* tw);
// //
IndexTFile* indexTFileCreate(const char* path); IndexTFile* indexTFileCreate(const char* path);

19
source/libs/index/src/index.c
View File

@ -58,6 +58,7 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
sIdx->index = index; sIdx->index = index;
#endif #endif
#ifdef USE_INVERTED_INDEX
sIdx->cache = (void*)indexCacheCreate(); sIdx->cache = (void*)indexCacheCreate();
sIdx->tindex = NULL; sIdx->tindex = NULL;
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
@ -67,6 +68,10 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
*index = sIdx; *index = sIdx;
return 0; return 0;
#endif
*index = NULL;
return -1;
} }
void indexClose(SIndex* sIdx) { void indexClose(SIndex* sIdx) {
@ -237,8 +242,13 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EInde
return 0; return 0;
} }
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, SIndexTerm* indexTermCreate(int64_t suid,
int32_t nColName, const char *colVal, int32_t nColVal) { SIndexOperOnColumn oper,
uint8_t colType,
const char* colName,
int32_t nColName,
const char* colVal,
int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { if (t == NULL) {
return NULL; return NULL;
@ -263,7 +273,9 @@ void indexTermDestroy(SIndexTerm *p) {
free(p); free(p);
} }
SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } SIndexMultiTerm* indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm*));
}
int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) {
taosArrayPush(terms, &term); taosArrayPush(terms, &term);
@ -336,6 +348,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType
SArray* first = taosArrayGetP(interResults, 0); SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare); taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL); taosArrayRemoveDuplicate(first, uidCompare, NULL);
if (oType == MUST) { if (oType == MUST) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, first);

44
source/libs/index/src/index_cache.c
View File

@ -21,10 +21,11 @@
// ref index_cache.h:22 // ref index_cache.h:22
#define CACHE_KEY_LEN(p) \ #define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \ (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
sizeof(p->operType))
static char * getIndexKey(const void *pData) { return NULL; } static char* getIndexKey(const void* pData) {
return NULL;
}
static int32_t compareKey(const void* l, const void* r) { static int32_t compareKey(const void* l, const void* r) {
char* lp = (char*)l; char* lp = (char*)l;
char* rp = (char*)r; char* rp = (char*)r;
@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) {
int16_t lf, rf; // field id int16_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf)); memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf)); memcpy(&rf, rp, sizeof(rf));
if (lf != rf) { if (lf != rf) { return lf < rf ? -1 : 1; }
return lf < rf ? -1 : 1;
}
lp += sizeof(lf); lp += sizeof(lf);
rp += sizeof(rf); rp += sizeof(rf);
@ -89,9 +88,8 @@ static int32_t compareKey(const void *l, const void *r) {
int32_t lv, rv; int32_t lv, rv;
memcpy(&lv, lp, sizeof(lv)); memcpy(&lv, lp, sizeof(lv));
memcpy(&rv, rp, sizeof(rv)); memcpy(&rv, rp, sizeof(rv));
if (lv != rv) { if (lv != rv) { return lv > rv ? -1 : 1; }
return lv > rv ? -1 : 1;
}
lp += sizeof(lv); lp += sizeof(lv);
rp += sizeof(rv); rp += sizeof(rv);
// not care item type // not care item type
@ -100,28 +98,29 @@ static int32_t compareKey(const void *l, const void *r) {
} }
IndexCache* indexCacheCreate() { IndexCache* indexCacheCreate() {
IndexCache* cache = calloc(1, sizeof(IndexCache)); IndexCache* cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate( if (cache == NULL) {
MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); indexError("failed to create index cache");
return NULL;
}
cache->skiplist =
tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache; return cache;
} }
void indexCacheDestroy(void* cache) { void indexCacheDestroy(void* cache) {
IndexCache* pCache = cache; IndexCache* pCache = cache;
if (pCache == NULL) { if (pCache == NULL) { return; }
return;
}
tSkipListDestroy(pCache->skiplist); tSkipListDestroy(pCache->skiplist);
free(pCache); free(pCache);
} }
int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
if (cache == NULL) { if (cache == NULL) { return -1; }
return -1;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
// encode data // encode data
int32_t total = CACHE_KEY_LEN(term); int32_t total = CACHE_KEY_LEN(term);
char* buf = calloc(1, total); char* buf = calloc(1, total);
char* p = buf; char* p = buf;
@ -145,11 +144,8 @@ int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t
IndexCache* pCache = cache; IndexCache* pCache = cache;
return 0; return 0;
} }
int indexCacheSearch( int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { if (cache == NULL) { return -1; }
if (cache == NULL) {
return -1;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
EIndexQueryType qtype = query->qType; EIndexQueryType qtype = query->qType;
@ -158,9 +154,13 @@ int indexCacheSearch(
char* buf = calloc(1, keyLen); char* buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) { if (qtype == QUERY_TERM) {
//
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
//
} else if (qtype == QUERY_SUFFIX) { } else if (qtype == QUERY_SUFFIX) {
//
} else if (qtype == QUERY_REGEX) { } else if (qtype == QUERY_REGEX) {
//
} }
return 0; return 0;

240
source/libs/index/src/index_fst.c
View File

@ -31,9 +31,7 @@ static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, Compi
FstUnFinishedNodes* fstUnFinishedNodesCreate() { FstUnFinishedNodes* fstUnFinishedNodesCreate() {
FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes));
if (nodes == NULL) { if (nodes == NULL) { return NULL; }
return NULL;
}
nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
fstUnFinishedNodesPushEmpty(nodes, false); fstUnFinishedNodesPushEmpty(nodes, false);
@ -46,9 +44,7 @@ void unFinishedNodeDestroyElem(void *elem) {
b->last = NULL; b->last = NULL;
} }
void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) {
if (nodes == NULL) { if (nodes == NULL) { return; }
return;
}
taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
free(nodes); free(nodes);
@ -91,15 +87,12 @@ void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) {
// un->node->trans = NULL; // un->node->trans = NULL;
} }
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) {
size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1);
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz);
fstBuilderNodeUnfinishedLastCompiled(un, addr); fstBuilderNodeUnfinishedLastCompiled(un, addr);
} }
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) {
FstSlice* s = &bs; FstSlice* s = &bs;
if (fstSliceIsEmpty(s)) { if (fstSliceIsEmpty(s)) { return; }
return;
}
size_t sz = taosArrayGetSize(nodes->stack) - 1; size_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL); assert(un->last == NULL);
@ -179,9 +172,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
FstState fs = {.state = EmptyFinal, .val = 0}; FstState fs = {.state = EmptyFinal, .val = 0};
if (addr == EMPTY_ADDRESS) { if (addr == EMPTY_ADDRESS) { return fs; }
return fs;
}
uint8_t* data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
uint8_t v = data[addr]; uint8_t v = data[addr];
@ -197,8 +188,10 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) {
return fs; return fs;
} }
static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000}, static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000},
{.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}}; {.state = OneTrans, .val = 0b10000000},
{.state = AnyTrans, .val = 0b00000000},
{.state = EmptyFinal, .val = 0b00000000}};
// debug // debug
static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"};
@ -236,9 +229,7 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran
fstStateSetCommInput(&st, trn->inp); fstStateSetCommInput(&st, trn->inp);
bool null = false; bool null = false;
uint8_t inp = fstStateCommInput(&st, &null); uint8_t inp = fstStateCommInput(&st, &null);
if (null == true) { if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); }
fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp));
}
fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
return; return;
} }
@ -272,9 +263,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
fstStateSetStateNtrans(&st, (uint8_t)sz); fstStateSetStateNtrans(&st, (uint8_t)sz);
if (anyOuts) { if (anyOuts) {
if (FST_BUILDER_NODE_IS_FINAL(node)) { if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); }
fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
}
for (int32_t i = sz - 1; i >= 0; i--) { for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition* t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
fstCountingWriterPackUintIn(w, t->out, oSize); fstCountingWriterPackUintIn(w, t->out, oSize);
@ -365,8 +354,7 @@ uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size
- FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes);
} }
uint64_t fstStateEndAddrForAnyTrans( uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) {
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes;
return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size
@ -415,8 +403,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans -
node->nTrans - (i * tSizes) - tSizes; (i * tSizes) - tSizes;
uint8_t* data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
return unpackDelta(data + at, tSizes, node->end); return unpackDelta(data + at, tSizes, node->end);
} }
@ -439,9 +427,7 @@ Output fstStateOutput(FstState *s, FstNode *node) {
assert(s->state == OneTrans); assert(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { if (oSizes == 0) { return 0; }
return 0;
}
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
@ -453,9 +439,7 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { if (oSizes == 0) { return 0; }
return 0;
}
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
uint8_t* data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -468,9 +452,7 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
void fstStateSetFinalState(FstState* s, bool yes) { void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (yes) { if (yes) { s->val |= 0b01000000; }
s->val |= 0b01000000;
}
return; return;
} }
bool fstStateIsFinalState(FstState* s) { bool fstStateIsFinalState(FstState* s) {
@ -480,9 +462,7 @@ bool fstStateIsFinalState(FstState *s) {
void fstStateSetStateNtrans(FstState* s, uint8_t n) { void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (n <= 0b00111111) { if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; }
s->val = (s->val & 0b11000000) | n;
}
return; return;
} }
// state_ntrans // state_ntrans
@ -514,9 +494,7 @@ uint64_t fstStateNtransLen(FstState *s) {
uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
bool null = false; bool null = false;
uint8_t n = fstStateStateNtrans(s, &null); uint8_t n = fstStateStateNtrans(s, &null);
if (null != true) { if (null != true) { return n; }
return n;
}
int32_t len; int32_t len;
uint8_t* data = fstSliceData(slice, &len); uint8_t* data = fstSliceData(slice, &len);
n = data[len - 2]; n = data[len - 2];
@ -526,9 +504,7 @@ uint64_t fstStateNtrans(FstState *s, FstSlice *slice) {
} }
Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
if (oSizes == 0 || !fstStateIsFinalState(s)) { if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; }
return 0;
}
uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size
- fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
@ -545,9 +521,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
uint8_t* data = fstSliceData(slice, &dlen); uint8_t* data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b]; uint64_t i = data[at + b];
// uint64_t i = slice->data[slice->start + at + b]; // uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) { if (i >= node->nTrans) { *null = true; }
*null = true;
}
return i; return i;
} else { } else {
uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size
@ -564,9 +538,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
return node->nTrans - i - 1; // bug return node->nTrans - i - 1; // bug
} }
} }
if (i == len) { if (i == len) { *null = true; }
*null = true;
}
fstSliceDestroy(&t); fstSliceDestroy(&t);
} }
} }
@ -575,9 +547,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) {
FstNode* n = (FstNode*)malloc(sizeof(FstNode)); FstNode* n = (FstNode*)malloc(sizeof(FstNode));
if (n == NULL) { if (n == NULL) { return NULL; }
return NULL;
}
FstState st = fstStateCreateFrom(slice, addr); FstState st = fstStateCreateFrom(slice, addr);
@ -625,8 +595,7 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->isFinal = fstStateIsFinalState(&st); // s.is_final_state();
n->nTrans = nTrans; n->nTrans = nTrans;
n->sizes = sz; n->sizes = sz;
n->finalOutput = n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans);
fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans);
} }
return n; return n;
} }
@ -643,9 +612,7 @@ void fstNodeDestroy(FstNode *node) {
} }
FstTransitions* fstNodeTransitions(FstNode* node) { FstTransitions* fstNodeTransitions(FstNode* node) {
FstTransitions* t = malloc(sizeof(FstTransitions)); FstTransitions* t = malloc(sizeof(FstTransitions));
if (NULL == t) { if (NULL == t) { return NULL; }
return NULL;
}
FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; FstRange range = {.start = 0, .end = FST_NODE_LEN(node)};
t->range = range; t->range = range;
t->node = node; t->node = node;
@ -752,9 +719,7 @@ bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, Compiled
FstBuilder* fstBuilderCreate(void* w, FstType ty) { FstBuilder* fstBuilderCreate(void* w, FstType ty) {
FstBuilder* b = malloc(sizeof(FstBuilder)); FstBuilder* b = malloc(sizeof(FstBuilder));
if (NULL == b) { if (NULL == b) { return b; }
return b;
}
b->wrt = fstCountingWriterCreate(w); b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate(); b->unfinished = fstUnFinishedNodesCreate();
@ -776,9 +741,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
return b; return b;
} }
void fstBuilderDestroy(FstBuilder* b) { void fstBuilderDestroy(FstBuilder* b) {
if (b == NULL) { if (b == NULL) { return; }
return;
}
fstCountingWriterDestroy(b->wrt); fstCountingWriterDestroy(b->wrt);
fstUnFinishedNodesDestroy(b->unfinished); fstUnFinishedNodesDestroy(b->unfinished);
@ -794,7 +757,7 @@ bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) {
fstBuilderInsertOutput(b, bs, in); fstBuilderInsertOutput(b, bs, in);
return true; return true;
} }
indexInfo("key must be ordered"); indexInfo("fst write key must be ordered");
return false; return false;
} }
@ -879,9 +842,7 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) {
fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);
b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);
if (entry->state == NOTFOUND) { if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); }
FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);
}
fstRegistryEntryDestroy(entry); fstRegistryEntryDestroy(entry);
return b->lastAddr; return b->lastAddr;
@ -914,7 +875,9 @@ void *fstBuilderInsertInner(FstBuilder *b) {
// b->wrt = NULL; // b->wrt = NULL;
return b->wrt; return b->wrt;
} }
void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); } void fstBuilderFinish(FstBuilder* b) {
fstBuilderInsertInner(b);
}
FstSlice fstNodeAsSlice(FstNode* node) { FstSlice fstNodeAsSlice(FstNode* node) {
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
@ -924,21 +887,19 @@ FstSlice fstNodeAsSlice(FstNode *node) {
FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
FstLastTransition* trn = malloc(sizeof(FstLastTransition)); FstLastTransition* trn = malloc(sizeof(FstLastTransition));
if (trn == NULL) { if (trn == NULL) { return NULL; }
return NULL;
}
trn->inp = inp; trn->inp = inp;
trn->out = out; trn->out = out;
return trn; return trn;
} }
void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); } void fstLastTransitionDestroy(FstLastTransition* trn) {
free(trn);
}
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
FstLastTransition* trn = unNode->last; FstLastTransition* trn = unNode->last;
if (trn == NULL) { if (trn == NULL) { return; }
return;
}
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
taosArrayPush(unNode->node->trans, &t); taosArrayPush(unNode->node->trans, &t);
fstLastTransitionDestroy(trn); fstLastTransitionDestroy(trn);
@ -947,35 +908,27 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp
} }
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) {
if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; }
unNode->node->finalOutput += out;
}
size_t sz = taosArrayGetSize(unNode->node->trans); size_t sz = taosArrayGetSize(unNode->node->trans);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition* trn = taosArrayGet(unNode->node->trans, i); FstTransition* trn = taosArrayGet(unNode->node->trans, i);
trn->out += out; trn->out += out;
} }
if (unNode->last) { if (unNode->last) { unNode->last->out += out; }
unNode->last->out += out;
}
return; return;
} }
Fst* fstCreate(FstSlice* slice) { Fst* fstCreate(FstSlice* slice) {
int32_t slen; int32_t slen;
char* buf = fstSliceData(slice, &slen); char* buf = fstSliceData(slice, &slen);
if (slen < 36) { if (slen < 36) { return NULL; }
return NULL;
}
uint64_t len = slen; uint64_t len = slen;
uint64_t skip = 0; uint64_t skip = 0;
uint64_t version; uint64_t version;
taosDecodeFixedU64(buf, &version); taosDecodeFixedU64(buf, &version);
skip += sizeof(version); skip += sizeof(version);
if (version == 0 || version > VERSION) { if (version == 0 || version > VERSION) { return NULL; }
return NULL;
}
uint64_t type; uint64_t type;
taosDecodeFixedU64(buf + skip, &type); taosDecodeFixedU64(buf + skip, &type);
@ -994,14 +947,10 @@ Fst *fstCreate(FstSlice *slice) {
taosDecodeFixedU64(buf + len, &fstLen); taosDecodeFixedU64(buf + len, &fstLen);
// TODO(validate root addr) // TODO(validate root addr)
Fst* fst = (Fst*)calloc(1, sizeof(Fst)); Fst* fst = (Fst*)calloc(1, sizeof(Fst));
if (fst == NULL) { if (fst == NULL) { return NULL; }
return NULL;
}
fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); fst->meta = (FstMeta*)malloc(sizeof(FstMeta));
if (NULL == fst->meta) { if (NULL == fst->meta) { goto FST_CREAT_FAILED; }
goto FST_CREAT_FAILED;
}
fst->meta->version = version; fst->meta->version = version;
fst->meta->rootAddr = rootAddr; fst->meta->rootAddr = rootAddr;
@ -1039,9 +988,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
for (uint32_t i = 0; i < len; i++) { for (uint32_t i = 0; i < len; i++) {
uint8_t inp = data[i]; uint8_t inp = data[i];
Output res = 0; Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) { if (false == fstNodeFindInput(root, inp, &res)) { return false; }
return false;
}
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(root, res, &trn); fstNodeGetTransitionAt(root, res, &trn);
@ -1066,26 +1013,32 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
return true; return true;
} }
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { return fstStreamBuilderCreate(fst, ctx);
if (sb == NULL) {
return NULL;
} }
StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
if (sb == NULL) { return NULL; }
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
} }
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
return fstStreamBuilderCreate(fst, ctx);
}
FstNode* fstGetRoot(Fst* fst) { FstNode* fstGetRoot(Fst* fst) {
if (fst->root != NULL) { if (fst->root != NULL) { return fst->root; }
return fst->root;
}
CompiledAddr rAddr = fstGetRootAddr(fst); CompiledAddr rAddr = fstGetRootAddr(fst);
fst->root = fstGetNode(fst, rAddr); fst->root = fstGetNode(fst, rAddr);
return fst->root; return fst->root;
} }
FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) {
FstType fstGetType(Fst *fst) { return fst->meta->ty; } return fstNodeCreate(fst->meta->version, addr, fst->data);
CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; } }
FstType fstGetType(Fst* fst) {
return fst->meta->ty;
}
CompiledAddr fstGetRootAddr(Fst* fst) {
return fst->meta->rootAddr;
}
Output fstEmptyFinalOutput(Fst* fst, bool* null) { Output fstEmptyFinalOutput(Fst* fst, bool* null) {
Output res = 0; Output res = 0;
@ -1104,18 +1057,14 @@ bool fstVerify(Fst *fst) {
int32_t len; int32_t len;
uint8_t* data = fstSliceData(fst->data, &len); uint8_t* data = fstSliceData(fst->data, &len);
TSCKSUM initSum = 0; TSCKSUM initSum = 0;
if (!taosCheckChecksumWhole(data, len)) { if (!taosCheckChecksumWhole(data, len)) { return false; }
return false;
}
return true; return true;
} }
// data bound function // data bound function
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) {
FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { if (b == NULL) { return NULL; }
return NULL;
}
if (data != NULL) { if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end); b->data = fstSliceCopy(data, data->start, data->end);
@ -1145,17 +1094,18 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
} }
} }
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; } bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) {
return bound->type == Excluded ? false : true;
void fstBoundDestroy(FstBoundWithData *bound) { free(bound); }
StreamWithState *streamWithStateCreate(
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) {
StreamWithState *sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) {
return NULL;
} }
void fstBoundDestroy(FstBoundWithData* bound) {
free(bound);
}
StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) {
StreamWithState* sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) { return NULL; }
sws->fst = fst; sws->fst = fst;
sws->aut = automation; sws->aut = automation;
sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t));
@ -1170,9 +1120,7 @@ StreamWithState *streamWithStateCreate(
return sws; return sws;
} }
void streamWithStateDestroy(StreamWithState* sws) { void streamWithStateDestroy(StreamWithState* sws) {
if (sws == NULL) { if (sws == NULL) { return; }
return;
}
taosArrayDestroy(sws->inp); taosArrayDestroy(sws->inp);
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
@ -1183,9 +1131,7 @@ void streamWithStateDestroy(StreamWithState *sws) {
bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
AutomationCtx* aut = sws->aut; AutomationCtx* aut = sws->aut;
if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsEmpty(min)) {
if (fstBoundWithDataIsIncluded(min)) { if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); }
sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
}
StreamState s = {.node = fstGetRoot(sws->fst), StreamState s = {.node = fstGetRoot(sws->fst),
.trans = 0, .trans = 0,
.out = {.null = false, .out = 0}, .out = {.null = false, .out = 0},
@ -1195,7 +1141,6 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
} }
FstSlice* key = NULL; FstSlice* key = NULL;
bool inclusize = false; bool inclusize = false;
;
if (min->type == Included) { if (min->type == Included) {
key = &min->data; key = &min->data;
@ -1239,9 +1184,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
uint64_t i = 0; uint64_t i = 0;
for (i = trans->range.start; i < trans->range.end; i++) { for (i = trans->range.start; i < trans->range.end; i++) {
FstTransition trn; FstTransition trn;
if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; }
break;
}
} }
StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
@ -1260,8 +1203,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
uint64_t trans = s->trans; uint64_t trans = s->trans;
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(n, trans - 1, &trn); fstNodeGetTransitionAt(n, trans - 1, &trn);
StreamState s = { StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
@ -1289,9 +1231,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
while (taosArrayGetSize(sws->stack) > 0) { while (taosArrayGetSize(sws->stack) > 0) {
StreamState* p = (StreamState*)taosArrayPop(sws->stack); StreamState* p = (StreamState*)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); }
taosArrayPop(sws->inp);
}
streamStateDestroy(p); streamStateDestroy(p);
continue; continue;
} }
@ -1308,9 +1248,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
if (FST_NODE_IS_FINAL(nextNode)) { if (FST_NODE_IS_FINAL(nextNode)) {
// void *eofState = sws->aut->acceptEof(nextState); // void *eofState = sws->aut->acceptEof(nextState);
void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
if (eofState != NULL) { if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); }
isMatch = automFuncs[aut->type].isMatch(aut, eofState);
}
} }
StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
taosArrayPush(sws->stack, &s1); taosArrayPush(sws->stack, &s1);
@ -1351,9 +1289,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { if (result == NULL) { return NULL; }
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut; result->out = fOut;
@ -1362,9 +1298,7 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
return result; return result;
} }
void swsResultDestroy(StreamWithStateResult* result) { void swsResultDestroy(StreamWithStateResult* result) {
if (NULL == result) { if (NULL == result) { return; }
return;
}
fstSliceDestroy(&result->data); fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state); startWithStateValueDestroy(result->state);
@ -1372,9 +1306,7 @@ void swsResultDestroy(StreamWithStateResult *result) {
} }
void streamStateDestroy(void* s) { void streamStateDestroy(void* s) {
if (NULL == s) { if (NULL == s) { return; }
return;
}
StreamState* ss = (StreamState*)s; StreamState* ss = (StreamState*)s;
fstNodeDestroy(ss->node); fstNodeDestroy(ss->node);
@ -1383,9 +1315,7 @@ void streamStateDestroy(void *s) {
FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { if (NULL == b) { return NULL; }
return NULL;
}
b->fst = fst; b->fst = fst;
b->aut = aut; b->aut = aut;
@ -1401,9 +1331,7 @@ void fstStreamBuilderDestroy(FstStreamBuilder *b) {
free(b); free(b);
} }
FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
if (b == NULL) { if (b == NULL) { return NULL; }
return NULL;
}
if (type == GE) { if (type == GE) {
b->min->type = Included; b->min->type = Included;

57
source/libs/index/src/index_fst_automation.c
View File

@ -17,9 +17,7 @@
StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { if (nsv == NULL) { return NULL; }
return NULL;
}
nsv->kind = kind; nsv->kind = kind;
nsv->type = ty; nsv->type = ty;
@ -37,9 +35,7 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
} }
void startWithStateValueDestroy(void* val) { void startWithStateValueDestroy(void* val) {
StartWithStateValue* sv = (StartWithStateValue*)val; StartWithStateValue* sv = (StartWithStateValue*)val;
if (sv == NULL) { if (sv == NULL) { return; }
return;
}
if (sv->type == FST_INT) { if (sv->type == FST_INT) {
// //
@ -52,9 +48,7 @@ void startWithStateValueDestroy(void *val) {
} }
StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { if (nsv == NULL) { return NULL; }
return NULL;
}
nsv->kind = sv->kind; nsv->kind = sv->kind;
nsv->type = sv->type; nsv->type = sv->type;
@ -65,6 +59,7 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
nsv->ptr = (char*)calloc(1, len + 1); nsv->ptr = (char*)calloc(1, len + 1);
memcpy(nsv->ptr, sv->ptr, len); memcpy(nsv->ptr, sv->ptr, len);
} else if (nsv->type == FST_ARRAY) { } else if (nsv->type == FST_ARRAY) {
//
} }
return nsv; return nsv;
} }
@ -83,17 +78,15 @@ static bool prefixCanMatch(AutomationCtx *ctx, void *sv) {
StartWithStateValue* ssv = (StartWithStateValue*)sv; StartWithStateValue* ssv = (StartWithStateValue*)sv;
return ssv->val >= 0; return ssv->val >= 0;
} }
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) {
return true;
}
static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
StartWithStateValue* ssv = (StartWithStateValue*)state; StartWithStateValue* ssv = (StartWithStateValue*)state;
if (ssv == NULL || ctx == NULL) { if (ssv == NULL || ctx == NULL) { return NULL; }
return NULL;
}
char* data = ctx->data; char* data = ctx->data;
if (ssv->kind == Done) { if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); }
return startWithStateValueCreate(Done, FST_INT, &ssv->val);
}
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1; int val = ssv->val + 1;
StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val);
@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
} }
return NULL; return NULL;
} }
static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } static void* prefixAcceptEof(AutomationCtx* ctx, void* state) {
return NULL;
}
// pattern query, impl later // pattern query, impl later
static void *patternStart(AutomationCtx *ctx) { return NULL; } static void* patternStart(AutomationCtx* ctx) {
static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; } return NULL;
static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; } }
static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } static bool patternIsMatch(AutomationCtx* ctx, void* data) {
return true;
}
static bool patternCanMatch(AutomationCtx* ctx, void* data) {
return true;
}
static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) {
return true;
}
static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; } static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
return NULL;
}
static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } static void* patternAcceptEof(AutomationCtx* ctx, void* state) {
return NULL;
}
AutomationFunc automFuncs[] = { AutomationFunc automFuncs[] = {
{prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof},
@ -127,9 +134,7 @@ AutomationFunc automFuncs[] = {
AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { if (ctx == NULL) { return NULL; }
return NULL;
}
StartWithStateValue* sv = NULL; StartWithStateValue* sv = NULL;
if (atype == AUTOMATION_PREFIX) { if (atype == AUTOMATION_PREFIX) {

272
source/libs/index/src/index_fst_common.c
View File

@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = {
}; };
const char COMMON_INPUTS_INV[] = { const char COMMON_INPUTS_INV[] = {
't', 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', '.', 'h', 'l', 'm',
'e', '-', 'd', 'u', '0', '1', '2', 'g', '=', ':', 'b', 'f', '3', 'y', '5', '&', '_',
'/', '4', 'v', '9', '6', '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I',
'o', 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'q', 'H', 'G', 'W',
'a', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 'Q', ';', ')', '(', '~', '[', ']', '$',
's', '!', '\'', '*', '@', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b',
'r', '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b',
'i', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', '#', '<', '>', '\\', '^', '`', '{', '|', '}', '\x7f',
'p', '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f',
'c', '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f',
'n', '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf',
'w', '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf',
'.', '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf',
'h', '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf',
'l', '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef',
'm', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff',
'-',
'd',
'u',
'0',
'1',
'2',
'g',
'=',
':',
'b',
'f',
'3',
'y',
'5',
'&',
'_',
'4',
'v',
'9',
'6',
'7',
'8',
'k',
'%',
'?',
'x',
'C',
'D',
'A',
'S',
'F',
'I',
'B',
'E',
'j',
'P',
'T',
'z',
'R',
'N',
'M',
'+',
'L',
'O',
'q',
'H',
'G',
'W',
'U',
'V',
',',
'Y',
'K',
'J',
'Z',
'X',
'Q',
';',
')',
'(',
'~',
'[',
']',
'$',
'!',
'\'',
'*',
'@',
'\x00',
'\x01',
'\x02',
'\x03',
'\x04',
'\x05',
'\x06',
'\x07',
'\x08',
'\t',
'\n',
'\x0b',
'\x0c',
'\r',
'\x0e',
'\x0f',
'\x10',
'\x11',
'\x12',
'\x13',
'\x14',
'\x15',
'\x16',
'\x17',
'\x18',
'\x19',
'\x1a',
'\x1b',
'\x1c',
'\x1d',
'\x1e',
'\x1f',
' ',
'"',
'#',
'<',
'>',
'\\',
'^',
'`',
'{',
'|',
'}',
'\x7f',
'\x80',
'\x81',
'\x82',
'\x83',
'\x84',
'\x85',
'\x86',
'\x87',
'\x88',
'\x89',
'\x8a',
'\x8b',
'\x8c',
'\x8d',
'\x8e',
'\x8f',
'\x90',
'\x91',
'\x92',
'\x93',
'\x94',
'\x95',
'\x96',
'\x97',
'\x98',
'\x99',
'\x9a',
'\x9b',
'\x9c',
'\x9d',
'\x9e',
'\x9f',
'\xa0',
'\xa1',
'\xa2',
'\xa3',
'\xa4',
'\xa5',
'\xa6',
'\xa7',
'\xa8',
'\xa9',
'\xaa',
'\xab',
'\xac',
'\xad',
'\xae',
'\xaf',
'\xb0',
'\xb1',
'\xb2',
'\xb3',
'\xb4',
'\xb5',
'\xb6',
'\xb7',
'\xb8',
'\xb9',
'\xba',
'\xbb',
'\xbc',
'\xbd',
'\xbe',
'\xbf',
'\xc0',
'\xc1',
'\xc2',
'\xc3',
'\xc4',
'\xc5',
'\xc6',
'\xc7',
'\xc8',
'\xc9',
'\xca',
'\xcb',
'\xcc',
'\xcd',
'\xce',
'\xcf',
'\xd0',
'\xd1',
'\xd2',
'\xd3',
'\xd4',
'\xd5',
'\xd6',
'\xd7',
'\xd8',
'\xd9',
'\xda',
'\xdb',
'\xdc',
'\xdd',
'\xde',
'\xdf',
'\xe0',
'\xe1',
'\xe2',
'\xe3',
'\xe4',
'\xe5',
'\xe6',
'\xe7',
'\xe8',
'\xe9',
'\xea',
'\xeb',
'\xec',
'\xed',
'\xee',
'\xef',
'\xf0',
'\xf1',
'\xf2',
'\xf3',
'\xf4',
'\xf5',
'\xf6',
'\xf7',
'\xf8',
'\xf9',
'\xfa',
'\xfb',
'\xfc',
'\xfd',
'\xfe',
'\xff',
}; };

30
source/libs/index/src/index_fst_counting_writer.c
View File

@ -18,9 +18,7 @@
#include "tutil.h" #include "tutil.h"
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->offset + len > ctx->limit) { if (ctx->offset + len > ctx->limit) { return -1; }
return -1;
}
if (ctx->type == TFile) { if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len)); assert(len == tfWrite(ctx->file.fd, buf, len));
@ -53,9 +51,7 @@ static int writeCtxDoFlush(WriterCtx *ctx) {
WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); WriterCtx* ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { if (ctx == NULL) { return NULL; }
return NULL;
}
ctx->type = type; ctx->type = type;
if (ctx->type == TFile) { if (ctx->type == TFile) {
@ -67,8 +63,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
ctx->file.fd = tfOpenReadWrite(tmpFile); ctx->file.fd = tfOpenReadWrite(tmpFile);
} }
if (ctx->file.fd < 0) { if (ctx->file.fd < 0) {
goto END;
indexError("open file error %d", errno); indexError("open file error %d", errno);
goto END;
} }
} else if (ctx->type == TMemory) { } else if (ctx->type == TMemory) {
ctx->mem.buf = calloc(1, sizeof(char) * capacity); ctx->mem.buf = calloc(1, sizeof(char) * capacity);
@ -83,9 +79,7 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
return ctx; return ctx;
END: END:
if (ctx->type == TMemory) { if (ctx->type == TMemory) { free(ctx->mem.buf); }
free(ctx->mem.buf);
}
free(ctx); free(ctx);
} }
void writerCtxDestroy(WriterCtx* ctx) { void writerCtxDestroy(WriterCtx* ctx) {
@ -99,9 +93,7 @@ void writerCtxDestroy(WriterCtx *ctx) {
FstCountingWriter* fstCountingWriterCreate(void* wrt) { FstCountingWriter* fstCountingWriterCreate(void* wrt) {
FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { if (cw == NULL) { return NULL; }
return NULL;
}
cw->wrt = wrt; cw->wrt = wrt;
//(void *)(writerCtxCreate(TFile, readOnly)); //(void *)(writerCtxCreate(TFile, readOnly));
@ -115,9 +107,7 @@ void fstCountingWriterDestroy(FstCountingWriter *cw) {
} }
int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
if (write == NULL) { if (write == NULL) { return 0; }
return 0;
}
// update checksum // update checksum
// write data to file/socket or mem // write data to file/socket or mem
WriterCtx* ctx = write->wrt; WriterCtx* ctx = write->wrt;
@ -128,16 +118,16 @@ int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len)
return len; return len;
} }
int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
if (write == NULL) { if (write == NULL) { return 0; }
return 0;
}
WriterCtx* ctx = write->wrt; WriterCtx* ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len); int nRead = ctx->read(ctx, buf, len);
// assert(nRead == len); // assert(nRead == len);
return nRead; return nRead;
} }
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) {
return 0;
}
int fstCountingWriterFlush(FstCountingWriter* write) { int fstCountingWriterFlush(FstCountingWriter* write) {
WriterCtx* ctx = write->wrt; WriterCtx* ctx = write->wrt;
ctx->flush(ctx); ctx->flush(ctx);

32
source/libs/index/src/index_fst_node.c
View File

@ -22,45 +22,31 @@ FstBuilderNode *fstBuilderNodeDefault() {
return bn; return bn;
} }
void fstBuilderNodeDestroy(FstBuilderNode* node) { void fstBuilderNodeDestroy(FstBuilderNode* node) {
if (node == NULL) { if (node == NULL) { return; }
return;
}
taosArrayDestroy(node->trans); taosArrayDestroy(node->trans);
free(node); free(node);
} }
bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) {
if (n1 == n2) { if (n1 == n2) { return true; }
return true; if (n1 == NULL || n2 == NULL) { return false; }
}
if (n1 == NULL || n2 == NULL) {
return false;
}
if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; }
return false;
}
size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0;
size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0;
if (s1 != s2) { if (s1 != s2) { return false; }
return false;
}
for (size_t i = 0; i < s1; i++) { for (size_t i = 0; i < s1; i++) {
FstTransition* t1 = taosArrayGet(n1->trans, i); FstTransition* t1 = taosArrayGet(n1->trans, i);
FstTransition* t2 = taosArrayGet(n2->trans, i); FstTransition* t2 = taosArrayGet(n2->trans, i);
if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; }
return false;
}
} }
return true; return true;
} }
FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) {
FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); FstBuilderNode* node = malloc(sizeof(FstBuilderNode));
if (node == NULL) { if (node == NULL) { return NULL; }
return NULL;
}
// //
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
@ -78,9 +64,7 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
} }
// not destroy src, User's bussiness // not destroy src, User's bussiness
void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
if (dst == NULL || src == NULL) { if (dst == NULL || src == NULL) { return; }
return;
}
dst->isFinal = src->isFinal; dst->isFinal = src->isFinal;
dst->finalOutput = src->finalOutput; dst->finalOutput = src->finalOutput;

24
source/libs/index/src/index_fst_registry.c
View File

@ -34,9 +34,7 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
} }
static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) { static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (a >= sz || b >= sz) { if (a >= sz || b >= sz) { return; }
return;
}
FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a); FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a);
FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b); FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b);
@ -53,9 +51,7 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (start >= sz && end >= sz) { if (start >= sz && end >= sz) { return; }
return;
}
assert(start >= end); assert(start >= end);
@ -69,9 +65,7 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
FstRegistry* registry = malloc(sizeof(FstRegistry)); FstRegistry* registry = malloc(sizeof(FstRegistry));
if (registry == NULL) { if (registry == NULL) { return NULL; }
return NULL;
}
uint64_t nCells = tableSize * mruSize; uint64_t nCells = tableSize * mruSize;
SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell));
@ -92,9 +86,7 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
} }
void fstRegistryDestroy(FstRegistry* registry) { void fstRegistryDestroy(FstRegistry* registry) {
if (registry == NULL) { if (registry == NULL) { return; }
return;
}
SArray* tb = registry->table; SArray* tb = registry->table;
size_t sz = taosArrayGetSize(tb); size_t sz = taosArrayGetSize(tb);
@ -107,9 +99,7 @@ void fstRegistryDestroy(FstRegistry *registry) {
} }
FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) {
if (taosArrayGetSize(registry->table) <= 0) { if (taosArrayGetSize(registry->table) <= 0) { return NULL; }
return NULL;
}
uint64_t bucket = fstRegistryHash(registry, bNode); uint64_t bucket = fstRegistryHash(registry, bNode);
uint64_t start = registry->mruSize * bucket; uint64_t start = registry->mruSize * bucket;
uint64_t end = start + registry->mruSize; uint64_t end = start + registry->mruSize;
@ -174,4 +164,6 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
} }
return entry; return entry;
} }
void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); } void fstRegistryEntryDestroy(FstRegistryEntry* entry) {
free(entry);
}

9
source/libs/index/src/index_fst_util.c
View File

@ -64,6 +64,7 @@ uint8_t packSize(uint64_t n) {
uint64_t unpackUint64(uint8_t* ch, uint8_t sz) { uint64_t unpackUint64(uint8_t* ch, uint8_t sz) {
uint64_t n = 0; uint64_t n = 0;
for (uint8_t i = 0; i < sz; i++) { for (uint8_t i = 0; i < sz; i++) {
//
n = n | (ch[i] << (8 * i)); n = n | (ch[i] << (8 * i));
} }
return n; return n;
@ -129,13 +130,13 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
ans.end = tlen - 1; ans.end = tlen - 1;
return ans; return ans;
} }
bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } bool fstSliceIsEmpty(FstSlice* s) {
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
uint8_t* fstSliceData(FstSlice* s, int32_t* size) { uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
FstString* str = s->str; FstString* str = s->str;
if (size != NULL) { if (size != NULL) { *size = s->end - s->start + 1; }
*size = s->end - s->start + 1;
}
return str->data + s->start; return str->data + s->start;
} }
void fstSliceDestroy(FstSlice* s) { void fstSliceDestroy(FstSlice* s) {

235
source/libs/index/src/index_tfile.c
View File

@ -21,15 +21,74 @@
#include "index_fst_counting_writer.h" #include "index_fst_counting_writer.h"
#include "index_util.h" #include "index_util.h"
#include "taosdef.h" #include "taosdef.h"
#include "tcompare.h"
#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
// static tfileGetCompareFunc(uint8_t byte) {}
static int tfileValueCompare(const void* a, const void* b, const void* param) {
__compar_fn_t fn = *(__compar_fn_t*)param;
TFileValue* av = (TFileValue*)a;
TFileValue* bv = (TFileValue*)b;
return fn(av->colVal, bv->colVal);
}
static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds) {
int tbSz = taosArrayGetSize(tableIds);
SERIALIZE_VAR_TO_BUF(buf, tbSz, int32_t);
for (size_t i = 0; i < tbSz; i++) {
uint64_t* v = taosArrayGet(tableIds, i);
SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t);
}
}
static FORCE_INLINE int tfileWriteHeader(TFileWriter* writer) {
char buf[TFILE_HEADER_SIZE] = {0};
char* p = buf;
TFileHeader* header = &writer->header;
SERIALIZE_MEM_TO_BUF(p, header, suid);
SERIALIZE_MEM_TO_BUF(p, header, version);
SERIALIZE_VAR_TO_BUF(p, strlen(header->colName), int32_t);
SERIALIZE_STR_MEM_TO_BUF(p, header, colName, strlen(header->colName));
SERIALIZE_MEM_TO_BUF(p, header, colType);
int offset = p - buf;
int nwrite = writer->ctx->write(writer->ctx, buf, offset);
if (offset != nwrite) { return -1; }
writer->offset = offset;
return 0;
}
static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
TFileHeader* header = &write->header;
uint8_t colType = header->colType;
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal));
if (fstBuilderInsert(write->fb, key, tval->offset)) {
fstSliceDestroy(&key);
return 0;
}
fstSliceDestroy(&key);
return -1;
} else {
// handle other type later
}
}
static FORCE_INLINE int tfileReadLoadHeader(TFileReader* reader) { static FORCE_INLINE int tfileReadLoadHeader(TFileReader* reader) {
// TODO simple tfile header later // TODO simple tfile header later
char buf[TFILE_HADER_PRE_SIZE]; char buf[TFILE_HADER_PRE_SIZE];
char* p = buf; char* p = buf;
TFileReadHeader *header = &reader->header;
int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE);
assert(nread == TFILE_HADER_PRE_SIZE); assert(nread == TFILE_HADER_PRE_SIZE);
TFileHeader* header = &reader->header;
memcpy(&header->suid, p, sizeof(header->suid)); memcpy(&header->suid, p, sizeof(header->suid));
p += sizeof(header->suid); p += sizeof(header->suid);
@ -44,12 +103,11 @@ static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) {
nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType)); nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType));
return 0; return 0;
}; }
static int tfileGetFileList(const char* path, SArray* result) { static int tfileGetFileList(const char* path, SArray* result) {
DIR* dir = opendir(path); DIR* dir = opendir(path);
if (NULL == dir) { if (NULL == dir) { return -1; }
return -1;
}
struct dirent* entry; struct dirent* entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
@ -68,8 +126,10 @@ static void tfileDestroyFileName(void *elem) {
static int tfileCompare(const void* a, const void* b) { static int tfileCompare(const void* a, const void* b) {
const char* aName = *(char**)a; const char* aName = *(char**)a;
const char* bName = *(char**)b; const char* bName = *(char**)b;
size_t aLen = strlen(aName); size_t aLen = strlen(aName);
size_t bLen = strlen(bName); size_t bLen = strlen(bName);
return strncmp(aName, bName, aLen > bLen ? aLen : bLen); return strncmp(aName, bName, aLen > bLen ? aLen : bLen);
} }
// tfile name suid-colId-version.tindex // tfile name suid-colId-version.tindex
@ -92,9 +152,7 @@ static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
TFileCache* tfileCacheCreate(const char* path) { TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = calloc(1, sizeof(TFileCache)); TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { if (tcache == NULL) { return NULL; }
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
@ -102,15 +160,14 @@ TFileCache *tfileCacheCreate(const char *path) {
SArray* files = taosArrayInit(4, sizeof(void*)); SArray* files = taosArrayInit(4, sizeof(void*));
tfileGetFileList(path, files); tfileGetFileList(path, files);
taosArraySort(files, tfileCompare); taosArraySort(files, tfileCompare);
uint64_t suid;
int32_t colId, version;
for (size_t i = 0; i < taosArrayGetSize(files); i++) { for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char* file = taosArrayGetP(files, i); char* file = taosArrayGetP(files, i);
uint64_t suid; if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) {
int colId, version; indexInfo("try parse invalid file: %s, skip it", file);
if (0 != tfileParseFileName(file, &suid, &colId, &version)) {
goto End;
continue; continue;
} }
WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64);
if (wc == NULL) { if (wc == NULL) {
indexError("failed to open index: %s", file); indexError("failed to open index: %s", file);
@ -118,10 +175,22 @@ TFileCache *tfileCacheCreate(const char *path) {
} }
TFileReader* reader = tfileReaderCreate(wc); TFileReader* reader = tfileReaderCreate(wc);
if (0 != tfileReadLoadHeader(reader)) { if (0 != tfileReadLoadHeader(reader)) {
TFileReaderDestroy(reader); tfileReaderDestroy(reader);
indexError("failed to load index header, index Id: %s", file); indexError("failed to load index header, index Id: %s", file);
goto End; goto End;
} }
// loader fst and validate it
TFileHeader* header = &reader->header;
TFileCacheKey key = {.suid = header->suid,
.version = header->version,
.colName = header->colName,
.nColName = strlen(header->colName),
.colType = header->colType};
char buf[128] = {0};
tfileSerialCacheKey(&key, buf);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
} }
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return tcache; return tcache;
@ -131,17 +200,15 @@ End:
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache* tcache) { void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { if (tcache == NULL) { return; }
return;
}
// free table cache // free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) { while (reader) {
TFileReader* p = *reader; TFileReader* p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType);
p->header.colType);
TFileReaderDestroy(p); tfileReaderDestroy(p);
reader = taosHashIterate(tcache->tableCache, reader); reader = taosHashIterate(tcache->tableCache, reader);
} }
taosHashCleanup(tcache->tableCache); taosHashCleanup(tcache->tableCache);
@ -163,45 +230,139 @@ void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader)
TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader* reader = calloc(1, sizeof(TFileReader)); TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { if (reader == NULL) { return NULL; }
return NULL;
}
reader->ctx = ctx;
// T_REF_INC(reader); // T_REF_INC(reader);
reader->ctx = ctx;
return reader; return reader;
} }
void TFileReaderDestroy(TFileReader *reader) { void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { if (reader == NULL) { return; }
return;
}
// T_REF_INC(reader); // T_REF_INC(reader);
writerCtxDestroy(reader->ctx); writerCtxDestroy(reader->ctx);
free(reader); free(reader);
} }
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) {
void tfileWriterDestroy(TFileWriter *tw); SIndexTerm* term = query->term;
// refactor to callback later
if (query->qType == QUERY_TERM) {
uint64_t offset;
FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
if (fstGet(reader->fst, &key, &offset)) {
//
} else {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found in tindex", term->suid, term->colName, term->colVal);
}
return 0;
} else if (query->qType == QUERY_PREFIX) {
//
//
}
return 0;
}
TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
// char pathBuf[128] = {0};
// sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
// TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
// memcpy(header.colName, );
// char buf[TFILE_HADER_PRE_SIZE];
// int len = TFILE_HADER_PRE_SIZE;
// if (len != ctx->write(ctx, buf, len)) {
// indexError("index: %" PRIu64 " failed to write header info", header->suid);
// return NULL;
//}
TFileWriter* tw = calloc(1, sizeof(TFileWriter));
if (tw == NULL) {
indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid);
return NULL;
}
tw->ctx = ctx;
tw->header = *header;
tfileWriteHeader(tw);
return tw;
}
int TFileWriterPut(TFileWriter* tw, void* data) {
// sort by coltype and write to tindex
__compar_fn_t fn = getComparFunc(tw->header.colType, 0);
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
int32_t bufLimit = 4096, offset = 0;
char* buf = calloc(1, sizeof(bufLimit));
char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data);
for (size_t i = 0; i < sz; i++) {
TFileValue* v = taosArrayGetP((SArray*)data, i);
int32_t tbsz = taosArrayGetSize(v->tableId);
// check buf has enough space or not
int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);
if (offset + ttsz > bufLimit) {
// batch write
tw->ctx->write(tw->ctx, buf, offset);
offset = 0;
memset(buf, 0, bufLimit);
p = buf;
}
tfileSerialTableIdsToBuf(p, v->tableId);
offset += ttsz;
p = buf + offset;
// set up value offset and
v->offset = tw->offset;
tw->offset += ttsz;
}
if (offset != 0) {
// write reversed data in buf to tindex
tw->ctx->write(tw->ctx, buf, offset);
}
// write fst
for (size_t i = 0; i < sz; i++) {
// TODO, fst batch write later
TFileValue* v = taosArrayGetP((SArray*)data, i);
if (tfileWriteData(tw, v) == 0) {
//
//
}
}
tfree(buf);
return 0;
}
void tfileWriterDestroy(TFileWriter* tw) {
if (tw == NULL) { return; }
writerCtxDestroy(tw->ctx);
free(tw);
}
IndexTFile* indexTFileCreate(const char* path) { IndexTFile* indexTFileCreate(const char* path) {
IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
tfile->cache = tfileCacheCreate(path); if (tfile == NULL) { return NULL; }
tfile->cache = tfileCacheCreate(path);
return tfile; return tfile;
} }
void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } void IndexTFileDestroy(IndexTFile* tfile) {
free(tfile);
}
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
if (tfile == NULL) { return -1; }
IndexTFile* pTfile = (IndexTFile*)tfile; IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
TFileCacheKey key = { TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName};
.suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key); TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
return 0; return tfileReaderSearch(reader, query, result);
} }
int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
TFileWriterOpt wOpt = { TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
return 0; return 0;
} }