From 32ac2d02c87d7f5ba46db2204fa4f78289b199ae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 Nov 2021 10:54:49 +0800 Subject: [PATCH 1/3] update fst core struct --- source/libs/index/inc/index_fst.h | 4 +++ source/libs/index/inc/index_fst_node.h | 2 +- source/libs/index/src/index_fst.c | 5 +++ source/libs/index/src/index_fst_node.c | 43 +++++++++++++------------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 37feb79ac8..01be2f8b2b 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -255,4 +255,8 @@ FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); Output fstEmptyFinalOutput(Fst *fst, bool *null); bool fstVerify(Fst *fst); + + +//refactor this function +bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); #endif diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index ddd7e1f450..0645aa1158 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -41,7 +41,7 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); void fstBuilderNodeDestroy(FstBuilderNode *node); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 6c1ea8cfeb..f9cb68baa9 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -733,6 +733,11 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr return true; } +bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { + return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); +} + + FstBuilder *fstBuilderCreate(void *w, FstType ty) { FstBuilder *b = malloc(sizeof(FstBuilder)); diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5452f9cb89..b33b8e4428 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -59,26 +59,27 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { src->trans = NULL; } -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { - size_t sz = taosArrayGetSize(b->trans); - assert(sz < 256); - if (FST_BUILDER_NODE_IS_FINAL(b) - && FST_BUILDER_NODE_TRANS_ISEMPTY(b) - && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { - return true; - } else if (sz != 1 || b->isFinal) { - // AnyTrans->Compile(w, addr, node); - } else { - FstTransition *tran = taosArrayGet(b->trans, 0); - if (tran->addr == lastAddr && tran->out == 0) { - //OneTransNext::compile(w, lastAddr, tran->inp); - return true; - } else { - //OneTrans::Compile(w, lastAddr, *tran); - return true; - } - } - return true; -} +//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { + + //size_t sz = taosArrayGetSize(b->trans); + //assert(sz < 256); + //if (FST_BUILDER_NODE_IS_FINAL(b) + // && FST_BUILDER_NODE_TRANS_ISEMPTY(b) + // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { + // return true; + //} else if (sz != 1 || b->isFinal) { + // // AnyTrans->Compile(w, addr, node); + //} else { + // FstTransition *tran = taosArrayGet(b->trans, 0); + // if (tran->addr == lastAddr && tran->out == 0) { + // //OneTransNext::compile(w, lastAddr, tran->inp); + // return true; + // } else { + // //OneTrans::Compile(w, lastAddr, *tran); + // return true; + // } + //} + //return true; +//} From a09dcbdd85fd343f49963d561bc3c805bd2479d4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 Nov 2021 10:59:51 +0800 Subject: [PATCH 2/3] update fst core struct --- source/libs/index/src/index_fst.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index f9cb68baa9..1ee364d4ad 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1026,10 +1026,10 @@ bool fstVerify(Fst *fst) { uint32_t checkSum = fst->meta->checkSum; FstSlice *data = fst->data; TSCKSUM initSum = 0; - if (taosCheckChecksumWhole(data->data, data->dLen)) { + if (!taosCheckChecksumWhole(data->data, data->dLen)) { return false; } - + return true; } From 2082e86476bbe7778e289013a7684619f7ad4e05 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 Nov 2021 14:01:25 +0800 Subject: [PATCH 3/3] update fst core struct --- source/libs/index/inc/index_fst.h | 2 ++ source/libs/index/src/index_fst.c | 27 +++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 01be2f8b2b..8f76c2d5cf 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -251,8 +251,10 @@ void fstDestroy(Fst *fst); bool fstGet(Fst *fst, FstSlice *b, Output *out); FstNode* fstGetNode(Fst *fst, CompiledAddr); +FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); + Output fstEmptyFinalOutput(Fst *fst, bool *null); bool fstVerify(Fst *fst); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 1ee364d4ad..94be05d0c1 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -990,15 +990,39 @@ void fstDestroy(Fst *fst) { } bool fstGet(Fst *fst, FstSlice *b, Output *out) { - + FstNode *root = fstGetRoot(fst); + Output tOut = 0; + for (uint32_t i = 0; i < b->dLen; i++) { + uint8_t inp = b->data[i]; + Output res = 0; + bool null = fstNodeFindInput(root, inp, &res); + if (null) { return false; } + + FstTransition trn; + fstNodeGetTransitionAt(root, res, &trn); + tOut += trn.out; + root = fstGetNode(fst, trn.addr); + } + if (!FST_NODE_IS_FINAL(root)) { + return false; + } else { + tOut = tOut + FST_NODE_FINAL_OUTPUT(root); + } + *out = tOut; + return false; } +FstNode *fstGetRoot(Fst *fst) { + CompiledAddr root = fstGetRootAddr(fst); + return fstGetNode(fst, root); +} FstNode* fstGetNode(Fst *fst, CompiledAddr addr) { if (fst->root != NULL) { return fst->root; } fst->root = fstNodeCreate(fst->meta->version, addr, fst->data); + return fst->root; } @@ -1021,7 +1045,6 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { return res; } - bool fstVerify(Fst *fst) { uint32_t checkSum = fst->meta->checkSum; FstSlice *data = fst->data;