From cf5e188d06d6178afafde4d327e73d928cc426de Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Dec 2021 17:39:11 +0800 Subject: [PATCH 1/4] add test case --- source/libs/index/test/indexTests.cpp | 29 +++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 86f19e8044..475775c01e 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -132,13 +132,13 @@ class FstReadMemory { void Performance_fstWriteRecords(FstWriter *b) { std::string str("aa"); - for (int i = 0; i < 26; i++) { + for (int i = 0; i < 100; i++) { str[0] = 'a' + i; str.resize(2); - for(int j = 0; j < 26; j++) { + for(int j = 0; j < 100; j++) { str[1] = 'a' + j; str.resize(2); - for (int k = 0; k < 10; k++) { + for (int k = 0; k < 100; k++) { str.push_back('a'); b->Put(str, k); } @@ -161,16 +161,17 @@ void Performance_fstReadRecords(FstReadMemory *m) { } } -int main(int argc, char** argv) { - // test write - // +void validateFst() { + r + int val = 100; + int count = 100; FstWriter *fw = new FstWriter; { std::string key("ab"); int64_t val = 100; - for (int i = 0; i < 26; i++) { + for (int i = 0; i < count; i++) { key.push_back('a' + i); - fw->Put(key, val++); + fw->Put(key, val + i); } } delete fw; @@ -188,16 +189,22 @@ int main(int argc, char** argv) { } else { printf("failed to get(%s)\n", key.c_str()); } - for (int i = 0; i < 26; i++) { + for (int i = 0; i < count; i++) { key.push_back('a' + i); - if (m->Get(key, &out)) { + if (m->Get(key, &out) ) { + assert(val + i == out); printf("success to get (%s, %" PRId64")\n", key.c_str(), out); } else { printf("failed to get(%s)\n", key.c_str()); } } } - + delete m; + +} +int main(int argc, char** argv) { + // test write + validateFst(); return 1; } From d502191a59e9193c4cbd7edb0f29841d741875e9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Dec 2021 19:30:03 +0800 Subject: [PATCH 2/4] update test case --- source/libs/index/test/indexTests.cpp | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 475775c01e..82b5ca1e24 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -130,25 +130,28 @@ class FstReadMemory { -void Performance_fstWriteRecords(FstWriter *b) { +int Performance_fstWriteRecords(FstWriter *b) { std::string str("aa"); - for (int i = 0; i < 100; i++) { + int L = 100, M = 100, N = 10; + for (int i = 0; i < L; i++) { str[0] = 'a' + i; str.resize(2); - for(int j = 0; j < 100; j++) { + for(int j = 0; j < M; j++) { str[1] = 'a' + j; str.resize(2); - for (int k = 0; k < 100; k++) { + for (int k = 0; k < N; k++) { str.push_back('a'); b->Put(str, k); + printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str()); } } } + return L * M * N; } void Performance_fstReadRecords(FstReadMemory *m) { std::string str("a"); - for (int i = 0; i < 500; i++) { + for (int i = 0; i < 50; i++) { //std::string str("aa"); str.push_back('a'); uint64_t out, cost; @@ -160,22 +163,32 @@ void Performance_fstReadRecords(FstReadMemory *m) { } } } +void checkFstPerf() { + FstWriter *fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int num = Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + + printf("write %d record cost %" PRId64"us\n", num, e - s); + +} + void validateFst() { - r int val = 100; int count = 100; FstWriter *fw = new FstWriter; + // write { std::string key("ab"); - int64_t val = 100; for (int i = 0; i < count; i++) { key.push_back('a' + i); - fw->Put(key, val + i); + fw->Put(key, val - i); } } delete fw; + // read FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init() == false) { std::cout << "init readMemory failed" << std::endl; @@ -192,7 +205,7 @@ void validateFst() { for (int i = 0; i < count; i++) { key.push_back('a' + i); if (m->Get(key, &out) ) { - assert(val + i == out); + assert(val - i == out); printf("success to get (%s, %" PRId64")\n", key.c_str(), out); } else { printf("failed to get(%s)\n", key.c_str()); @@ -203,9 +216,7 @@ void validateFst() { } int main(int argc, char** argv) { - // test write - validateFst(); - + checkFstPerf(); return 1; } From 984f3023537b90409c0d050603dc562c57bc3961 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Dec 2021 20:08:40 +0800 Subject: [PATCH 3/4] update test case --- source/libs/index/src/index_fst.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 457b5422a4..cdd62c5f11 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -297,9 +297,10 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // any value greater than or equal to the number of transitions in // this node indicates an absent transition. uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); - for (uint8_t i = 0; i < 256; i++) { - index[i] = 255; - } + memset(index, 255, sizeof(uint8_t) * 256); + ///for (uint8_t i = 0; i < 256; i++) { + // index[i] = 255; + ///} for (size_t i = 0; i < sz; i++) { FstTransition *t = taosArrayGet(node->trans, i); index[t->inp] = i; @@ -1126,6 +1127,7 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { return b; } + bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { @@ -1378,7 +1380,9 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { } void fstStreamBuilderDestroy(FstStreamBuilder *b) { fstSliceDestroy(&b->min->data); + tfree(b->min); fstSliceDestroy(&b->max->data); + tfree(b->max); free(b); } FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { From 210db65b7d86a810be64ed6a239138c3185f6533 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Dec 2021 20:24:04 +0800 Subject: [PATCH 4/4] fix error resp on large kvs set --- source/libs/index/src/index_fst.c | 2 +- source/libs/index/test/indexTests.cpp | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index cdd62c5f11..403b4a9122 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -304,9 +304,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil for (size_t i = 0; i < sz; i++) { FstTransition *t = taosArrayGet(node->trans, i); index[t->inp] = i; - fstCountingWriterWrite(w, (char *)index, sizeof(index)); //fstPackDeltaIn(w, addr, t->addr, tSize); } + fstCountingWriterWrite(w, (char *)index, 256); free(index); } fstCountingWriterWrite(w, (char *)&packSizes, 1); diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 82b5ca1e24..928c3875b0 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -168,9 +168,18 @@ void checkFstPerf() { int64_t s = taosGetTimestampUs(); int num = Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); - printf("write %d record cost %" PRId64"us\n", num, e - s); - + delete fw; + + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init()) { + uint64_t val; + if(m->Get("aaaaaaa", &val)) { + std::cout << "succes to Get val: " << val << std::endl; + } else { + std::cout << "failed to Get " << std::endl; + } + } }