diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5167196031..6398259a96 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -360,6 +360,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { if (sz > 0) { // TODO(yihao): remove duplicate tableid TFileValue* lv = taosArrayGetP(result, sz - 1); + // indexError("merge colVal: %s", lv->colVal); if (strcmp(lv->colVal, tv->colVal) == 0) { taosArrayAddAll(lv->tableId, tv->tableId); tfileValueDestroy(tv); @@ -368,6 +369,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { } } else { taosArrayPush(result, &tv); + // indexError("merge colVal: %s", tv->colVal); } } static void indexDestroyTempResult(SArray* result) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 0e46445a00..503d7cd928 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10000 * 10 +#define MEM_TERM_LIMIT 10 * 10000 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -353,6 +353,9 @@ static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; if (iter == NULL) { return false; } IterateValue* iv = &itera->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); bool next = tSkipListIterNext(iter); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 04a08dafd2..088c7369d5 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; - COMMON_INDEX(inp, 0x111111, val); + COMMON_INDEX(inp, 0b111111, val); s->val = (s->val & fstStateDict[s->state].val) | val; } @@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { bool null = false; uint8_t inp = fstStateCommInput(s, &null); uint8_t* data = fstSliceData(slice, NULL); - return null == false ? inp : data[-1]; + return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fc4f8593a1..6669198861 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -385,8 +385,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { } static bool tfileIteratorNext(Iterate* iiter) { IterateValue* iv = &iiter->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); - // SArray* tblIds = iv->val; char* colVal = NULL; uint64_t offset = 0; @@ -406,7 +408,7 @@ static bool tfileIteratorNext(Iterate* iiter) { if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; - + return true; // std::string key(ch, sz); } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index da974ce6c4..3d978c05a5 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -24,8 +24,13 @@ class FstWriter { _b = fstBuilderCreate(_wc, 0); } bool Put(const std::string& key, uint64_t val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); return ok; } @@ -61,6 +66,11 @@ class FstReadMemory { return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstGet(_fst, &skey, val); fstSliceDestroy(&skey); @@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) { } return L * M * N; } +void Performance_fstReadRecords(FstReadMemory* m) { + std::string str("aa"); + for (int i = 0; i < M; i++) { + str[0] = 'a' + i; + str.resize(2); + for (int j = 0; j < N; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < L; k++) { + str.push_back('a'); + uint64_t val, cost; + if (m->GetWithTimeCostUs(str, &val, &cost)) { + printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost); + } else { + printf("failed to get key: %s\n", str.c_str()); + } + } + } + } +} + +void checkMillonWriteAndReadOfFst() { + tfInit(); + FstWriter* fw = new FstWriter; + Performance_fstWriteRecords(fw); + delete fw; + FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); + + if (fr->init()) { printf("success to init fst read"); } + + Performance_fstReadRecords(fr); + tfCleanup(); + delete fr; +} +void checkFstLongTerm() { + tfInit(); + FstWriter* fw = new FstWriter; + // Performance_fstWriteRecords(fw); + + fw->Put("A B", 1); + fw->Put("C", 2); + fw->Put("a", 3); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + uint64_t val = 0; + if (m->Get("A B", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("C", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("a", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + + // prefix search + // std::vector result; + + // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + // m->Search(ctx, result); + // std::cout << "size: " << result.size() << std::endl; + // assert(result.size() == count); + // for (int i = 0; i < result.size(); i++) { + // assert(result[i] == i); // check result + //} + tfCleanup(); + // free(ctx); + // delete m; +} void checkFstCheckIterator() { tfInit(); FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); int count = 2; - Performance_fstWriteRecords(fw); + // Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("Hello world", 1); + fw->Put("hello world", 2); + fw->Put("hello worle", 3); + fw->Put("hello worlf", 4); delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); @@ -171,7 +275,7 @@ void checkFstCheckIterator() { void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { - std::string term = "Hello"; + std::string term = "Hello World"; FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size()); uint64_t offset = 0; bool ret = fstGet(fst, &key, &offset); @@ -189,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -203,9 +307,12 @@ void validateTFile(char* arg) { tfCleanup(); } int main(int argc, char* argv[]) { - if (argc > 1) { validateTFile(argv[1]); } + // tool to check all kind of fst test + // if (argc > 1) { validateTFile(argv[1]); } // checkFstCheckIterator(); + // checkFstLongTerm(); // checkFstPrefixSearch(); + checkMillonWriteAndReadOfFst(); return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 080becccf1..3ad64cd03e 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -787,14 +787,15 @@ TEST_F(IndexEnv2, testIndexOpen) { } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/test"; + std::string path = "/tmp/test1"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; } int numOfTable = 100 * 10000; - index->WriteMillonData("tag1", "Hello", numOfTable); - int target = index->SearchOne("tag1", "Hello"); + index->WriteMillonData("tag1", "Hello Wolrd", numOfTable); + int target = index->SearchOne("tag1", "Hello Wolrd"); + std::cout << "Get Index: " << target << std::endl; assert(numOfTable == target); } @@ -821,14 +822,6 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { threads[i].join(); } } -TEST_F(IndexEnv2, testIndex_multi_thread_write) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} -TEST_F(IndexEnv2, testIndex_multi_thread_read) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp";