xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix fst bug

This commit is contained in:
yihaoDeng 2021-12-31 18:06:13 +08:00
parent 2eb0053e5a
commit 9920b43be5
6 changed files with 127 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/libs/index/src/index.c
View File

@ -360,6 +360,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
if (sz > 0) {
// TODO(yihao): remove duplicate tableid
TFileValue* lv = taosArrayGetP(result, sz - 1);
// indexError("merge colVal: %s", lv->colVal);
if (strcmp(lv->colVal, tv->colVal) == 0) {
taosArrayAddAll(lv->tableId, tv->tableId);
tfileValueDestroy(tv);
@ -368,6 +369,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
}
} else {
taosArrayPush(result, &tv);
// indexError("merge colVal: %s", tv->colVal);
}
}
static void indexDestroyTempResult(SArray* result) {

5
source/libs/index/src/index_cache.c
View File

@ -20,7 +20,7 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later
#define MEM_TERM_LIMIT 10000 * 10
#define MEM_TERM_LIMIT 10 * 10000
// ref index_cache.h:22
//#define CACHE_KEY_LEN(p) \
// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) +
@ -353,6 +353,9 @@ static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter;
if (iter == NULL) { return false; }
IterateValue* iv = &itera->val;
if (iv->colVal != NULL && iv->val != NULL) {
// indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
}
iterateValueDestroy(iv, false);
bool next = tSkipListIterNext(iter);

4
source/libs/index/src/index_fst.c
View File

@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) {
assert(s->state == OneTransNext || s->state == OneTrans);
uint8_t val;
COMMON_INDEX(inp, 0x111111, val);
COMMON_INDEX(inp, 0b111111, val);
s->val = (s->val & fstStateDict[s->state].val) | val;
}
@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) {
bool null = false;
uint8_t inp = fstStateCommInput(s, &null);
uint8_t* data = fstSliceData(slice, NULL);
return null == false ? inp : data[-1];
return null == false ? inp : data[node->start - 1];
}
uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);

6
source/libs/index/src/index_tfile.c
View File

@ -385,8 +385,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
}
static bool tfileIteratorNext(Iterate* iiter) {
IterateValue* iv = &iiter->val;
if (iv->colVal != NULL && iv->val != NULL) {
// indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
}
iterateValueDestroy(iv, false);
// SArray* tblIds = iv->val;
char* colVal = NULL;
uint64_t offset = 0;
@ -406,7 +408,7 @@ static bool tfileIteratorNext(Iterate* iiter) {
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
iv->colVal = colVal;
return true;
// std::string key(ch, sz);
}

115
source/libs/index/test/fstTest.cc
View File

@ -24,8 +24,13 @@ class FstWriter {
_b = fstBuilderCreate(_wc, 0);
}
bool Put(const std::string& key, uint64_t val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
@ -61,6 +66,11 @@ class FstReadMemory {
return _fst != NULL;
}
bool Get(const std::string& key, uint64_t* val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) {
}
return L * M * N;
}
void Performance_fstReadRecords(FstReadMemory* m) {
std::string str("aa");
for (int i = 0; i < M; i++) {
str[0] = 'a' + i;
str.resize(2);
for (int j = 0; j < N; j++) {
str[1] = 'a' + j;
str.resize(2);
for (int k = 0; k < L; k++) {
str.push_back('a');
uint64_t val, cost;
if (m->GetWithTimeCostUs(str, &val, &cost)) {
printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
} else {
printf("failed to get key: %s\n", str.c_str());
}
}
}
}
}
void checkMillonWriteAndReadOfFst() {
tfInit();
FstWriter* fw = new FstWriter;
Performance_fstWriteRecords(fw);
delete fw;
FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
if (fr->init()) { printf("success to init fst read"); }
Performance_fstReadRecords(fr);
tfCleanup();
delete fr;
}
void checkFstLongTerm() {
tfInit();
FstWriter* fw = new FstWriter;
// Performance_fstWriteRecords(fw);
fw->Put("A B", 1);
fw->Put("C", 2);
fw->Put("a", 3);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
uint64_t val = 0;
if (m->Get("A B", &val)) {
std::cout << "success to Get: " << val << std::endl;
} else {
std::cout << "failed to Get:" << val << std::endl;
}
}
{
uint64_t val = 0;
if (m->Get("C", &val)) {
std::cout << "success to Get: " << val << std::endl;
} else {
std::cout << "failed to Get:" << val << std::endl;
}
}
{
uint64_t val = 0;
if (m->Get("a", &val)) {
std::cout << "success to Get: " << val << std::endl;
} else {
std::cout << "failed to Get:" << val << std::endl;
}
}
// prefix search
// std::vector<uint64_t> result;
// AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
// m->Search(ctx, result);
// std::cout << "size: " << result.size() << std::endl;
// assert(result.size() == count);
// for (int i = 0; i < result.size(); i++) {
// assert(result[i] == i); // check result
//}
tfCleanup();
// free(ctx);
// delete m;
}
void checkFstCheckIterator() {
tfInit();
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
Performance_fstWriteRecords(fw);
// Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("Hello world", 1);
fw->Put("hello world", 2);
fw->Put("hello worle", 3);
fw->Put("hello worlf", 4);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
@ -171,7 +275,7 @@ void checkFstCheckIterator() {
void fst_get(Fst* fst) {
for (int i = 0; i < 10000; i++) {
std::string term = "Hello";
std::string term = "Hello World";
FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size());
uint64_t offset = 0;
bool ret = fstGet(fst, &key, &offset);
@ -189,7 +293,7 @@ void validateTFile(char* arg) {
std::thread threads[NUM_OF_THREAD];
// std::vector<std::thread> threads;
TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1");
TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1");
for (int i = 0; i < NUM_OF_THREAD; i++) {
threads[i] = std::thread(fst_get, reader->fst);
@ -203,9 +307,12 @@ void validateTFile(char* arg) {
tfCleanup();
}
int main(int argc, char* argv[]) {
if (argc > 1) { validateTFile(argv[1]); }
// tool to check all kind of fst test
// if (argc > 1) { validateTFile(argv[1]); }
// checkFstCheckIterator();
// checkFstLongTerm();
// checkFstPrefixSearch();
checkMillonWriteAndReadOfFst();
return 1;
}

15
source/libs/index/test/indexTests.cc
View File

@ -787,14 +787,15 @@ TEST_F(IndexEnv2, testIndexOpen) {
}
TEST_F(IndexEnv2, testIndex_TrigeFlush) {
std::string path = "/tmp/test";
std::string path = "/tmp/test1";
if (index->Init(path) != 0) {
// r
std::cout << "failed to init" << std::endl;
}
int numOfTable = 100 * 10000;
index->WriteMillonData("tag1", "Hello", numOfTable);
int target = index->SearchOne("tag1", "Hello");
index->WriteMillonData("tag1", "Hello Wolrd", numOfTable);
int target = index->SearchOne("tag1", "Hello Wolrd");
std::cout << "Get Index: " << target << std::endl;
assert(numOfTable == target);
}
@ -821,14 +822,6 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
threads[i].join();
}
}
TEST_F(IndexEnv2, testIndex_multi_thread_write) {
std::string path = "/tmp";
if (index->Init(path) != 0) {}
}
TEST_F(IndexEnv2, testIndex_multi_thread_read) {
std::string path = "/tmp";
if (index->Init(path) != 0) {}
}
TEST_F(IndexEnv2, testIndex_restart) {
std::string path = "/tmp";