xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add tfile analysis tool

This commit is contained in:
yihaoDeng 2022-01-08 18:56:54 +08:00
parent f4389433f6
commit 2a17fa3948
3 changed files with 50 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
source/libs/index/src/index_tfile.c
View File

@ -527,9 +527,12 @@ static int tfileReaderLoadFst(TFileReader* reader) {
if (buf == NULL) { return -1; } if (buf == NULL) { return -1; }
WriterCtx* ctx = reader->ctx; WriterCtx* ctx = reader->ctx;
int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d ", nread, reader->header.fstOffset, ctx->file.buf, int64_t ts = taosGetTimestampUs();
ctx->file.size); int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
int64_t cost = taosGetTimestampUs() - ts;
indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost);
// we assuse fst size less than FST_MAX_SIZE // we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread < FST_MAX_SIZE); assert(nread > 0 && nread < FST_MAX_SIZE);

34
source/libs/index/test/fstTest.cc
View File

@ -1,4 +1,5 @@
#include <algorithm>
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <thread> #include <thread>
@ -12,7 +13,6 @@
#include "index_tfile.h" #include "index_tfile.h"
#include "tskiplist.h" #include "tskiplist.h"
#include "tutil.h" #include "tutil.h"
void* callback(void* s) { return s; } void* callback(void* s) { return s; }
static std::string fileName = "/tmp/tindex.tindex"; static std::string fileName = "/tmp/tindex.tindex";
@ -293,7 +293,7 @@ void validateTFile(char* arg) {
std::thread threads[NUM_OF_THREAD]; std::thread threads[NUM_OF_THREAD];
// std::vector<std::thread> threads; // std::vector<std::thread> threads;
TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1");
for (int i = 0; i < NUM_OF_THREAD; i++) { for (int i = 0; i < NUM_OF_THREAD; i++) {
threads[i] = std::thread(fst_get, reader->fst); threads[i] = std::thread(fst_get, reader->fst);
@ -306,13 +306,41 @@ void validateTFile(char* arg) {
} }
tfCleanup(); tfCleanup();
} }
void iterTFileReader(char* path, char* ver) {
tfInit();
int version = atoi(ver);
TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1");
Iterate* iter = tfileIteratorCreate(reader);
bool tn = iter ? iter->next(iter) : false;
int count = 0;
int termCount = 0;
while (tn == true) {
count++;
IterateValue* cv = iter->getValue(iter);
termCount += (int)taosArrayGetSize(cv->val);
printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val));
tn = iter->next(iter);
}
printf("total size: %d\n term count: %d\n", count, termCount);
tfileIteratorDestroy(iter);
tfCleanup();
}
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
// tool to check all kind of fst test // tool to check all kind of fst test
// if (argc > 1) { validateTFile(argv[1]); } // if (argc > 1) { validateTFile(argv[1]); }
if (argc > 2) {
// opt
iterTFileReader(argv[1], argv[2]);
}
// checkFstCheckIterator(); // checkFstCheckIterator();
// checkFstLongTerm(); // checkFstLongTerm();
// checkFstPrefixSearch(); // checkFstPrefixSearch();
checkMillonWriteAndReadOfFst(); // checkMillonWriteAndReadOfFst();
return 1; return 1;
} }

20
source/libs/index/test/indexTests.cc
View File

@ -665,14 +665,19 @@ class IndexObj {
size_t numOfTable = 100 * 10000) { size_t numOfTable = 100 * 10000) {
std::string tColVal = colVal; std::string tColVal = colVal;
size_t colValSize = tColVal.size(); size_t colValSize = tColVal.size();
int skip = 100;
numOfTable /= skip;
for (int i = 0; i < numOfTable; i++) { for (int i = 0; i < numOfTable; i++) {
tColVal[i % colValSize] = 'a' + i % 26; for (int k = 0; k < 10 && k < colVal.size(); k++) {
// opt
tColVal[rand() % colValSize] = 'a' + k % 26;
}
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
tColVal.c_str(), tColVal.size()); tColVal.c_str(), tColVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term); indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 10; i++) { for (size_t j = 0; j < skip; j++) {
int ret = Put(terms, i); int ret = Put(terms, j);
assert(ret == 0); assert(ret == 0);
} }
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
@ -939,10 +944,11 @@ TEST_F(IndexEnv2, testIndex_read_performance) {
TEST_F(IndexEnv2, testIndexMultiTag) { TEST_F(IndexEnv2, testIndexMultiTag) {
std::string path = "/tmp/multi_tag"; std::string path = "/tmp/multi_tag";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {}
index->WriteMultiMillonData("tag1", "Hello", 100 * 10000); int64_t st = taosGetTimestampUs();
index->WriteMultiMillonData("tag2", "Test", 100 * 10000); int32_t num = 1000 * 10000;
index->WriteMultiMillonData("tag3", "Test", 100 * 10000); index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num);
index->WriteMultiMillonData("tag4", "Test", 100 * 10000); std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl;
// index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000);
} }
TEST_F(IndexEnv2, testLongComVal) { TEST_F(IndexEnv2, testLongComVal) {
std::string path = "/tmp/long_colVal"; std::string path = "/tmp/long_colVal";