294 lines
7.4 KiB
C++
294 lines
7.4 KiB
C++
#include <gtest/gtest.h>
|
|
#include <string>
|
|
#include <iostream>
|
|
#include "index.h"
|
|
#include "tutil.h"
|
|
#include "indexInt.h"
|
|
#include "index_fst.h"
|
|
#include "index_fst_util.h"
|
|
#include "index_fst_counting_writer.h"
|
|
|
|
|
|
class FstWriter {
|
|
public:
|
|
FstWriter() {
|
|
_b = fstBuilderCreate(NULL, 0);
|
|
}
|
|
bool Put(const std::string &key, uint64_t val) {
|
|
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
|
|
bool ok = fstBuilderInsert(_b, skey, val);
|
|
fstSliceDestroy(&skey);
|
|
return ok;
|
|
}
|
|
~FstWriter() {
|
|
fstBuilderFinish(_b);
|
|
fstBuilderDestroy(_b);
|
|
}
|
|
private:
|
|
FstBuilder *_b;
|
|
};
|
|
|
|
class FstReadMemory {
|
|
public:
|
|
FstReadMemory(size_t size) {
|
|
_w = fstCountingWriterCreate(NULL, true);
|
|
_size = size;
|
|
memset((void *)&_s, 0, sizeof(_s));
|
|
}
|
|
bool init() {
|
|
char *buf = (char *)calloc(1, sizeof(char) * _size);
|
|
int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size);
|
|
if (nRead <= 0) { return false; }
|
|
_size = nRead;
|
|
_s = fstSliceCreate((uint8_t *)buf, _size);
|
|
_fst = fstCreate(&_s);
|
|
free(buf);
|
|
return _fst != NULL;
|
|
}
|
|
bool Get(const std::string &key, uint64_t *val) {
|
|
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
|
|
bool ok = fstGet(_fst, &skey, val);
|
|
fstSliceDestroy(&skey);
|
|
return ok;
|
|
}
|
|
bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) {
|
|
int64_t s = taosGetTimestampUs();
|
|
bool ok = this->Get(key, val);
|
|
int64_t e = taosGetTimestampUs();
|
|
*elapse = e - s;
|
|
return ok;
|
|
}
|
|
// add later
|
|
bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) {
|
|
FstStreamBuilder *sb = fstSearch(_fst, ctx);
|
|
StreamWithState *st = streamBuilderIntoStream(sb);
|
|
StreamWithStateResult *rt = NULL;
|
|
|
|
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
|
|
result.push_back((uint64_t)(rt->out.out));
|
|
}
|
|
return true;
|
|
}
|
|
bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
|
|
int64_t s = taosGetTimestampUs();
|
|
bool ok = this->Search(ctx, result);
|
|
int64_t e = taosGetTimestampUs();
|
|
return ok;
|
|
}
|
|
|
|
~FstReadMemory() {
|
|
fstCountingWriterDestroy(_w);
|
|
fstDestroy(_fst);
|
|
fstSliceDestroy(&_s);
|
|
}
|
|
|
|
private:
|
|
FstCountingWriter *_w;
|
|
Fst *_fst;
|
|
FstSlice _s;
|
|
size_t _size;
|
|
|
|
};
|
|
|
|
//TEST(IndexTest, index_create_test) {
|
|
// SIndexOpts *opts = indexOptsCreate();
|
|
// SIndex *index = indexOpen(opts, "./test");
|
|
// if (index == NULL) {
|
|
// std::cout << "index open failed" << std::endl;
|
|
// }
|
|
//
|
|
//
|
|
// // write
|
|
// for (int i = 0; i < 100000; i++) {
|
|
// SIndexMultiTerm* terms = indexMultiTermCreate();
|
|
// std::string val = "field";
|
|
//
|
|
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
|
|
//
|
|
// val.append(std::to_string(i));
|
|
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
|
|
//
|
|
// val.insert(0, std::to_string(i));
|
|
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
|
|
//
|
|
// val.append("const");
|
|
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
|
|
//
|
|
//
|
|
// indexPut(index, terms, i);
|
|
// indexMultiTermDestroy(terms);
|
|
// }
|
|
//
|
|
//
|
|
// // query
|
|
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
|
|
//
|
|
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
|
|
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
|
|
//
|
|
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
|
|
// indexSearch(index, multiQuery, result);
|
|
//
|
|
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
|
|
// for (int i = 0; i < taosArrayGetSize(result); i++) {
|
|
// int *v = (int *)taosArrayGet(result, i);
|
|
// std::cout << "value --->" << *v << std::endl;
|
|
// }
|
|
// // add more test case
|
|
// indexMultiTermQueryDestroy(multiQuery);
|
|
//
|
|
// indexOptsDestroy(opts);
|
|
// indexClose(index);
|
|
// //
|
|
//}
|
|
|
|
|
|
#define L 100
|
|
#define M 100
|
|
#define N 100
|
|
|
|
int Performance_fstWriteRecords(FstWriter *b) {
|
|
std::string str("aa");
|
|
for (int i = 0; i < L; i++) {
|
|
str[0] = 'a' + i;
|
|
str.resize(2);
|
|
for(int j = 0; j < M; j++) {
|
|
str[1] = 'a' + j;
|
|
str.resize(2);
|
|
for (int k = 0; k < N; k++) {
|
|
str.push_back('a');
|
|
b->Put(str, k);
|
|
printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
|
|
}
|
|
}
|
|
}
|
|
return L * M * N;
|
|
}
|
|
|
|
void Performance_fstReadRecords(FstReadMemory *m) {
|
|
std::string str("aa");
|
|
for (int i = 0; i < M; i++) {
|
|
str[0] = 'a' + i;
|
|
str.resize(2);
|
|
for(int j = 0; j < N; j++) {
|
|
str[1] = 'a' + j;
|
|
str.resize(2);
|
|
for (int k = 0; k < L; k++) {
|
|
str.push_back('a');
|
|
uint64_t val, cost;
|
|
if (m->GetWithTimeCostUs(str, &val, &cost)) {
|
|
printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
|
|
} else {
|
|
printf("failed to get key: %s\n", str.c_str());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
void checkFstPerf() {
|
|
FstWriter *fw = new FstWriter;
|
|
int64_t s = taosGetTimestampUs();
|
|
|
|
int num = Performance_fstWriteRecords(fw);
|
|
int64_t e = taosGetTimestampUs();
|
|
printf("write %d record cost %" PRId64"us\n", num, e - s);
|
|
delete fw;
|
|
|
|
FstReadMemory *m = new FstReadMemory(1024 * 64);
|
|
if (m->init()) {
|
|
printf("success to init fst read");
|
|
}
|
|
Performance_fstReadRecords(m);
|
|
delete m;
|
|
}
|
|
|
|
void checkFstPrefixSearch() {
|
|
FstWriter *fw = new FstWriter;
|
|
int64_t s = taosGetTimestampUs();
|
|
int count = 2;
|
|
std::string key("ab");
|
|
|
|
for (int i = 0; i < count; i++) {
|
|
key[1] = key[1] + i;
|
|
fw->Put(key, i);
|
|
}
|
|
int64_t e = taosGetTimestampUs();
|
|
|
|
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
|
|
delete fw;
|
|
|
|
FstReadMemory *m = new FstReadMemory(1024 * 64);
|
|
if (m->init() == false) {
|
|
std::cout << "init readMemory failed" << std::endl;
|
|
delete m;
|
|
return;
|
|
}
|
|
|
|
// prefix search
|
|
std::vector<uint64_t> result;
|
|
AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX);
|
|
m->Search(ctx, result);
|
|
assert(result.size() == count);
|
|
for (int i = 0; i < result.size(); i++) {
|
|
assert(result[i] == i); // check result
|
|
}
|
|
|
|
free(ctx);
|
|
delete m;
|
|
}
|
|
void validateFst() {
|
|
int val = 100;
|
|
int count = 100;
|
|
FstWriter *fw = new FstWriter;
|
|
// write
|
|
{
|
|
std::string key("ab");
|
|
for (int i = 0; i < count; i++) {
|
|
key.push_back('a' + i);
|
|
fw->Put(key, val - i);
|
|
}
|
|
}
|
|
delete fw;
|
|
|
|
// read
|
|
FstReadMemory *m = new FstReadMemory(1024 * 64);
|
|
if (m->init() == false) {
|
|
std::cout << "init readMemory failed" << std::endl;
|
|
delete m;
|
|
return;
|
|
}
|
|
|
|
{
|
|
std::string key("ab");
|
|
uint64_t out;
|
|
if (m->Get(key, &out)) {
|
|
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
|
|
} else {
|
|
printf("failed to get(%s)\n", key.c_str());
|
|
}
|
|
for (int i = 0; i < count; i++) {
|
|
key.push_back('a' + i);
|
|
if (m->Get(key, &out) ) {
|
|
assert(val - i == out);
|
|
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
|
|
} else {
|
|
printf("failed to get(%s)\n", key.c_str());
|
|
}
|
|
}
|
|
}
|
|
delete m;
|
|
}
|
|
|
|
|
|
int main(int argc, char** argv) {
|
|
checkFstPerf();
|
|
//checkFstPrefixSearch();
|
|
return 1;
|
|
}
|
|
|
|
//TEST(IndexFstBuilder, IndexFstInput) {
|
|
//
|
|
//}
|
|
|
|
|