xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add skiplist add

This commit is contained in:
yihaoDeng 2021-12-24 23:55:15 +08:00
parent 6694e4b2da
commit 7ca0a88ee2
1 changed files with 286 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

364
source/libs/index/test/indexTests.cc
View File

@ -16,10 +16,12 @@
#include <string> #include <string>
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h" #include "index_fst.h"
#include "index_fst_counting_writer.h" #include "index_fst_counting_writer.h"
#include "index_fst_util.h" #include "index_fst_util.h"
#include "index_tfile.h" #include "index_tfile.h"
#include "tskiplist.h"
#include "tutil.h" #include "tutil.h"
using namespace std; using namespace std;
class FstWriter { class FstWriter {
@ -110,58 +112,6 @@ class FstReadMemory {
size_t _size; size_t _size;
}; };
// TEST(IndexTest, index_create_test) {
// SIndexOpts *opts = indexOptsCreate();
// SIndex *index = indexOpen(opts, "./test");
// if (index == NULL) {
// std::cout << "index open failed" << std::endl;
// }
//
//
// // write
// for (int i = 0; i < 100000; i++) {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// std::string val = "field";
//
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
//
// val.append(std::to_string(i));
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
//
// val.insert(0, std::to_string(i));
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
//
// val.append("const");
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
//
//
// indexPut(index, terms, i);
// indexMultiTermDestroy(terms);
// }
//
//
// // query
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
//
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// indexSearch(index, multiQuery, result);
//
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
// for (int i = 0; i < taosArrayGetSize(result); i++) {
// int *v = (int *)taosArrayGet(result, i);
// std::cout << "value --->" << *v << std::endl;
// }
// // add more test case
// indexMultiTermQueryDestroy(multiQuery);
//
// indexOptsDestroy(opts);
// indexClose(index);
// //
//}
#define L 100 #define L 100
#define M 100 #define M 100
#define N 100 #define N 100
@ -421,6 +371,7 @@ class TFileObj {
int colId_; int colId_;
}; };
class IndexTFileEnv : public ::testing::Test { class IndexTFileEnv : public ::testing::Test {
protected: protected:
virtual void SetUp() { virtual void SetUp() {
@ -428,22 +379,6 @@ class IndexTFileEnv : public ::testing::Test {
taosMkDir(dir.c_str()); taosMkDir(dir.c_str());
tfInit(); tfInit();
fObj = new TFileObj(dir, colName); fObj = new TFileObj(dir, colName);
// std::string colName("voltage");
// header.suid = 1;
// header.version = 1;
// memcpy(header.colName, colName.c_str(), colName.size());
// header.colType = TSDB_DATA_TYPE_BINARY;
// std::string path(dir);
// int colId = 2;
// char buf[64] = {0};
// sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version);
// path.append("/").append(buf);
// ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
// twrite = tfileWriterCreate(ctx, &header);
} }
virtual void TearDown() { virtual void TearDown() {
@ -460,16 +395,8 @@ class IndexTFileEnv : public ::testing::Test {
int coldId = 2; int coldId = 2;
int version = 1; int version = 1;
int colType = TSDB_DATA_TYPE_BINARY; int colType = TSDB_DATA_TYPE_BINARY;
// WriterCtx* ctx = NULL;
// TFileHeader header;
// TFileWriter* twrite = NULL;
}; };
// static TFileWriter* genTFileWriter(const char* path, TFileHeader* header) {
// char buf[128] = {0};
// WriterCtx* ctx = writerCtxCreate(TFile, path, false, )
//}
static TFileValue* genTFileValue(const char* val) { static TFileValue* genTFileValue(const char* val) {
TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue));
int32_t vlen = strlen(val) + 1; int32_t vlen = strlen(val) + 1;
@ -492,7 +419,7 @@ static void destroyTFileValue(void* val) {
TEST_F(IndexTFileEnv, test_tfile_write) { TEST_F(IndexTFileEnv, test_tfile_write) {
TFileValue* v1 = genTFileValue("c"); TFileValue* v1 = genTFileValue("c");
TFileValue* v2 = genTFileValue("a"); TFileValue* v2 = genTFileValue("ab");
TFileValue* v3 = genTFileValue("b"); TFileValue* v3 = genTFileValue("b");
TFileValue* v4 = genTFileValue("d"); TFileValue* v4 = genTFileValue("d");
@ -510,7 +437,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
taosArrayDestroy(data); taosArrayDestroy(data);
std::string colName("voltage"); std::string colName("voltage");
std::string colVal("b"); std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
@ -521,3 +448,284 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
// tfileWriterDestroy(twrite); // tfileWriterDestroy(twrite);
} }
class CacheObj {
public:
CacheObj() {
// TODO
cache = indexCacheCreate();
}
int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
int ret = indexCachePut(cache, term, colId, version, uid);
if (ret != 0) {
//
std::cout << "failed to put into cache: " << ret << std::endl;
}
return ret;
}
int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
int ret = indexCacheSearch(cache, query, colId, version, result, s);
if (ret != 0) {
//
std::cout << "failed to get from cache:" << ret << std::endl;
}
return ret;
}
~CacheObj() {
// TODO
indexCacheDestroy(cache);
}
private:
IndexCache* cache = NULL;
};
class IndexCacheEnv : public ::testing::Test {
protected:
virtual void SetUp() {
// TODO
coj = new CacheObj();
}
virtual void TearDown() {
delete coj;
// formate
}
CacheObj* coj;
};
TEST_F(IndexCacheEnv, cache_test) {
int count = 10;
int16_t colId = 1;
int32_t version = 10;
uint64_t suid = 100;
std::string colName("voltage");
std::string colVal("My God");
for (size_t i = 0; i < count; i++) {
colVal += ('a' + i);
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
coj->Put(term, colId, version, suid);
version++;
}
// coj->Get();
}
typedef struct CTerm {
char buf[16];
char version[8];
int val;
int other;
} CTerm;
CTerm* cTermCreate(const char* str, const char* version, int val) {
CTerm* tm = (CTerm*)calloc(1, sizeof(CTerm));
memcpy(tm->buf, str, strlen(str));
memcpy(tm->version, version, strlen(version));
tm->val = val;
tm->other = -100;
return tm;
}
int termCompar(const void* a, const void* b) {
printf("a: %s \t b: %s\n", (char*)a, (char*)b);
int ret = strncmp((char*)a, (char*)b, 16);
if (ret == 0) {
//
return strncmp((char*)a + 16, (char*)b + 16, 8);
}
return ret;
}
int SerialTermTo(char* buf, CTerm* term) {
char* p = buf;
memcpy(buf, term->buf, sizeof(term->buf));
buf += sizeof(term->buf);
// memcpy(buf, term->version, sizeof(term->version));
// buf += sizeof(term->version);
return buf - p;
}
static char* getTermKey(const void* pData) {
CTerm* p = (CTerm*)pData;
return (char*)p->buf;
}
#define MAX_TERM_KEY_LEN 128
class SkiplistObj {
public:
// max_key_len:
//
SkiplistObj() {
slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey);
}
int Put(CTerm* term, uint64_t suid) {
char buf[MAX_TERM_KEY_LEN] = {0};
int sz = SerialTermTo(buf, term);
char* pBuf = (char*)calloc(1, sz + sizeof(suid));
memcpy(pBuf, buf, sz);
memcpy(pBuf + sz, &suid, sizeof(suid));
// int32_t level, headsize;
// tSkipListNewNodeInfo(slt, &level, &headsize);
// SSkipListNode* node = (SSkipListNode*)calloc(1, headsize + strlen(buf) + sizeof(suid));
// node->level = level;
// char* d = (char*)SL_GET_NODE_DATA(node);
// memcpy(d, buf, strlen(buf));
// memcpy(d + strlen(buf), &suid, sizeof(suid));
SSkipListNode* node = tSkipListPut(slt, pBuf);
tSkipListPrint(slt, 1);
free(pBuf);
return 0;
}
int Get(int key, char* buf, int version) {
// CTerm term;
// term.key = key;
//// term.version = version;
// memcpy(term.buf, buf, strlen(buf));
// char tbuf[128] = {0};
// SerialTermTo(tbuf, &term);
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
// SSkipListNode* node = tSkipListIterGet(iter);
// CTerm* ct = (CTerm*)SL_GET_NODE_DATA(node);
// printf("key: %d\t, version: %d\t, buf: %s\n", ct->key, ct->version, ct->buf);
// while (iter) {
// assert(tSkipListIterNext(iter) == true);
// SSkipListNode* node = tSkipListIterGet(iter);
// // ugly formate
// CTerm* t = (CTerm*)SL_GET_NODE_KEY(slt, node);
// printf("key: %d\t, version: %d\t, buf: %s\n", t->key, t->version, t->buf);
//}
return 0;
}
~SkiplistObj() {
// TODO
// indexCacheDestroy(cache);
}
private:
SSkipList* slt;
};
typedef struct KV {
int32_t k;
int32_t v;
} KV;
int kvCompare(const void* a, const void* b) {
int32_t av = *(int32_t*)a;
int32_t bv = *(int32_t*)b;
return av - bv;
}
char* getKVkey(const void* a) {
return (char*)(&(((KV*)a)->v));
// KV* kv = (KV*)a;
}
int testKV() {
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, kvCompare, SL_DISCARD_DUP_KEY, getKVkey);
{
KV t = {.k = 1, .v = 5};
tSkipListPut(slt, (void*)&t);
}
{
KV t = {.k = 2, .v = 3};
tSkipListPut(slt, (void*)&t);
}
KV value = {.k = 4, .v = 5};
char* key = getKVkey(&value);
// const char* key = "Hello";
SArray* arr = tSkipListGet(slt, (SSkipListKey)&key);
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
int32_t* ct = (int32_t*)SL_GET_NODE_KEY(slt, node);
printf("Get key: %d\n", *ct);
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
}
return 1;
}
int testComplicate() {
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey);
{
CTerm* tm = cTermCreate("val", "v1", 10);
tSkipListPut(slt, (char*)tm);
}
{
CTerm* tm = cTermCreate("val1", "v2", 2);
tSkipListPut(slt, (char*)tm);
}
{
CTerm* tm = cTermCreate("val3", "v3", -1);
tSkipListPut(slt, (char*)tm);
}
{
CTerm* tm = cTermCreate("val3", "v4", 2);
tSkipListPut(slt, (char*)tm);
}
{
CTerm* tm = cTermCreate("val3", "v5", -1);
char* key = getTermKey(tm);
SArray* arr = tSkipListGet(slt, (SSkipListKey)key);
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
CTerm* ct = (CTerm*)SL_GET_NODE_KEY(slt, node);
printf("other; %d\tbuf: %s\t, version: %s, val: %d\n", ct->other, ct->buf, ct->version, ct->val);
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
}
free(tm);
taosArrayDestroy(arr);
}
return 1;
}
int strCompare(const void* a, const void* b) {
const char* sa = (char*)a;
const char* sb = (char*)b;
return strcmp(sa, sb);
}
void testString() {
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, strCompare, SL_ALLOW_DUP_KEY, getTermKey);
{
tSkipListPut(slt, (void*)"Hello");
tSkipListPut(slt, (void*)"World");
tSkipListPut(slt, (void*)"YI");
}
const char* key = "YI";
SArray* arr = tSkipListGet(slt, (SSkipListKey)key);
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
char* ct = (char*)SL_GET_NODE_KEY(slt, node);
printf("Get key: %s\n", ct);
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
}
}
// class IndexSkip : public ::testing::Test {
// protected:
// virtual void SetUp() {
// // TODO
// sObj = new SkiplistObj();
// }
// virtual void TearDown() {
// delete sObj;
// // formate
// }
// SkiplistObj* sObj;
//};
// TEST_F(IndexSkip, skip_test) {
// std::string val("Hello");
// std::string minVal = val;
// for (size_t i = 0; i < 10; i++) {
// CTerm* t = (CTerm*)calloc(1, sizeof(CTerm));
// t->key = 1;
// t->version = i;
//
// val[val.size() - 1] = 'a' + i;
// memcpy(t->buf, val.c_str(), val.size());
// sObj->Put(t, 10);
// free(t);
// }
// sObj->Get(1, (char*)(minVal.c_str()), 1000000);
//}