add skiplist add
This commit is contained in:
yihaoDeng
parent
6694e4b2da
commit
7ca0a88ee2
|
|
@ -16,10 +16,12 @@
|
|||
#include <string>
|
||||
#include "index.h"
|
||||
#include "indexInt.h"
|
||||
#include "index_cache.h"
|
||||
#include "index_fst.h"
|
||||
#include "index_fst_counting_writer.h"
|
||||
#include "index_fst_util.h"
|
||||
#include "index_tfile.h"
|
||||
#include "tskiplist.h"
|
||||
#include "tutil.h"
|
||||
using namespace std;
|
||||
class FstWriter {
|
||||
|
|
@ -110,58 +112,6 @@ class FstReadMemory {
|
|||
size_t _size;
|
||||
};
|
||||
|
||||
// TEST(IndexTest, index_create_test) {
|
||||
// SIndexOpts *opts = indexOptsCreate();
|
||||
// SIndex *index = indexOpen(opts, "./test");
|
||||
// if (index == NULL) {
|
||||
// std::cout << "index open failed" << std::endl;
|
||||
// }
|
||||
//
|
||||
//
|
||||
// // write
|
||||
// for (int i = 0; i < 100000; i++) {
|
||||
// SIndexMultiTerm* terms = indexMultiTermCreate();
|
||||
// std::string val = "field";
|
||||
//
|
||||
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
|
||||
//
|
||||
// val.append(std::to_string(i));
|
||||
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
|
||||
//
|
||||
// val.insert(0, std::to_string(i));
|
||||
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
|
||||
//
|
||||
// val.append("const");
|
||||
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
|
||||
//
|
||||
//
|
||||
// indexPut(index, terms, i);
|
||||
// indexMultiTermDestroy(terms);
|
||||
// }
|
||||
//
|
||||
//
|
||||
// // query
|
||||
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
|
||||
//
|
||||
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
|
||||
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
|
||||
//
|
||||
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
|
||||
// indexSearch(index, multiQuery, result);
|
||||
//
|
||||
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
|
||||
// for (int i = 0; i < taosArrayGetSize(result); i++) {
|
||||
// int *v = (int *)taosArrayGet(result, i);
|
||||
// std::cout << "value --->" << *v << std::endl;
|
||||
// }
|
||||
// // add more test case
|
||||
// indexMultiTermQueryDestroy(multiQuery);
|
||||
//
|
||||
// indexOptsDestroy(opts);
|
||||
// indexClose(index);
|
||||
// //
|
||||
//}
|
||||
|
||||
#define L 100
|
||||
#define M 100
|
||||
#define N 100
|
||||
|
|
@ -421,6 +371,7 @@ class TFileObj {
|
|||
|
||||
int colId_;
|
||||
};
|
||||
|
||||
class IndexTFileEnv : public ::testing::Test {
|
||||
protected:
|
||||
virtual void SetUp() {
|
||||
|
|
@ -428,22 +379,6 @@ class IndexTFileEnv : public ::testing::Test {
|
|||
taosMkDir(dir.c_str());
|
||||
tfInit();
|
||||
fObj = new TFileObj(dir, colName);
|
||||
|
||||
// std::string colName("voltage");
|
||||
// header.suid = 1;
|
||||
// header.version = 1;
|
||||
// memcpy(header.colName, colName.c_str(), colName.size());
|
||||
// header.colType = TSDB_DATA_TYPE_BINARY;
|
||||
|
||||
// std::string path(dir);
|
||||
// int colId = 2;
|
||||
// char buf[64] = {0};
|
||||
// sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version);
|
||||
// path.append("/").append(buf);
|
||||
|
||||
// ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
|
||||
|
||||
// twrite = tfileWriterCreate(ctx, &header);
|
||||
}
|
||||
|
||||
virtual void TearDown() {
|
||||
|
|
@ -460,16 +395,8 @@ class IndexTFileEnv : public ::testing::Test {
|
|||
int coldId = 2;
|
||||
int version = 1;
|
||||
int colType = TSDB_DATA_TYPE_BINARY;
|
||||
|
||||
// WriterCtx* ctx = NULL;
|
||||
// TFileHeader header;
|
||||
// TFileWriter* twrite = NULL;
|
||||
};
|
||||
|
||||
// static TFileWriter* genTFileWriter(const char* path, TFileHeader* header) {
|
||||
// char buf[128] = {0};
|
||||
// WriterCtx* ctx = writerCtxCreate(TFile, path, false, )
|
||||
//}
|
||||
static TFileValue* genTFileValue(const char* val) {
|
||||
TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue));
|
||||
int32_t vlen = strlen(val) + 1;
|
||||
|
|
@ -492,7 +419,7 @@ static void destroyTFileValue(void* val) {
|
|||
|
||||
TEST_F(IndexTFileEnv, test_tfile_write) {
|
||||
TFileValue* v1 = genTFileValue("c");
|
||||
TFileValue* v2 = genTFileValue("a");
|
||||
TFileValue* v2 = genTFileValue("ab");
|
||||
TFileValue* v3 = genTFileValue("b");
|
||||
TFileValue* v4 = genTFileValue("d");
|
||||
|
||||
|
|
@ -510,7 +437,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
|
|||
taosArrayDestroy(data);
|
||||
|
||||
std::string colName("voltage");
|
||||
std::string colVal("b");
|
||||
std::string colVal("ab");
|
||||
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
|
||||
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
|
||||
|
||||
|
|
@ -521,3 +448,284 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
|
|||
|
||||
// tfileWriterDestroy(twrite);
|
||||
}
|
||||
class CacheObj {
|
||||
public:
|
||||
CacheObj() {
|
||||
// TODO
|
||||
cache = indexCacheCreate();
|
||||
}
|
||||
int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
|
||||
int ret = indexCachePut(cache, term, colId, version, uid);
|
||||
if (ret != 0) {
|
||||
//
|
||||
std::cout << "failed to put into cache: " << ret << std::endl;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
|
||||
int ret = indexCacheSearch(cache, query, colId, version, result, s);
|
||||
if (ret != 0) {
|
||||
//
|
||||
std::cout << "failed to get from cache:" << ret << std::endl;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
~CacheObj() {
|
||||
// TODO
|
||||
indexCacheDestroy(cache);
|
||||
}
|
||||
|
||||
private:
|
||||
IndexCache* cache = NULL;
|
||||
};
|
||||
|
||||
class IndexCacheEnv : public ::testing::Test {
|
||||
protected:
|
||||
virtual void SetUp() {
|
||||
// TODO
|
||||
coj = new CacheObj();
|
||||
}
|
||||
virtual void TearDown() {
|
||||
delete coj;
|
||||
// formate
|
||||
}
|
||||
CacheObj* coj;
|
||||
};
|
||||
|
||||
TEST_F(IndexCacheEnv, cache_test) {
|
||||
int count = 10;
|
||||
|
||||
int16_t colId = 1;
|
||||
int32_t version = 10;
|
||||
uint64_t suid = 100;
|
||||
std::string colName("voltage");
|
||||
std::string colVal("My God");
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
colVal += ('a' + i);
|
||||
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
|
||||
coj->Put(term, colId, version, suid);
|
||||
version++;
|
||||
}
|
||||
|
||||
// coj->Get();
|
||||
}
|
||||
|
||||
typedef struct CTerm {
|
||||
char buf[16];
|
||||
char version[8];
|
||||
int val;
|
||||
int other;
|
||||
} CTerm;
|
||||
CTerm* cTermCreate(const char* str, const char* version, int val) {
|
||||
CTerm* tm = (CTerm*)calloc(1, sizeof(CTerm));
|
||||
memcpy(tm->buf, str, strlen(str));
|
||||
memcpy(tm->version, version, strlen(version));
|
||||
tm->val = val;
|
||||
tm->other = -100;
|
||||
return tm;
|
||||
}
|
||||
int termCompar(const void* a, const void* b) {
|
||||
printf("a: %s \t b: %s\n", (char*)a, (char*)b);
|
||||
int ret = strncmp((char*)a, (char*)b, 16);
|
||||
if (ret == 0) {
|
||||
//
|
||||
return strncmp((char*)a + 16, (char*)b + 16, 8);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int SerialTermTo(char* buf, CTerm* term) {
|
||||
char* p = buf;
|
||||
memcpy(buf, term->buf, sizeof(term->buf));
|
||||
buf += sizeof(term->buf);
|
||||
|
||||
// memcpy(buf, term->version, sizeof(term->version));
|
||||
// buf += sizeof(term->version);
|
||||
return buf - p;
|
||||
}
|
||||
static char* getTermKey(const void* pData) {
|
||||
CTerm* p = (CTerm*)pData;
|
||||
return (char*)p->buf;
|
||||
}
|
||||
#define MAX_TERM_KEY_LEN 128
|
||||
class SkiplistObj {
|
||||
public:
|
||||
// max_key_len:
|
||||
//
|
||||
SkiplistObj() {
|
||||
slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey);
|
||||
}
|
||||
int Put(CTerm* term, uint64_t suid) {
|
||||
char buf[MAX_TERM_KEY_LEN] = {0};
|
||||
int sz = SerialTermTo(buf, term);
|
||||
|
||||
char* pBuf = (char*)calloc(1, sz + sizeof(suid));
|
||||
|
||||
memcpy(pBuf, buf, sz);
|
||||
memcpy(pBuf + sz, &suid, sizeof(suid));
|
||||
// int32_t level, headsize;
|
||||
// tSkipListNewNodeInfo(slt, &level, &headsize);
|
||||
|
||||
// SSkipListNode* node = (SSkipListNode*)calloc(1, headsize + strlen(buf) + sizeof(suid));
|
||||
// node->level = level;
|
||||
// char* d = (char*)SL_GET_NODE_DATA(node);
|
||||
// memcpy(d, buf, strlen(buf));
|
||||
// memcpy(d + strlen(buf), &suid, sizeof(suid));
|
||||
SSkipListNode* node = tSkipListPut(slt, pBuf);
|
||||
tSkipListPrint(slt, 1);
|
||||
free(pBuf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int Get(int key, char* buf, int version) {
|
||||
// CTerm term;
|
||||
// term.key = key;
|
||||
//// term.version = version;
|
||||
// memcpy(term.buf, buf, strlen(buf));
|
||||
|
||||
// char tbuf[128] = {0};
|
||||
// SerialTermTo(tbuf, &term);
|
||||
|
||||
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
|
||||
// SSkipListNode* node = tSkipListIterGet(iter);
|
||||
// CTerm* ct = (CTerm*)SL_GET_NODE_DATA(node);
|
||||
// printf("key: %d\t, version: %d\t, buf: %s\n", ct->key, ct->version, ct->buf);
|
||||
// while (iter) {
|
||||
// assert(tSkipListIterNext(iter) == true);
|
||||
// SSkipListNode* node = tSkipListIterGet(iter);
|
||||
// // ugly formate
|
||||
// CTerm* t = (CTerm*)SL_GET_NODE_KEY(slt, node);
|
||||
// printf("key: %d\t, version: %d\t, buf: %s\n", t->key, t->version, t->buf);
|
||||
//}
|
||||
return 0;
|
||||
}
|
||||
~SkiplistObj() {
|
||||
// TODO
|
||||
// indexCacheDestroy(cache);
|
||||
}
|
||||
|
||||
private:
|
||||
SSkipList* slt;
|
||||
};
|
||||
|
||||
typedef struct KV {
|
||||
int32_t k;
|
||||
int32_t v;
|
||||
} KV;
|
||||
int kvCompare(const void* a, const void* b) {
|
||||
int32_t av = *(int32_t*)a;
|
||||
int32_t bv = *(int32_t*)b;
|
||||
return av - bv;
|
||||
}
|
||||
char* getKVkey(const void* a) {
|
||||
return (char*)(&(((KV*)a)->v));
|
||||
// KV* kv = (KV*)a;
|
||||
}
|
||||
int testKV() {
|
||||
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, kvCompare, SL_DISCARD_DUP_KEY, getKVkey);
|
||||
{
|
||||
KV t = {.k = 1, .v = 5};
|
||||
tSkipListPut(slt, (void*)&t);
|
||||
}
|
||||
{
|
||||
KV t = {.k = 2, .v = 3};
|
||||
tSkipListPut(slt, (void*)&t);
|
||||
}
|
||||
|
||||
KV value = {.k = 4, .v = 5};
|
||||
char* key = getKVkey(&value);
|
||||
// const char* key = "Hello";
|
||||
SArray* arr = tSkipListGet(slt, (SSkipListKey)&key);
|
||||
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
|
||||
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
|
||||
int32_t* ct = (int32_t*)SL_GET_NODE_KEY(slt, node);
|
||||
|
||||
printf("Get key: %d\n", *ct);
|
||||
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int testComplicate() {
|
||||
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey);
|
||||
{
|
||||
CTerm* tm = cTermCreate("val", "v1", 10);
|
||||
tSkipListPut(slt, (char*)tm);
|
||||
}
|
||||
{
|
||||
CTerm* tm = cTermCreate("val1", "v2", 2);
|
||||
tSkipListPut(slt, (char*)tm);
|
||||
}
|
||||
{
|
||||
CTerm* tm = cTermCreate("val3", "v3", -1);
|
||||
tSkipListPut(slt, (char*)tm);
|
||||
}
|
||||
{
|
||||
CTerm* tm = cTermCreate("val3", "v4", 2);
|
||||
tSkipListPut(slt, (char*)tm);
|
||||
}
|
||||
{
|
||||
CTerm* tm = cTermCreate("val3", "v5", -1);
|
||||
char* key = getTermKey(tm);
|
||||
SArray* arr = tSkipListGet(slt, (SSkipListKey)key);
|
||||
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
|
||||
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
|
||||
CTerm* ct = (CTerm*)SL_GET_NODE_KEY(slt, node);
|
||||
printf("other; %d\tbuf: %s\t, version: %s, val: %d\n", ct->other, ct->buf, ct->version, ct->val);
|
||||
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
|
||||
}
|
||||
free(tm);
|
||||
taosArrayDestroy(arr);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
int strCompare(const void* a, const void* b) {
|
||||
const char* sa = (char*)a;
|
||||
const char* sb = (char*)b;
|
||||
return strcmp(sa, sb);
|
||||
}
|
||||
void testString() {
|
||||
SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, strCompare, SL_ALLOW_DUP_KEY, getTermKey);
|
||||
{
|
||||
tSkipListPut(slt, (void*)"Hello");
|
||||
tSkipListPut(slt, (void*)"World");
|
||||
tSkipListPut(slt, (void*)"YI");
|
||||
}
|
||||
|
||||
const char* key = "YI";
|
||||
SArray* arr = tSkipListGet(slt, (SSkipListKey)key);
|
||||
for (size_t i = 0; i < taosArrayGetSize(arr); i++) {
|
||||
SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i);
|
||||
char* ct = (char*)SL_GET_NODE_KEY(slt, node);
|
||||
printf("Get key: %s\n", ct);
|
||||
// SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
|
||||
}
|
||||
}
|
||||
// class IndexSkip : public ::testing::Test {
|
||||
// protected:
|
||||
// virtual void SetUp() {
|
||||
// // TODO
|
||||
// sObj = new SkiplistObj();
|
||||
// }
|
||||
// virtual void TearDown() {
|
||||
// delete sObj;
|
||||
// // formate
|
||||
// }
|
||||
// SkiplistObj* sObj;
|
||||
//};
|
||||
|
||||
// TEST_F(IndexSkip, skip_test) {
|
||||
// std::string val("Hello");
|
||||
// std::string minVal = val;
|
||||
// for (size_t i = 0; i < 10; i++) {
|
||||
// CTerm* t = (CTerm*)calloc(1, sizeof(CTerm));
|
||||
// t->key = 1;
|
||||
// t->version = i;
|
||||
//
|
||||
// val[val.size() - 1] = 'a' + i;
|
||||
// memcpy(t->buf, val.c_str(), val.size());
|
||||
// sObj->Put(t, 10);
|
||||
// free(t);
|
||||
// }
|
||||
// sObj->Get(1, (char*)(minVal.c_str()), 1000000);
|
||||
//}
|
||||
|
|
|
|||
Loading…
Reference in New Issue