From 7ca0a88ee274d25ecf6e5dadacf82b9569fe659e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Dec 2021 23:55:15 +0800 Subject: [PATCH] add skiplist add --- source/libs/index/test/indexTests.cc | 364 +++++++++++++++++++++------ 1 file changed, 286 insertions(+), 78 deletions(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 6c6f45bd11..de82350aad 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -16,10 +16,12 @@ #include #include "index.h" #include "indexInt.h" +#include "index_cache.h" #include "index_fst.h" #include "index_fst_counting_writer.h" #include "index_fst_util.h" #include "index_tfile.h" +#include "tskiplist.h" #include "tutil.h" using namespace std; class FstWriter { @@ -110,58 +112,6 @@ class FstReadMemory { size_t _size; }; -// TEST(IndexTest, index_create_test) { -// SIndexOpts *opts = indexOptsCreate(); -// SIndex *index = indexOpen(opts, "./test"); -// if (index == NULL) { -// std::cout << "index open failed" << std::endl; -// } -// -// -// // write -// for (int i = 0; i < 100000; i++) { -// SIndexMultiTerm* terms = indexMultiTermCreate(); -// std::string val = "field"; -// -// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); -// -// val.append(std::to_string(i)); -// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); -// -// val.insert(0, std::to_string(i)); -// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); -// -// val.append("const"); -// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); -// -// -// indexPut(index, terms, i); -// indexMultiTermDestroy(terms); -// } -// -// -// // query -// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); -// -// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); -// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); -// -// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); -// indexSearch(index, multiQuery, result); -// -// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; -// for (int i = 0; i < taosArrayGetSize(result); i++) { -// int *v = (int *)taosArrayGet(result, i); -// std::cout << "value --->" << *v << std::endl; -// } -// // add more test case -// indexMultiTermQueryDestroy(multiQuery); -// -// indexOptsDestroy(opts); -// indexClose(index); -// // -//} - #define L 100 #define M 100 #define N 100 @@ -421,6 +371,7 @@ class TFileObj { int colId_; }; + class IndexTFileEnv : public ::testing::Test { protected: virtual void SetUp() { @@ -428,22 +379,6 @@ class IndexTFileEnv : public ::testing::Test { taosMkDir(dir.c_str()); tfInit(); fObj = new TFileObj(dir, colName); - - // std::string colName("voltage"); - // header.suid = 1; - // header.version = 1; - // memcpy(header.colName, colName.c_str(), colName.size()); - // header.colType = TSDB_DATA_TYPE_BINARY; - - // std::string path(dir); - // int colId = 2; - // char buf[64] = {0}; - // sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version); - // path.append("/").append(buf); - - // ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024); - - // twrite = tfileWriterCreate(ctx, &header); } virtual void TearDown() { @@ -460,16 +395,8 @@ class IndexTFileEnv : public ::testing::Test { int coldId = 2; int version = 1; int colType = TSDB_DATA_TYPE_BINARY; - - // WriterCtx* ctx = NULL; - // TFileHeader header; - // TFileWriter* twrite = NULL; }; -// static TFileWriter* genTFileWriter(const char* path, TFileHeader* header) { -// char buf[128] = {0}; -// WriterCtx* ctx = writerCtxCreate(TFile, path, false, ) -//} static TFileValue* genTFileValue(const char* val) { TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); int32_t vlen = strlen(val) + 1; @@ -492,7 +419,7 @@ static void destroyTFileValue(void* val) { TEST_F(IndexTFileEnv, test_tfile_write) { TFileValue* v1 = genTFileValue("c"); - TFileValue* v2 = genTFileValue("a"); + TFileValue* v2 = genTFileValue("ab"); TFileValue* v3 = genTFileValue("b"); TFileValue* v4 = genTFileValue("d"); @@ -510,7 +437,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { taosArrayDestroy(data); std::string colName("voltage"); - std::string colVal("b"); + std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; @@ -521,3 +448,284 @@ TEST_F(IndexTFileEnv, test_tfile_write) { // tfileWriterDestroy(twrite); } +class CacheObj { + public: + CacheObj() { + // TODO + cache = indexCacheCreate(); + } + int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { + int ret = indexCachePut(cache, term, colId, version, uid); + if (ret != 0) { + // + std::cout << "failed to put into cache: " << ret << std::endl; + } + return ret; + } + int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { + int ret = indexCacheSearch(cache, query, colId, version, result, s); + if (ret != 0) { + // + std::cout << "failed to get from cache:" << ret << std::endl; + } + return ret; + } + ~CacheObj() { + // TODO + indexCacheDestroy(cache); + } + + private: + IndexCache* cache = NULL; +}; + +class IndexCacheEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // TODO + coj = new CacheObj(); + } + virtual void TearDown() { + delete coj; + // formate + } + CacheObj* coj; +}; + +TEST_F(IndexCacheEnv, cache_test) { + int count = 10; + + int16_t colId = 1; + int32_t version = 10; + uint64_t suid = 100; + std::string colName("voltage"); + std::string colVal("My God"); + for (size_t i = 0; i < count; i++) { + colVal += ('a' + i); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + coj->Put(term, colId, version, suid); + version++; + } + + // coj->Get(); +} + +typedef struct CTerm { + char buf[16]; + char version[8]; + int val; + int other; +} CTerm; +CTerm* cTermCreate(const char* str, const char* version, int val) { + CTerm* tm = (CTerm*)calloc(1, sizeof(CTerm)); + memcpy(tm->buf, str, strlen(str)); + memcpy(tm->version, version, strlen(version)); + tm->val = val; + tm->other = -100; + return tm; +} +int termCompar(const void* a, const void* b) { + printf("a: %s \t b: %s\n", (char*)a, (char*)b); + int ret = strncmp((char*)a, (char*)b, 16); + if (ret == 0) { + // + return strncmp((char*)a + 16, (char*)b + 16, 8); + } + return ret; +} + +int SerialTermTo(char* buf, CTerm* term) { + char* p = buf; + memcpy(buf, term->buf, sizeof(term->buf)); + buf += sizeof(term->buf); + + // memcpy(buf, term->version, sizeof(term->version)); + // buf += sizeof(term->version); + return buf - p; +} +static char* getTermKey(const void* pData) { + CTerm* p = (CTerm*)pData; + return (char*)p->buf; +} +#define MAX_TERM_KEY_LEN 128 +class SkiplistObj { + public: + // max_key_len: + // + SkiplistObj() { + slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey); + } + int Put(CTerm* term, uint64_t suid) { + char buf[MAX_TERM_KEY_LEN] = {0}; + int sz = SerialTermTo(buf, term); + + char* pBuf = (char*)calloc(1, sz + sizeof(suid)); + + memcpy(pBuf, buf, sz); + memcpy(pBuf + sz, &suid, sizeof(suid)); + // int32_t level, headsize; + // tSkipListNewNodeInfo(slt, &level, &headsize); + + // SSkipListNode* node = (SSkipListNode*)calloc(1, headsize + strlen(buf) + sizeof(suid)); + // node->level = level; + // char* d = (char*)SL_GET_NODE_DATA(node); + // memcpy(d, buf, strlen(buf)); + // memcpy(d + strlen(buf), &suid, sizeof(suid)); + SSkipListNode* node = tSkipListPut(slt, pBuf); + tSkipListPrint(slt, 1); + free(pBuf); + return 0; + } + + int Get(int key, char* buf, int version) { + // CTerm term; + // term.key = key; + //// term.version = version; + // memcpy(term.buf, buf, strlen(buf)); + + // char tbuf[128] = {0}; + // SerialTermTo(tbuf, &term); + + // SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + // SSkipListNode* node = tSkipListIterGet(iter); + // CTerm* ct = (CTerm*)SL_GET_NODE_DATA(node); + // printf("key: %d\t, version: %d\t, buf: %s\n", ct->key, ct->version, ct->buf); + // while (iter) { + // assert(tSkipListIterNext(iter) == true); + // SSkipListNode* node = tSkipListIterGet(iter); + // // ugly formate + // CTerm* t = (CTerm*)SL_GET_NODE_KEY(slt, node); + // printf("key: %d\t, version: %d\t, buf: %s\n", t->key, t->version, t->buf); + //} + return 0; + } + ~SkiplistObj() { + // TODO + // indexCacheDestroy(cache); + } + + private: + SSkipList* slt; +}; + +typedef struct KV { + int32_t k; + int32_t v; +} KV; +int kvCompare(const void* a, const void* b) { + int32_t av = *(int32_t*)a; + int32_t bv = *(int32_t*)b; + return av - bv; +} +char* getKVkey(const void* a) { + return (char*)(&(((KV*)a)->v)); + // KV* kv = (KV*)a; +} +int testKV() { + SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, kvCompare, SL_DISCARD_DUP_KEY, getKVkey); + { + KV t = {.k = 1, .v = 5}; + tSkipListPut(slt, (void*)&t); + } + { + KV t = {.k = 2, .v = 3}; + tSkipListPut(slt, (void*)&t); + } + + KV value = {.k = 4, .v = 5}; + char* key = getKVkey(&value); + // const char* key = "Hello"; + SArray* arr = tSkipListGet(slt, (SSkipListKey)&key); + for (size_t i = 0; i < taosArrayGetSize(arr); i++) { + SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i); + int32_t* ct = (int32_t*)SL_GET_NODE_KEY(slt, node); + + printf("Get key: %d\n", *ct); + // SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + } + return 1; +} + +int testComplicate() { + SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, termCompar, SL_ALLOW_DUP_KEY, getTermKey); + { + CTerm* tm = cTermCreate("val", "v1", 10); + tSkipListPut(slt, (char*)tm); + } + { + CTerm* tm = cTermCreate("val1", "v2", 2); + tSkipListPut(slt, (char*)tm); + } + { + CTerm* tm = cTermCreate("val3", "v3", -1); + tSkipListPut(slt, (char*)tm); + } + { + CTerm* tm = cTermCreate("val3", "v4", 2); + tSkipListPut(slt, (char*)tm); + } + { + CTerm* tm = cTermCreate("val3", "v5", -1); + char* key = getTermKey(tm); + SArray* arr = tSkipListGet(slt, (SSkipListKey)key); + for (size_t i = 0; i < taosArrayGetSize(arr); i++) { + SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i); + CTerm* ct = (CTerm*)SL_GET_NODE_KEY(slt, node); + printf("other; %d\tbuf: %s\t, version: %s, val: %d\n", ct->other, ct->buf, ct->version, ct->val); + // SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + } + free(tm); + taosArrayDestroy(arr); + } + return 1; +} +int strCompare(const void* a, const void* b) { + const char* sa = (char*)a; + const char* sb = (char*)b; + return strcmp(sa, sb); +} +void testString() { + SSkipList* slt = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_TERM_KEY_LEN, strCompare, SL_ALLOW_DUP_KEY, getTermKey); + { + tSkipListPut(slt, (void*)"Hello"); + tSkipListPut(slt, (void*)"World"); + tSkipListPut(slt, (void*)"YI"); + } + + const char* key = "YI"; + SArray* arr = tSkipListGet(slt, (SSkipListKey)key); + for (size_t i = 0; i < taosArrayGetSize(arr); i++) { + SSkipListNode* node = (SSkipListNode*)taosArrayGetP(arr, i); + char* ct = (char*)SL_GET_NODE_KEY(slt, node); + printf("Get key: %s\n", ct); + // SSkipListIterator* iter = tSkipListCreateIterFromVal(slt, tbuf, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + } +} +// class IndexSkip : public ::testing::Test { +// protected: +// virtual void SetUp() { +// // TODO +// sObj = new SkiplistObj(); +// } +// virtual void TearDown() { +// delete sObj; +// // formate +// } +// SkiplistObj* sObj; +//}; + +// TEST_F(IndexSkip, skip_test) { +// std::string val("Hello"); +// std::string minVal = val; +// for (size_t i = 0; i < 10; i++) { +// CTerm* t = (CTerm*)calloc(1, sizeof(CTerm)); +// t->key = 1; +// t->version = i; +// +// val[val.size() - 1] = 'a' + i; +// memcpy(t->buf, val.c_str(), val.size()); +// sObj->Put(t, 10); +// free(t); +// } +// sObj->Get(1, (char*)(minVal.c_str()), 1000000); +//}