xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9335 from taosdata/feature/index_cache 

fix tfile bug and add unit test
This commit is contained in:
Shengliang Guan 2021-12-23 19:37:26 +08:00 committed by GitHub
parent fe12d1d6ee aedd147359
commit 74d5d12c05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 333 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
source/libs/index/inc/index_tfile.h
View File

@ -42,7 +42,12 @@ typedef struct TFileHeader {
#define TFILE_HEADER_SIZE (sizeof(TFileHeader))
#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t))
//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
typedef struct TFileCacheKey {
uint64_t suid;

30
source/libs/index/src/index.c
View File

@ -41,7 +41,7 @@ static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit();
static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
static int indexMergeCacheIntoTindex(SIndex* sIdx);
static int indexFlushCacheToTindex(SIndex* sIdx);
static void indexInterResultsDestroy(SArray* results);
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult);
@ -49,9 +49,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) {
return -1;
}
if (sIdx == NULL) { return -1; }
#ifdef USE_LUCENE
index_t* index = index_open(path);
@ -131,9 +129,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
int32_t colId = fi->colId;
int32_t version = index->cVersion;
int ret = indexCachePut(index->cache, p, colId, version, uid);
if (ret != 0) {
return ret;
}
if (ret != 0) { return ret; }
}
#endif
@ -221,9 +217,7 @@ void indexOptsDestroy(SIndexOpts* opts){
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) {
return NULL;
}
if (p == NULL) { return NULL; }
p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p;
@ -250,9 +244,7 @@ SIndexTerm* indexTermCreate(int64_t suid,
const char* colVal,
int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) {
return NULL;
}
if (t == NULL) { return NULL; }
t->suid = suid;
t->operType = oper;
@ -332,9 +324,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
return 0;
}
static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) {
return;
}
if (results == NULL) { return; }
size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) {
@ -363,10 +353,10 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
}
return 0;
}
static int indexMergeCacheIntoTindex(SIndex* sIdx) {
if (sIdx == NULL) {
return -1;
}
static int indexFlushCacheToTindex(SIndex* sIdx) {
if (sIdx == NULL) { return -1; }
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
return 0;
}

3
source/libs/index/src/index_cache.c
View File

@ -151,8 +151,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t
EIndexQueryType qtype = query->qType;
int32_t keyLen = CACHE_KEY_LEN(term);
char* buf = calloc(1, keyLen);
char* buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) {
//
} else if (qtype == QUERY_PREFIX) {

5
source/libs/index/src/index_fst_counting_writer.c
View File

@ -69,9 +69,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) {
ctx->file.fd = tfOpenCreateWriteAppend(tmpFile);
ctx->file.fd = tfOpenCreateWriteAppend(path);
} else {
ctx->file.fd = tfOpenReadWrite(tmpFile);
ctx->file.fd = tfOpenReadWrite(path);
}
if (ctx->file.fd < 0) {
indexError("open file error %d", errno);
@ -93,6 +93,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
END:
if (ctx->type == TMemory) { free(ctx->mem.buf); }
free(ctx);
return NULL;
}
void writerCtxDestroy(WriterCtx* ctx) {
if (ctx->type == TMemory) {

37
source/libs/index/src/index_tfile.c
View File

@ -25,12 +25,7 @@
#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
static int tfileStrCompare(const void* a, const void* b);
static int tfileValueCompare(const void* a, const void* b, const void* param);
static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds);
@ -49,6 +44,7 @@ static int tfileRmExpireFile(SArray* result);
static void tfileDestroyFileName(void* elem);
static int tfileCompare(const void* a, const void* b);
static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version);
static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version);
static void tfileSerialCacheKey(TFileCacheKey* key, char* buf);
TFileCache* tfileCacheCreate(const char* path) {
@ -209,16 +205,28 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
tw->ctx = ctx;
tw->header = *header;
tfileWriteHeader(tw);
tw->fb = fstBuilderCreate(ctx, 0);
if (tw->fb == NULL) {
tfileWriterDestroy(tw);
return NULL;
}
return tw;
}
int tfileWriterPut(TFileWriter* tw, void* data) {
// sort by coltype and write to tindex
__compar_fn_t fn = getComparFunc(tw->header.colType, 0);
__compar_fn_t fn;
int8_t colType = tw->header.colType;
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
fn = tfileStrCompare;
} else {
fn = getComparFunc(colType, 0);
}
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
int32_t bufLimit = 4096, offset = 0;
char* buf = calloc(1, sizeof(bufLimit));
char* buf = calloc(1, sizeof(char) * bufLimit);
char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data);
int32_t fstOffset = tw->offset;
@ -267,6 +275,9 @@ int tfileWriterPut(TFileWriter* tw, void* data) {
//
}
}
fstBuilderFinish(tw->fb);
fstBuilderDestroy(tw->fb);
tw->fb = NULL;
return 0;
}
void tfileWriterDestroy(TFileWriter* tw) {
@ -305,6 +316,12 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
return 0;
}
static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; }
return ret < 0 ? -1 : 1;
}
static int tfileValueCompare(const void* a, const void* b, const void* param) {
__compar_fn_t fn = *(__compar_fn_t*)param;
@ -445,6 +462,10 @@ static int tfileCompare(const void* a, const void* b) {
return strncmp(aName, bName, aLen > bLen ? aLen : bLen);
}
// tfile name suid-colId-version.tindex
static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) {
sprintf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version);
return;
}
static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) {
if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) {
// read suid & colid & version success

509
source/libs/index/test/indexTests.cc
View File

@ -2,8 +2,7 @@
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@ -13,141 +12,141 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <string>
#include <iostream>
#include <string>
#include "index.h"
#include "tutil.h"
#include "indexInt.h"
#include "index_fst.h"
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "tutil.h"
class FstWriter {
public:
FstWriter() {
_wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0);
_b = fstBuilderCreate(NULL, 0);
}
bool Put(const std::string &key, uint64_t val) {
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
public:
FstWriter() {
_wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024);
_b = fstBuilderCreate(NULL, 0);
}
bool Put(const std::string& key, uint64_t val) {
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
writerCtxDestroy(_wc);
}
private:
FstBuilder *_b;
WriterCtx *_wc;
writerCtxDestroy(_wc);
}
private:
FstBuilder* _b;
WriterCtx* _wc;
};
class FstReadMemory {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, "/tmp/tindex", true, 0);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void *)&_s, 0, sizeof(_s));
}
bool init() {
char *buf = (char *)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size);
if (nRead <= 0) { return false; }
_size = nRead;
_s = fstSliceCreate((uint8_t *)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string &key, uint64_t *val) {
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) {
FstStreamBuilder *sb = fstSearch(_fst, ctx);
StreamWithState *st = streamBuilderIntoStream(sb);
StreamWithStateResult *rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
result.push_back((uint64_t)(rt->out.out));
}
return true;
}
bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void*)&_s, 0, sizeof(_s));
}
bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; }
_size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string& key, uint64_t* val) {
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
FstStreamBuilder* sb = fstSearch(_fst, ctx);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
result.push_back((uint64_t)(rt->out.out));
}
return true;
}
bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
fstCountingWriterDestroy(_w);
fstDestroy(_fst);
fstSliceDestroy(&_s);
writerCtxDestroy(_wc);
}
private:
FstCountingWriter *_w;
Fst *_fst;
FstSlice _s;
WriterCtx *_wc;
size_t _size;
};
}
//TEST(IndexTest, index_create_test) {
private:
FstCountingWriter* _w;
Fst* _fst;
FstSlice _s;
WriterCtx* _wc;
size_t _size;
};
// TEST(IndexTest, index_create_test) {
// SIndexOpts *opts = indexOptsCreate();
// SIndex *index = indexOpen(opts, "./test");
// if (index == NULL) {
// std::cout << "index open failed" << std::endl;
// std::cout << "index open failed" << std::endl;
// }
//
//
// // write
//
// // write
// for (int i = 0; i < 100000; i++) {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// std::string val = "field";
// std::string val = "field";
//
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
//
// val.append(std::to_string(i));
// val.append(std::to_string(i));
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
//
// val.insert(0, std::to_string(i));
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
//
// val.append("const");
// val.append("const");
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
//
//
//
// indexPut(index, terms, i);
// indexMultiTermDestroy(terms);
// }
//
// }
//
//
// // query
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
//
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// indexSearch(index, multiQuery, result);
//
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
@ -155,25 +154,24 @@ class FstReadMemory {
// int *v = (int *)taosArrayGet(result, i);
// std::cout << "value --->" << *v << std::endl;
// }
// // add more test case
// // add more test case
// indexMultiTermQueryDestroy(multiQuery);
//
// indexOptsDestroy(opts);
// indexClose(index);
// indexOptsDestroy(opts);
// indexClose(index);
// //
//}
#define L 100
#define M 100
#define N 100
int Performance_fstWriteRecords(FstWriter *b) {
std::string str("aa");
int Performance_fstWriteRecords(FstWriter* b) {
std::string str("aa");
for (int i = 0; i < L; i++) {
str[0] = 'a' + i;
str.resize(2);
for(int j = 0; j < M; j++) {
str.resize(2);
for (int j = 0; j < M; j++) {
str[1] = 'a' + j;
str.resize(2);
for (int k = 0; k < N; k++) {
@ -181,87 +179,86 @@ int Performance_fstWriteRecords(FstWriter *b) {
b->Put(str, k);
printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
}
}
}
}
return L * M * N;
}
void Performance_fstReadRecords(FstReadMemory *m) {
void Performance_fstReadRecords(FstReadMemory* m) {
std::string str("aa");
for (int i = 0; i < M; i++) {
str[0] = 'a' + i;
str.resize(2);
for(int j = 0; j < N; j++) {
str.resize(2);
for (int j = 0; j < N; j++) {
str[1] = 'a' + j;
str.resize(2);
for (int k = 0; k < L; k++) {
str.push_back('a');
uint64_t val, cost;
uint64_t val, cost;
if (m->GetWithTimeCostUs(str, &val, &cost)) {
printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
} else {
printf("failed to get key: %s\n", str.c_str());
}
}
}
}
}
}
void checkFstPerf() {
FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs();
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int num = Performance_fstWriteRecords(fw);
int num = Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
printf("write %d record cost %" PRId64"us\n", num, e - s);
printf("write %d record cost %" PRId64 "us\n", num, e - s);
delete fw;
FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(m);
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init()) { printf("success to init fst read"); }
Performance_fstReadRecords(m);
delete m;
}
}
void checkFstPrefixSearch() {
FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
std::string key("ab");
for (int i = 0; i < count; i++) {
key[1] = key[1] + i;
fw->Put(key, i);
key[1] = key[1] + i;
fw->Put(key, i);
}
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
delete fw;
FstReadMemory *m = new FstReadMemory(1024 * 64);
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
// prefix search
// prefix search
std::vector<uint64_t> result;
AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == count);
AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == count);
for (int i = 0; i < result.size(); i++) {
assert(result[i] == i); // check result
assert(result[i] == i); // check result
}
free(ctx);
delete m;
}
}
void validateFst() {
int val = 100;
int count = 100;
FstWriter *fw = new FstWriter;
// write
int val = 100;
int count = 100;
FstWriter* fw = new FstWriter;
// write
{
std::string key("ab");
for (int i = 0; i < count; i++) {
@ -272,99 +269,161 @@ void validateFst() {
delete fw;
// read
FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
std::string key("ab");
uint64_t out;
if (m->Get(key, &out)) {
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
for (int i = 0; i < count; i++) {
key.push_back('a' + i);
if (m->Get(key, &out) ) {
assert(val - i == out);
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
std::string key("ab");
uint64_t out;
if (m->Get(key, &out)) {
printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
}
}
for (int i = 0; i < count; i++) {
key.push_back('a' + i);
if (m->Get(key, &out)) {
assert(val - i == out);
printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
}
}
delete m;
}
class IndexEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(path);
opts = indexOptsCreate();
int ret = indexOpen(opts, path, &index);
assert(ret == 0);
}
virtual void TearDown() {
indexClose(index);
indexOptsDestroy(opts);
}
const char *path = "/tmp/tindex";
SIndexOpts *opts;
SIndex *index;
};
TEST_F(IndexEnv, testPut) {
// single index column
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
SIndexMultiTerm *terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
// multi index column
{
SIndexMultiTerm *terms = indexMultiTermCreate();
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
{
std::string colName("tag2"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
for (int i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
//
}
TEST_F(IndexEnv, testDel) {
}
class IndexEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(path);
opts = indexOptsCreate();
int ret = indexOpen(opts, path, &index);
assert(ret == 0);
}
virtual void TearDown() {
indexClose(index);
indexOptsDestroy(opts);
}
const char* path = "/tmp/tindex";
SIndexOpts* opts;
SIndex* index;
};
// TEST_F(IndexEnv, testPut) {
// // single index column
// {
// std::string colName("tag1"), colVal("Hello world");
// SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term);
//
// for (size_t i = 0; i < 100; i++) {
// int tableId = i;
// int ret = indexPut(index, terms, tableId);
// assert(ret == 0);
// }
// indexMultiTermDestroy(terms);
// }
// // multi index column
// {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// {
// std::string colName("tag1"), colVal("Hello world");
// SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
// indexMultiTermAdd(terms, term);
// }
// {
// std::string colName("tag2"), colVal("Hello world");
// SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
// indexMultiTermAdd(terms, term);
// }
//
// for (int i = 0; i < 100; i++) {
// int tableId = i;
// int ret = indexPut(index, terms, tableId);
// assert(ret == 0);
// }
// indexMultiTermDestroy(terms);
// }
// //
//}
class IndexTFileEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(dir);
taosMkDir(dir);
tfInit();
std::string colName("voltage");
header.suid = 1;
header.version = 1;
memcpy(header.colName, colName.c_str(), colName.size());
header.colType = TSDB_DATA_TYPE_BINARY;
std::string path(dir);
int colId = 2;
char buf[64] = {0};
sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version);
path.append("/").append(buf);
ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
twrite = tfileWriterCreate(ctx, &header);
}
virtual void TearDown() {
// indexClose(index);
// indexeptsDestroy(opts);
tfCleanup();
tfileWriterDestroy(twrite);
}
const char* dir = "/tmp/tindex";
WriterCtx* ctx = NULL;
TFileHeader header;
TFileWriter* twrite = NULL;
};
static TFileValue* genTFileValue(const char* val) {
TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue));
int32_t vlen = strlen(val) + 1;
tv->colVal = (char*)calloc(1, vlen);
memcpy(tv->colVal, val, vlen);
tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t));
for (size_t i = 0; i < 10; i++) {
uint64_t v = i;
taosArrayPush(tv->tableId, &v);
}
return tv;
}
static void destroyTFileValue(void* val) {
TFileValue* tv = (TFileValue*)val;
free(tv->colVal);
taosArrayDestroy(tv->tableId);
free(tv);
}
TEST_F(IndexTFileEnv, test_tfile_write) {
TFileValue* v1 = genTFileValue("c");
TFileValue* v2 = genTFileValue("a");
SArray* data = (SArray*)taosArrayInit(4, sizeof(void*));
taosArrayPush(data, &v1);
taosArrayPush(data, &v2);
tfileWriterPut(twrite, data);
// tfileWriterDestroy(twrite);
for (size_t i = 0; i < taosArrayGetSize(data); i++) {
destroyTFileValue(taosArrayGetP(data, i));
}
taosArrayDestroy(data);
}