xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix index search crash without tfile generated

This commit is contained in:
yihaoDeng 2021-12-28 23:47:44 +08:00
parent dbadbef312
commit 4e94a21082
2 changed files with 206 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
source/libs/index/src/index_tfile.c
View File

@ -54,7 +54,9 @@ static void tfileSerialCacheKey(TFileCacheKey* key, char* buf);
TFileCache* tfileCacheCreate(const char* path) { TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = calloc(1, sizeof(TFileCache)); TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; } if (tcache == NULL) {
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
@ -83,7 +85,10 @@ TFileCache* tfileCacheCreate(const char* path) {
tfileReaderRef(reader); tfileReaderRef(reader);
// loader fst and validate it // loader fst and validate it
TFileHeader* header = &reader->header; TFileHeader* header = &reader->header;
TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; TFileCacheKey key = {.suid = header->suid,
.colName = header->colName,
.nColName = strlen(header->colName),
.colType = header->colType};
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(&key, buf); tfileSerialCacheKey(&key, buf);
@ -97,13 +102,16 @@ End:
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache* tcache) { void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { return; } if (tcache == NULL) {
return;
}
// free table cache // free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) { while (reader) {
TFileReader* p = *reader; TFileReader* p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
p->header.colType);
tfileReaderUnRef(p); tfileReaderUnRef(p);
reader = taosHashIterate(tcache->tableCache, reader); reader = taosHashIterate(tcache->tableCache, reader);
@ -116,10 +124,13 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
tfileReaderRef(reader); if (reader == NULL) {
return NULL;
}
tfileReaderRef(*reader);
return reader; return *reader;
} }
void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) {
char buf[128] = {0}; char buf[128] = {0};
@ -138,14 +149,17 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader)
} }
TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader* reader = calloc(1, sizeof(TFileReader)); TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { return NULL; } if (reader == NULL) {
return NULL;
}
// T_REF_INC(reader); // T_REF_INC(reader);
reader->ctx = ctx; reader->ctx = ctx;
if (0 != tfileReaderLoadHeader(reader)) { if (0 != tfileReaderLoadHeader(reader)) {
tfileReaderDestroy(reader); tfileReaderDestroy(reader);
indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid,
reader->header.colName);
return NULL; return NULL;
} }
@ -158,7 +172,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
return reader; return reader;
} }
void tfileReaderDestroy(TFileReader* reader) { void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
// T_REF_INC(reader); // T_REF_INC(reader);
fstDestroy(reader->fst); fstDestroy(reader->fst);
writerCtxDestroy(reader->ctx); writerCtxDestroy(reader->ctx);
@ -175,10 +191,12 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
uint64_t offset; uint64_t offset;
FstSlice key = fstSliceCreate(term->colVal, term->nColVal); FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
if (fstGet(reader->fst, &key, &offset)) { if (fstGet(reader->fst, &key, &offset)) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName,
term->colVal);
ret = tfileReaderLoadTableIds(reader, offset, result); ret = tfileReaderLoadTableIds(reader, offset, result);
} else { } else {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal); indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
term->colVal);
} }
fstSliceDestroy(&key); fstSliceDestroy(&key);
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
@ -304,12 +322,16 @@ int tfileWriterPut(TFileWriter* tw, void* data) {
return 0; return 0;
} }
void tfileWriteClose(TFileWriter* tw) { void tfileWriteClose(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx); writerCtxDestroy(tw->ctx);
free(tw); free(tw);
} }
void tfileWriterDestroy(TFileWriter* tw) { void tfileWriterDestroy(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx); writerCtxDestroy(tw->ctx);
free(tw); free(tw);
@ -317,29 +339,35 @@ void tfileWriterDestroy(TFileWriter* tw) {
IndexTFile* indexTFileCreate(const char* path) { IndexTFile* indexTFileCreate(const char* path) {
IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
if (tfile == NULL) { return NULL; } if (tfile == NULL) {
return NULL;
}
tfile->cache = tfileCacheCreate(path); tfile->cache = tfileCacheCreate(path);
return tfile; return tfile;
} }
void IndexTFileDestroy(IndexTFile* tfile) { void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); }
free(tfile);
}
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
int ret = -1; int ret = -1;
if (tfile == NULL) { return ret; } if (tfile == NULL) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile; IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileCacheKey key = {
TFileReader* reader = tfileCacheGet(pTfile->cache, &key); .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
if (reader == NULL) {
return 0;
}
return tfileReaderSearch(reader, query, result); return tfileReaderSearch(reader, query, result);
} }
int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
// TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName =
// 1}; // term->nColName, .version = 1};
return 0; return 0;
} }
@ -353,7 +381,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
TFileFstIter* tIter = iiter->iter; TFileFstIter* tIter = iiter->iter;
StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
if (rt == NULL) { return false; } if (rt == NULL) {
return false;
}
int32_t sz = 0; int32_t sz = 0;
char* ch = (char*)fstSliceData(&rt->data, &sz); char* ch = (char*)fstSliceData(&rt->data, &sz);
@ -364,20 +394,22 @@ static bool tfileIteratorNext(Iterate* iiter) {
swsResultDestroy(rt); swsResultDestroy(rt);
// set up iterate value // set up iterate value
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) {
return false;
}
iv->colVal = colVal; iv->colVal = colVal;
// std::string key(ch, sz); // std::string key(ch, sz);
} }
static IterateValue* tifileIterateGetValue(Iterate* iter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
return &iter->val;
}
static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
TFileFstIter* tIter = calloc(1, sizeof(Iterate)); TFileFstIter* tIter = calloc(1, sizeof(Iterate));
if (tIter == NULL) { return NULL; } if (tIter == NULL) {
return NULL;
}
tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->fb = fstSearch(reader->fst, tIter->ctx);
tIter->st = streamBuilderIntoStream(tIter->fb); tIter->st = streamBuilderIntoStream(tIter->fb);
@ -389,14 +421,18 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
Iterate* iter = calloc(1, sizeof(Iterate)); Iterate* iter = calloc(1, sizeof(Iterate));
iter->iter = tfileFstIteratorCreate(reader); iter->iter = tfileFstIteratorCreate(reader);
if (iter->iter == NULL) { return NULL; } if (iter->iter == NULL) {
return NULL;
}
iter->next = tfileIteratorNext; iter->next = tfileIteratorNext;
iter->getValue = tifileIterateGetValue; iter->getValue = tifileIterateGetValue;
return iter; return iter;
} }
void tfileIteratorDestroy(Iterate* iter) { void tfileIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; } if (iter == NULL) {
return;
}
IterateValue* iv = &iter->val; IterateValue* iv = &iter->val;
iterateValueDestroy(iv, true); iterateValueDestroy(iv, true);
@ -409,14 +445,18 @@ void tfileIteratorDestroy(Iterate* iter) {
} }
TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) {
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
return tfileCacheGet(tf->cache, &key); return tfileCacheGet(tf->cache, &key);
} }
static int tfileStrCompare(const void* a, const void* b) { static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b); int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; } if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1; return ret < 0 ? -1 : 1;
} }
@ -431,13 +471,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
TFileValue* tfileValueCreate(char* val) { TFileValue* tfileValueCreate(char* val) {
TFileValue* tf = calloc(1, sizeof(TFileValue)); TFileValue* tf = calloc(1, sizeof(TFileValue));
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
tf->tableId = taosArrayInit(32, sizeof(uint64_t)); tf->tableId = taosArrayInit(32, sizeof(uint64_t));
return tf; return tf;
} }
int tfileValuePush(TFileValue* tf, uint64_t val) { int tfileValuePush(TFileValue* tf, uint64_t val) {
if (tf == NULL) { return -1; } if (tf == NULL) {
return -1;
}
taosArrayPush(tf->tableId, &val); taosArrayPush(tf->tableId, &val);
return 0; return 0;
} }
@ -457,7 +501,9 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) {
static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset); int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset; tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) {
return -1;
}
tw->offset += sizeof(fstOffset); tw->offset += sizeof(fstOffset);
return 0; return 0;
} }
@ -468,7 +514,9 @@ static int tfileWriteHeader(TFileWriter* writer) {
memcpy(buf, (char*)header, sizeof(buf)); memcpy(buf, (char*)header, sizeof(buf));
int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
if (sizeof(buf) != nwrite) { return -1; } if (sizeof(buf) != nwrite) {
return -1;
}
writer->offset = nwrite; writer->offset = nwrite;
return 0; return 0;
} }
@ -502,7 +550,9 @@ static int tfileReaderLoadFst(TFileReader* reader) {
static int FST_MAX_SIZE = 16 * 1024; static int FST_MAX_SIZE = 16 * 1024;
char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
WriterCtx* ctx = reader->ctx; WriterCtx* ctx = reader->ctx;
int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
@ -525,7 +575,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
int32_t total = sizeof(uint64_t) * nid; int32_t total = sizeof(uint64_t) * nid;
char* buf = calloc(1, total); char* buf = calloc(1, total);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
nread = ctx->read(ctx, buf, total); nread = ctx->read(ctx, buf, total);
assert(total == nread); assert(total == nread);
@ -543,12 +595,16 @@ void tfileReaderRef(TFileReader* reader) {
void tfileReaderUnRef(TFileReader* reader) { void tfileReaderUnRef(TFileReader* reader) {
int ref = T_REF_DEC(reader); int ref = T_REF_DEC(reader);
if (ref == 0) { tfileReaderDestroy(reader); } if (ref == 0) {
tfileReaderDestroy(reader);
}
} }
static int tfileGetFileList(const char* path, SArray* result) { static int tfileGetFileList(const char* path, SArray* result) {
DIR* dir = opendir(path); DIR* dir = opendir(path);
if (NULL == dir) { return -1; } if (NULL == dir) {
return -1;
}
struct dirent* entry; struct dirent* entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
@ -576,7 +632,9 @@ static int tfileCompare(const void* a, const void* b) {
size_t bLen = strlen(bName); size_t bLen = strlen(bName);
int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen);
if (ret == 0) { return ret; } if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1; return ret < 0 ? -1 : 1;
} }
// tfile name suid-colId-version.tindex // tfile name suid-colId-version.tindex

146
source/libs/index/test/indexTests.cc
View File

@ -2,7 +2,8 @@
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com> * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
* *
* This program is free software: you can use, redistribute, and/or modify * This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
* *
* This program is distributed in the hope that it will be useful, but WITHOUT * This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@ -75,7 +76,9 @@ class FstReadMemory {
bool init() { bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size); char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; } if (nRead <= 0) {
return false;
}
_size = nRead; _size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size); _s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s); _fst = fstCreate(&_s);
@ -179,7 +182,9 @@ void checkFstPerf() {
delete fw; delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64); FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init()) { printf("success to init fst read"); } if (m->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(m); Performance_fstReadRecords(m);
delete m; delete m;
} }
@ -283,7 +288,8 @@ class IndexEnv : public ::testing::Test {
// / { // / {
// / std::string colName("tag1"), colVal("Hello world"); // / std::string colName("tag1"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), / colVal.size()); // indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), /
// colVal.size());
// SIndexMultiTerm* terms = indexMultiTermCreate(); // SIndexMultiTerm* terms = indexMultiTermCreate();
// indexMultiTermAdd(terms, term); // indexMultiTermAdd(terms, term);
// / / for (size_t i = 0; i < 100; i++) { // / / for (size_t i = 0; i < 100; i++) {
@ -301,14 +307,16 @@ class IndexEnv : public ::testing::Test {
// / { // / {
// / std::string colName("tag1"), colVal("Hello world"); // / std::string colName("tag1"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); // / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size());
// / indexMultiTermAdd(terms, term); // / indexMultiTermAdd(terms, term);
// / // /
// } // }
// / { // / {
// / std::string colName("tag2"), colVal("Hello world"); // / std::string colName("tag2"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); // / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size());
// / indexMultiTermAdd(terms, term); // / indexMultiTermAdd(terms, term);
// / // /
// } // }
@ -327,7 +335,8 @@ class IndexEnv : public ::testing::Test {
class TFileObj { class TFileObj {
public: public:
TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage")
: path_(path), colName_(colName) {
colId_ = 10; colId_ = 10;
// Do Nothing // Do Nothing
// //
@ -337,7 +346,9 @@ class TFileObj {
tfileReaderDestroy(reader_); tfileReaderDestroy(reader_);
reader_ = NULL; reader_ = NULL;
} }
if (writer_ == NULL) { InitWriter(); } if (writer_ == NULL) {
InitWriter();
}
return tfileWriterPut(writer_, tv); return tfileWriterPut(writer_, tv);
} }
bool InitWriter() { bool InitWriter() {
@ -377,8 +388,12 @@ class TFileObj {
return tfileReaderSearch(reader_, query, result); return tfileReaderSearch(reader_, query, result);
} }
~TFileObj() { ~TFileObj() {
if (writer_) { tfileWriterDestroy(writer_); } if (writer_) {
if (reader_) { tfileReaderDestroy(reader_); } tfileWriterDestroy(writer_);
}
if (reader_) {
tfileReaderDestroy(reader_);
}
} }
private: private:
@ -455,9 +470,10 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
} }
taosArrayDestroy(data); taosArrayDestroy(data);
std::string colName("voltage"); std::string colName("voltage");
std::string colVal("ab"); std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -525,54 +541,62 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colName("voltage"); std::string colName("voltage");
{ {
std::string colVal("v1"); std::string colVal("v1");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v2"); std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++); coj->Put(term, othColId, version++, suid++);
} }
{ {
std::string colVal("v4"); std::string colVal("v4");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++); coj->Put(term, othColId, version++, suid++);
} }
{ {
std::string colVal("v4"); std::string colVal("v4");
for (size_t i = 0; i < 10; i++) { for (size_t i = 0; i < 10; i++) {
colVal[colVal.size() - 1] = 'a' + i; colVal[colVal.size() - 1] = 'a' + i;
SIndexTerm* term = SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
} }
coj->Debug(); coj->Debug();
// begin query // begin query
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType; STermValueType valType;
@ -582,8 +606,9 @@ TEST_F(IndexCacheEnv, cache_test) {
assert(taosArrayGetSize(ret) == 4); assert(taosArrayGetSize(ret) == 4);
} }
{ {
std::string colVal("v2"); std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType; STermValueType valType;
@ -651,32 +676,73 @@ class IndexEnv2 : public ::testing::Test {
}; };
TEST_F(IndexEnv2, testIndexOpen) { TEST_F(IndexEnv2, testIndexOpen) {
std::string path = "/tmp"; std::string path = "/tmp";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {
std::string colName("tag1"), colVal("Hello world"); std::cout << "failed to init index" << std::endl;
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); exit(1);
SIndexMultiTerm* terms = indexMultiTermCreate(); }
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) { int targetSize = 100;
int tableId = i; {
int ret = index->Put(terms, tableId); std::string colName("tag1"), colVal("Hello world");
assert(ret == 0);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < targetSize; i++) {
int tableId = i;
int ret = index->Put(terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
{
size_t size = 100;
std::string colName("tag1"), colVal("hello world");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < size; i++) {
int tableId = i;
int ret = index->Put(terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("tag1"), colVal("Hello world");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
index->Search(mq, result);
assert(taosArrayGetSize(result) == targetSize);
} }
indexMultiTermDestroy(terms);
} }
TEST_F(IndexEnv2, testIndex_CachePut) { TEST_F(IndexEnv2, testIndex_CachePut) {
std::string path = "/tmp"; std::string path = "/tmp";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {
}
} }
TEST_F(IndexEnv2, testIndexr_TFilePut) { TEST_F(IndexEnv2, testIndexr_TFilePut) {
std::string path = "/tmp"; std::string path = "/tmp";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {
}
} }
TEST_F(IndexEnv2, testIndex_CacheSearch) { TEST_F(IndexEnv2, testIndex_CacheSearch) {
std::string path = "/tmp"; std::string path = "/tmp";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {
}
} }
TEST_F(IndexEnv2, testIndex_TFileSearch) { TEST_F(IndexEnv2, testIndex_TFileSearch) {
std::string path = "/tmp"; std::string path = "/tmp";
if (index->Init(path) != 0) {} if (index->Init(path) != 0) {
}
} }