xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #10363 from taosdata/feature/revertIndexImpl 

enhance index
This commit is contained in:
Yihao Deng 2022-02-23 14:28:22 +08:00 committed by GitHub
parent c06aaf01fa 8bedb23757
commit 80b8ca722c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
source/libs/index/src/index_tfile.c
View File

@ -598,25 +598,23 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
return 0; return 0;
} }
static int tfileReaderLoadFst(TFileReader* reader) { static int tfileReaderLoadFst(TFileReader* reader) {
// current load fst into memory, refactor it later WriterCtx* ctx = reader->ctx;
static int FST_MAX_SIZE = 64 * 1024 * 1024; int size = ctx->size(ctx);
char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); // current load fst into memory, refactor it later
int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber);
char* buf = calloc(1, fstSize);
if (buf == NULL) { if (buf == NULL) {
return -1; return -1;
} }
WriterCtx* ctx = reader->ctx;
int size = ctx->size(ctx);
int64_t ts = taosGetTimestampUs(); int64_t ts = taosGetTimestampUs();
int32_t nread = int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset);
ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset);
int64_t cost = taosGetTimestampUs() - ts; int64_t cost = taosGetTimestampUs() - ts;
indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, indexInfo("nread = %d, and fst offset=%d, size: %d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost);
// we assuse fst size less than FST_MAX_SIZE // we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread < FST_MAX_SIZE); assert(nread > 0 && nread <= fstSize);
FstSlice st = fstSliceCreate((uint8_t*)buf, nread); FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
reader->fst = fstCreate(&st); reader->fst = fstCreate(&st);
@ -626,25 +624,35 @@ static int tfileReaderLoadFst(TFileReader* reader) {
return reader->fst != NULL ? 0 : -1; return reader->fst != NULL ? 0 : -1;
} }
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
int32_t nid; // TODO(yihao): opt later
WriterCtx* ctx = reader->ctx; WriterCtx* ctx = reader->ctx;
char block[1024] = {0};
int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
assert(nread >= sizeof(uint32_t));
int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); char* p = block;
assert(sizeof(nid) == nread); int32_t nid = *(int32_t*)p;
p += sizeof(nid);
int32_t total = sizeof(uint64_t) * nid; while (nid > 0) {
char* buf = calloc(1, total); int32_t left = block + sizeof(block) - p;
if (buf == NULL) { if (left >= sizeof(uint64_t)) {
return -1; taosArrayPush(result, (uint64_t*)p);
p += sizeof(uint64_t);
} else {
char buf[sizeof(uint64_t)] = {0};
memcpy(buf, p, left);
memset(block, 0, sizeof(block));
offset += sizeof(block);
nread = ctx->readFrom(ctx, block, sizeof(block), offset);
memcpy(buf + left, block, sizeof(uint64_t) - left);
taosArrayPush(result, (uint64_t*)buf);
p = block + sizeof(uint64_t) - left;
} }
nid -= 1;
nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid));
assert(total == nread);
for (int32_t i = 0; i < nid; i++) {
taosArrayPush(result, (uint64_t*)buf + i);
} }
free(buf);
return 0; return 0;
} }
static int tfileReaderVerify(TFileReader* reader) { static int tfileReaderVerify(TFileReader* reader) {
@ -686,11 +694,10 @@ void tfileReaderUnRef(TFileReader* reader) {
} }
static SArray* tfileGetFileList(const char* path) { static SArray* tfileGetFileList(const char* path) {
SArray* files = taosArrayInit(4, sizeof(void*));
char buf[128] = {0}; char buf[128] = {0};
uint64_t suid; uint64_t suid;
uint32_t version; uint32_t version;
SArray* files = taosArrayInit(4, sizeof(void*));
DIR* dir = opendir(path); DIR* dir = opendir(path);
if (NULL == dir) { if (NULL == dir) {

1
source/libs/index/test/fstUT.cc
View File

@ -230,3 +230,4 @@ TEST_F(FstEnv, writeNormal) {
assert(fst->Get("aa", &val) == true); assert(fst->Get("aa", &val) == true);
assert(val == 0); assert(val == 0);
} }
TEST_F(FstEnv, writeExcpet) {}