xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor tindex write

This commit is contained in:
yihaoDeng 2021-12-22 15:23:31 +08:00
parent ebcb9be39f
commit 0cd932e8ef
2 changed files with 42 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
source/libs/index/inc/index_tfile.h
View File

@ -27,18 +27,22 @@ extern "C" {
#endif #endif
// tfile header content // tfile header content
// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| // |<---suid--->|<---version--->|<-------colName------>|<---type-->|<--fstOffset->|
// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| // |<-uint64_t->|<---int32_t--->|<--TSDB_COL_NAME_LEN-->|<-uint8_t->|<---int32_t-->|
#pragma pack(push, 1)
typedef struct TFileHeader { typedef struct TFileHeader {
uint64_t suid; uint64_t suid;
int32_t version; int32_t version;
char colName[128]; // char colName[TSDB_COL_NAME_LEN]; //
uint8_t colType; uint8_t colType;
int32_t fstOffset;
} TFileHeader; } TFileHeader;
#pragma pack(pop)
#define TFILE_HEADER_SIZE (sizeof(TFileHeader) + sizeof(uint32_t)) #define TFILE_HEADER_SIZE (sizeof(TFileHeader))
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) #define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t))
//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileCacheKey { typedef struct TFileCacheKey {
uint64_t suid; uint64_t suid;

63
source/libs/index/src/index_tfile.c
View File

@ -48,21 +48,23 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds) {
SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t); SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t);
} }
} }
static FORCE_INLINE int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; }
return 0;
}
static FORCE_INLINE int tfileWriteHeader(TFileWriter* writer) { static FORCE_INLINE int tfileWriteHeader(TFileWriter* writer) {
char buf[TFILE_HEADER_SIZE] = {0}; char buf[TFILE_HEADER_NO_FST] = {0};
char* p = buf; char* p = buf;
TFileHeader* header = &writer->header; TFileHeader* header = &writer->header;
SERIALIZE_MEM_TO_BUF(p, header, suid); memcpy(buf, (char*)header, sizeof(buf));
SERIALIZE_MEM_TO_BUF(p, header, version);
SERIALIZE_VAR_TO_BUF(p, strlen(header->colName), int32_t);
SERIALIZE_STR_MEM_TO_BUF(p, header, colName, strlen(header->colName)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
SERIALIZE_MEM_TO_BUF(p, header, colType); if (sizeof(buf) != nwrite) { return -1; }
int offset = p - buf; writer->offset = nwrite;
int nwrite = writer->ctx->write(writer->ctx, buf, offset);
if (offset != nwrite) { return -1; }
writer->offset = offset;
return 0; return 0;
} }
static int tfileWriteData(TFileWriter* write, TFileValue* tval) { static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
@ -82,26 +84,12 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
} }
static FORCE_INLINE int tfileReadLoadHeader(TFileReader* reader) { static FORCE_INLINE int tfileReadLoadHeader(TFileReader* reader) {
// TODO simple tfile header later // TODO simple tfile header later
char buf[TFILE_HADER_PRE_SIZE]; char buf[TFILE_HEADER_SIZE];
char* p = buf; char* p = buf;
int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HEADER_SIZE);
assert(nread == TFILE_HADER_PRE_SIZE); assert(nread == TFILE_HEADER_SIZE);
memcpy(&reader->header, buf, sizeof(buf));
TFileHeader* header = &reader->header;
memcpy(&header->suid, p, sizeof(header->suid));
p += sizeof(header->suid);
memcpy(&header->version, p, sizeof(header->version));
p += sizeof(header->version);
int32_t colLen = 0;
memcpy(&colLen, p, sizeof(colLen));
assert(colLen < sizeof(header->colName));
nread = reader->ctx->read(reader->ctx, header->colName, colLen);
assert(nread == colLen);
nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType));
return 0; return 0;
} }
@ -285,7 +273,7 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
return tw; return tw;
} }
int TFileWriterPut(TFileWriter* tw, void* data) { int tfileWriterPut(TFileWriter* tw, void* data) {
// sort by coltype and write to tindex // sort by coltype and write to tindex
__compar_fn_t fn = getComparFunc(tw->header.colType, 0); __compar_fn_t fn = getComparFunc(tw->header.colType, 0);
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
@ -294,6 +282,21 @@ int TFileWriterPut(TFileWriter* tw, void* data) {
char* buf = calloc(1, sizeof(bufLimit)); char* buf = calloc(1, sizeof(bufLimit));
char* p = buf; char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data); int32_t sz = taosArrayGetSize((SArray*)data);
int32_t fstOffset = tw->offset;
// ugly code, refactor later
for (size_t i = 0; i < sz; i++) {
TFileValue* v = taosArrayGetP((SArray*)data, i);
int32_t tbsz = taosArrayGetSize(v->tableId);
int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);
fstOffset += ttsz;
}
// check result or not
tfileWriteFstOffset(tw, fstOffset);
// tw->ctx->header.fstOffset = fstOffset;
// tw->ctx->write(tw->ctx, &fstOffset, sizeof(fstOffset));
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
TFileValue* v = taosArrayGetP((SArray*)data, i); TFileValue* v = taosArrayGetP((SArray*)data, i);
@ -311,7 +314,7 @@ int TFileWriterPut(TFileWriter* tw, void* data) {
tfileSerialTableIdsToBuf(p, v->tableId); tfileSerialTableIdsToBuf(p, v->tableId);
offset += ttsz; offset += ttsz;
p = buf + offset; p = buf + offset;
// set up value offset and // set up value offset
v->offset = tw->offset; v->offset = tw->offset;
tw->offset += ttsz; tw->offset += ttsz;
} }