xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more work

This commit is contained in:
Hongze Cheng 2022-06-13 03:25:16 +00:00
parent a26ccade3e
commit cc7640321b
2 changed files with 227 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
source/dnode/vnode/src/inc/tsdb.h
View File

@ -22,10 +22,6 @@
extern "C" { extern "C" {
#endif #endif
#define TSDB_OFFSET_U8 ((uint8_t)0x1)
#define TSDB_OFFSET_U16 ((uint8_t)0x2)
#define TSDB_OFFSET_U32 ((uint8_t)0x4)
// tsdbDebug ================ // tsdbDebug ================
// clang-format off // clang-format off
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) #define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
@ -48,6 +44,7 @@ typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct SMergeInfo SMergeInfo; typedef struct SMergeInfo SMergeInfo;
typedef struct STable STable; typedef struct STable STable;
typedef struct SOffset SOffset;
// tsdbMemTable ============================================================================================== // tsdbMemTable ==============================================================================================
@ -153,16 +150,16 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
int32_t tsdbKeyCmprFn(const void *p1, const void *p2); int32_t tsdbKeyCmprFn(const void *p1, const void *p2);
// SDelIdx // SDelIdx
int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id);
int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx);
int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem); int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem);
int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx);
int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id);
int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx); int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx);
int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx); int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx);
// SDelData // SDelData
int32_t tDelDataGetItem(SDelData *pDelData, SDelDataItem *pItem, int64_t version);
int32_t tDelDataGetItemByIdx(SDelData *pDelData, SDelDataItem *pItem, int32_t idx);
int32_t tDelDataPutItem(SDelData *pDelData, SDelDataItem *pItem); int32_t tDelDataPutItem(SDelData *pDelData, SDelDataItem *pItem);
int32_t tDelDataGetItemByIdx(SDelData *pDelData, SDelDataItem *pItem, int32_t idx);
int32_t tDelDataGetItem(SDelData *pDelData, SDelDataItem *pItem, int64_t version);
int32_t tPutDelData(uint8_t *p, SDelData *pDelData); int32_t tPutDelData(uint8_t *p, SDelData *pDelData);
int32_t tGetDelData(uint8_t *p, SDelData *pDelData); int32_t tGetDelData(uint8_t *p, SDelData *pDelData);
@ -170,6 +167,12 @@ int32_t tPutDelFileHdr(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFileHdr(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFileHdr(uint8_t *p, SDelFile *pDelFile);
// structs // structs
struct SOffset {
int32_t nOffset;
uint8_t flag;
uint8_t *pOffset;
};
typedef struct { typedef struct {
int minFid; int minFid;
int midFid; int midFid;
@ -358,9 +361,7 @@ struct SDelData {
uint32_t delimiter; uint32_t delimiter;
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
uint8_t flags; SOffset offset;
uint32_t nItem;
uint8_t *pOffset;
uint32_t nData; uint32_t nData;
uint8_t *pData; uint8_t *pData;
}; };
@ -378,9 +379,7 @@ struct SDelIdxItem {
struct SDelIdx { struct SDelIdx {
uint32_t delimiter; uint32_t delimiter;
uint8_t flags; SOffset offset;
uint32_t nItem;
uint8_t *pOffset;
uint32_t nData; uint32_t nData;
uint8_t *pData; uint8_t *pData;
}; };

322
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -15,28 +15,118 @@
#include "tsdb.h" #include "tsdb.h"
static FORCE_INLINE int32_t tsdbOffsetSize(uint8_t flags) { #define TSDB_OFFSET_I32 ((uint8_t)0)
if (flags & TSDB_OFFSET_U8) { #define TSDB_OFFSET_I16 ((uint8_t)1)
return sizeof(uint8_t); #define TSDB_OFFSET_I8 ((uint8_t)2)
} else if (flags & TSDB_OFFSET_U16) {
return sizeof(uint16_t); static FORCE_INLINE int32_t tsdbOffsetSize(SOffset *pOfst) {
} else if (flags & TSDB_OFFSET_U32) { switch (pOfst->flag) {
return sizeof(uint32_t); case TSDB_OFFSET_I32:
} else { return sizeof(int32_t);
case TSDB_OFFSET_I16:
return sizeof(int16_t);
case TSDB_OFFSET_I8:
return sizeof(int8_t);
default:
ASSERT(0); ASSERT(0);
} }
} }
static FORCE_INLINE uint32_t tsdbGetOffset(uint8_t *pOffset, uint8_t flags, int32_t idx) { static FORCE_INLINE int32_t tsdbGetOffset(SOffset *pOfst, int32_t idx) {
if (flags & TSDB_OFFSET_U8) { int32_t offset = -1;
return ((uint8_t *)pOffset)[idx];
} else if (flags & TSDB_OFFSET_U16) { if (idx >= 0 && idx < pOfst->nOffset) {
return ((uint16_t *)pOffset)[idx]; switch (pOfst->flag) {
} else if (flags & TSDB_OFFSET_U32) { case TSDB_OFFSET_I32:
return ((uint32_t *)pOffset)[idx]; offset = ((int32_t *)pOfst->pOffset)[idx];
} else { break;
case TSDB_OFFSET_I16:
offset = ((int16_t *)pOfst->pOffset)[idx];
break;
case TSDB_OFFSET_I8:
offset = ((int8_t *)pOfst->pOffset)[idx];
break;
default:
ASSERT(0); ASSERT(0);
} }
ASSERT(offset >= 0);
}
return offset;
}
static FORCE_INLINE int32_t tsdbAddOffset(SOffset *pOfst, int32_t offset) {
int32_t code = 0;
int32_t nOffset = pOfst->nOffset;
ASSERT(pOfst->flag == TSDB_OFFSET_I32);
ASSERT(offset >= 0);
pOfst->nOffset++;
// alloc
code = tsdbRealloc(&pOfst->pOffset, sizeof(int32_t) * pOfst->nOffset);
if (code) goto _exit;
// put
((int32_t *)pOfst->pOffset)[nOffset] = offset;
_exit:
return code;
}
static FORCE_INLINE int32_t tPutOffset(uint8_t *p, SOffset *pOfst) {
int32_t n = 0;
int32_t maxOffset;
ASSERT(pOfst->flag == TSDB_OFFSET_I32);
ASSERT(pOfst->nOffset > 0);
maxOffset = tsdbGetOffset(pOfst, pOfst->nOffset - 1);
n += tPutI32v(p ? p + n : p, pOfst->nOffset);
if (maxOffset <= INT8_MAX) {
n += tPutU8(p ? p + n : p, TSDB_OFFSET_I8);
for (int32_t iOffset = 0; iOffset < pOfst->nOffset; iOffset++) {
n += tPutI8(p ? p + n : p, (int8_t)tsdbGetOffset(pOfst, iOffset));
}
} else if (maxOffset <= INT16_MAX) {
n += tPutU8(p ? p + n : p, TSDB_OFFSET_I16);
for (int32_t iOffset = 0; iOffset < pOfst->nOffset; iOffset++) {
n += tPutI16(p ? p + n : p, (int16_t)tsdbGetOffset(pOfst, iOffset));
}
} else {
n += tPutU8(p ? p + n : p, TSDB_OFFSET_I32);
for (int32_t iOffset = 0; iOffset < pOfst->nOffset; iOffset++) {
n += tPutI32(p ? p + n : p, (int32_t)tsdbGetOffset(pOfst, iOffset));
}
}
return n;
}
static FORCE_INLINE int32_t tGetOffset(uint8_t *p, SOffset *pOfst) {
int32_t n = 0;
n += tGetI32v(p + n, &pOfst->nOffset);
n += tGetU8(p + n, &pOfst->flag);
pOfst->pOffset = p + n;
switch (pOfst->flag) {
case TSDB_OFFSET_I32:
n = n + pOfst->nOffset + sizeof(int32_t);
break;
case TSDB_OFFSET_I16:
n = n + pOfst->nOffset + sizeof(int16_t);
break;
case TSDB_OFFSET_I8:
n = n + pOfst->nOffset + sizeof(int8_t);
break;
default:
ASSERT(0);
}
return n;
} }
int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size) { int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size) {
@ -131,23 +221,59 @@ static FORCE_INLINE int32_t tPutDelIdxItem(uint8_t *p, SDelIdxItem *pDelIdxItem)
static FORCE_INLINE int32_t tGetDelIdxItem(uint8_t *p, SDelIdxItem *pDelIdxItem) { static FORCE_INLINE int32_t tGetDelIdxItem(uint8_t *p, SDelIdxItem *pDelIdxItem) {
int32_t n = 0; int32_t n = 0;
n += tGetI64(p, &pDelIdxItem->suid); n += tGetI64(p + n, &pDelIdxItem->suid);
n += tGetI64(p, &pDelIdxItem->uid); n += tGetI64(p + n, &pDelIdxItem->uid);
n += tGetI64(p, &pDelIdxItem->minKey); n += tGetI64(p + n, &pDelIdxItem->minKey);
n += tGetI64(p, &pDelIdxItem->maxKey); n += tGetI64(p + n, &pDelIdxItem->maxKey);
n += tGetI64v(p, &pDelIdxItem->minVersion); n += tGetI64v(p + n, &pDelIdxItem->minVersion);
n += tGetI64v(p, &pDelIdxItem->maxVersion); n += tGetI64v(p + n, &pDelIdxItem->maxVersion);
n += tGetI64v(p, &pDelIdxItem->offset); n += tGetI64v(p + n, &pDelIdxItem->offset);
n += tGetI64v(p, &pDelIdxItem->size); n += tGetI64v(p + n, &pDelIdxItem->size);
return n; return n;
} }
// SDelIdx ====================================================== // SDelIdx ======================================================
int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem) {
int32_t code = 0;
uint32_t offset = pDelIdx->nData;
// offset
code = tsdbAddOffset(&pDelIdx->offset, offset);
if (code) goto _exit;
// alloc
pDelIdx->nData += tPutDelIdxItem(NULL, pItem);
code = tsdbRealloc(&pDelIdx->pData, pDelIdx->nData);
if (code) goto _exit;
// put
tPutDelIdxItem(pDelIdx->pData + offset, pItem);
_exit:
return code;
}
int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx) {
int32_t code = 0;
int32_t offset;
offset = tsdbGetOffset(&pDelIdx->offset, idx);
if (offset < 0) {
code = TSDB_CODE_NOT_FOUND;
goto _exit;
}
tGetDelIdxItem(pDelIdx->pData + offset, pItem);
_exit:
return code;
}
int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id) { int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id) {
int32_t code = 0; int32_t code = 0;
int32_t lidx = 0; int32_t lidx = 0;
int32_t ridx = pDelIdx->nItem - 1; int32_t ridx = pDelIdx->offset.nOffset - 1;
int32_t midx; int32_t midx;
uint64_t offset; uint64_t offset;
int32_t c; int32_t c;
@ -155,7 +281,9 @@ int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id) {
while (lidx <= ridx) { while (lidx <= ridx) {
midx = (lidx + ridx) / 2; midx = (lidx + ridx) / 2;
tDelIdxGetItemByIdx(pDelIdx, pItem, midx); code = tDelIdxGetItemByIdx(pDelIdx, pItem, midx);
if (code) goto _exit;
c = tTABLEIDCmprFn(&id, pItem); c = tTABLEIDCmprFn(&id, pItem);
if (c == 0) { if (c == 0) {
goto _exit; goto _exit;
@ -172,41 +300,11 @@ _exit:
return code; return code;
} }
int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx) {
tGetDelIdxItem(pDelIdx->pData + tsdbGetOffset(pDelIdx->pOffset, pDelIdx->flags, idx), pItem);
return 0;
}
int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem) {
int32_t code = 0;
int32_t size = tPutDelIdxItem(NULL, pItem);
uint32_t offset = pDelIdx->nData;
uint32_t nItem = pDelIdx->nItem;
pDelIdx->nItem++;
pDelIdx->nData += size;
// alloc
code = tsdbRealloc(&pDelIdx->pOffset, pDelIdx->nItem * sizeof(uint32_t));
if (code) goto _exit;
code = tsdbRealloc(&pDelIdx->pData, pDelIdx->nData);
if (code) goto _exit;
// put
((uint32_t *)pDelIdx->pOffset)[nItem] = offset;
tPutDelIdxItem(pDelIdx->pData + offset, pItem);
_exit:
return code;
}
int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx) { int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx) {
int32_t n = 0; int32_t n = 0;
n += tPutU32(p ? p + n : p, pDelIdx->delimiter); n += tPutU32(p ? p + n : p, pDelIdx->delimiter);
n += tPutU8(p ? p + n : p, pDelIdx->flags); n += tPutOffset(p ? p + n : p, &pDelIdx->offset);
n += tPutU32v(p ? p + n : p, pDelIdx->nItem);
n += tPutBinary(p ? p + n : p, pDelIdx->pOffset, pDelIdx->nItem * tsdbOffsetSize(pDelIdx->flags));
n += tPutBinary(p ? p + n : p, pDelIdx->pData, pDelIdx->nData); n += tPutBinary(p ? p + n : p, pDelIdx->pData, pDelIdx->nData);
return n; return n;
@ -215,11 +313,9 @@ int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx) {
int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx) { int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx) {
int32_t n = 0; int32_t n = 0;
n += tGetU32(p, &pDelIdx->delimiter); n += tGetU32(p + n, &pDelIdx->delimiter);
n += tGetU8(p, &pDelIdx->flags); n += tGetOffset(p + n, &pDelIdx->offset);
n += tGetU32v(p, &pDelIdx->nItem); n += tGetBinary(p + n, &pDelIdx->pData, &pDelIdx->nData);
n += tGetBinary(p, &pDelIdx->pOffset, NULL);
n += tGetBinary(p, &pDelIdx->pData, &pDelIdx->nData);
return n; return n;
} }
@ -238,24 +334,61 @@ static FORCE_INLINE int32_t tPutDelDataItem(uint8_t *p, SDelDataItem *pItem) {
static FORCE_INLINE int32_t tGetDelDataItem(uint8_t *p, SDelDataItem *pItem) { static FORCE_INLINE int32_t tGetDelDataItem(uint8_t *p, SDelDataItem *pItem) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p, &pItem->version); n += tGetI64v(p + n, &pItem->version);
n += tGetI64(p, &pItem->sKey); n += tGetI64(p + n, &pItem->sKey);
n += tGetI64(p, &pItem->eKey); n += tGetI64(p + n, &pItem->eKey);
return n; return n;
} }
// SDelData ====================================================== // SDelData ======================================================
int32_t tDelDataPutItem(SDelData *pDelData, SDelDataItem *pItem) {
int32_t code = 0;
uint32_t offset = pDelData->nData;
// offset
code = tsdbAddOffset(&pDelData->offset, offset);
if (code) goto _exit;
// alloc
pDelData->nData += tPutDelDataItem(NULL, pItem);
code = tsdbRealloc(&pDelData->pData, pDelData->nData);
if (code) goto _exit;
// put
tPutDelDataItem(pDelData->pData + offset, pItem);
_exit:
return code;
}
int32_t tDelDataGetItemByIdx(SDelData *pDelData, SDelDataItem *pItem, int32_t idx) {
int32_t code = 0;
int32_t offset;
offset = tsdbGetOffset(&pDelData->offset, idx);
if (offset < 0) {
code = TSDB_CODE_NOT_FOUND;
goto _exit;
}
tGetDelDataItem(pDelData->pData + offset, pItem);
_exit:
return code;
}
int32_t tDelDataGetItem(SDelData *pDelData, SDelDataItem *pItem, int64_t version) { int32_t tDelDataGetItem(SDelData *pDelData, SDelDataItem *pItem, int64_t version) {
int32_t code = 0; int32_t code = 0;
int32_t lidx = 0; int32_t lidx = 0;
int32_t ridx = pDelData->nItem - 1; int32_t ridx = pDelData->offset.nOffset - 1;
int32_t midx; int32_t midx;
while (lidx <= ridx) { while (lidx <= ridx) {
midx = (lidx + ridx) / 2; midx = (lidx + ridx) / 2;
tDelDataGetItemByIdx(pDelData, pItem, midx); code = tDelDataGetItemByIdx(pDelData, pItem, midx);
if (code) goto _exit;
if (version == pItem->version) { if (version == pItem->version) {
goto _exit; goto _exit;
} else if (version < pItem->version) { } else if (version < pItem->version) {
@ -271,42 +404,13 @@ _exit:
return code; return code;
} }
int32_t tDelDataGetItemByIdx(SDelData *pDelData, SDelDataItem *pItem, int32_t idx) {
tGetDelDataItem(pDelData->pData + tsdbGetOffset(pDelData->pOffset, pDelData->flags, idx), pItem);
return 0;
}
int32_t tDelDataPutItem(SDelData *pDelData, SDelDataItem *pItem) {
int32_t code = 0;
uint32_t nItem = pDelData->nItem;
uint32_t offset = pDelData->nData;
pDelData->nItem++;
pDelData->nData += tPutDelDataItem(NULL, pItem);
// alloc
code = tsdbRealloc(&pDelData->pOffset, pDelData->nItem * sizeof(uint32_t));
if (code) goto _exit;
code = tsdbRealloc(&pDelData->pData, pDelData->nData);
if (code) goto _exit;
// put
((uint32_t *)pDelData->pOffset)[nItem] = offset;
tPutDelDataItem(pDelData->pData + offset, pItem);
_exit:
return code;
}
int32_t tPutDelData(uint8_t *p, SDelData *pDelData) { int32_t tPutDelData(uint8_t *p, SDelData *pDelData) {
int32_t n = 0; int32_t n = 0;
n += tPutU32(p ? p + n : p, pDelData->delimiter); n += tPutU32(p ? p + n : p, pDelData->delimiter);
n += tPutI64(p ? p + n : p, pDelData->suid); n += tPutI64(p ? p + n : p, pDelData->suid);
n += tPutI64(p ? p + n : p, pDelData->uid); n += tPutI64(p ? p + n : p, pDelData->uid);
n += tPutU8(p ? p + n : p, pDelData->flags); n += tPutOffset(p ? p + n : p, &pDelData->offset);
n += tPutU32v(p ? p + n : p, pDelData->nItem);
n += tPutBinary(p ? p + n : p, pDelData->pOffset, pDelData->nItem * tsdbOffsetSize(pDelData->flags));
n += tPutBinary(p ? p + n : p, pDelData->pData, pDelData->nData); n += tPutBinary(p ? p + n : p, pDelData->pData, pDelData->nData);
return n; return n;
@ -315,13 +419,11 @@ int32_t tPutDelData(uint8_t *p, SDelData *pDelData) {
int32_t tGetDelData(uint8_t *p, SDelData *pDelData) { int32_t tGetDelData(uint8_t *p, SDelData *pDelData) {
int32_t n = 0; int32_t n = 0;
n += tGetU32(p, &pDelData->delimiter); n += tGetU32(p + n, &pDelData->delimiter);
n += tGetI64(p, &pDelData->suid); n += tGetI64(p + n, &pDelData->suid);
n += tGetI64(p, &pDelData->uid); n += tGetI64(p + n, &pDelData->uid);
n += tGetU8(p, &pDelData->flags); n += tGetOffset(p + n, &pDelData->offset);
n += tGetU32v(p, &pDelData->nItem); n += tGetBinary(p + n, &pDelData->pData, &pDelData->nData);
n += tGetBinary(p, &pDelData->pOffset, NULL);
n += tGetBinary(p, &pDelData->pData, &pDelData->nData);
return n; return n;
} }
@ -342,12 +444,12 @@ int32_t tPutDelFileHdr(uint8_t *p, SDelFile *pDelFile) {
int32_t tGetDelFileHdr(uint8_t *p, SDelFile *pDelFile) { int32_t tGetDelFileHdr(uint8_t *p, SDelFile *pDelFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64(p, &pDelFile->minKey); n += tGetI64(p + n, &pDelFile->minKey);
n += tGetI64(p, &pDelFile->maxKey); n += tGetI64(p + n, &pDelFile->maxKey);
n += tGetI64v(p, &pDelFile->minVersion); n += tGetI64v(p + n, &pDelFile->minVersion);
n += tGetI64v(p, &pDelFile->maxVersion); n += tGetI64v(p + n, &pDelFile->maxVersion);
n += tGetI64v(p, &pDelFile->size); n += tGetI64v(p + n, &pDelFile->size);
n += tGetI64v(p, &pDelFile->offset); n += tGetI64v(p + n, &pDelFile->offset);
return n; return n;
} }