xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor checkpoint

This commit is contained in:
yihaoDeng 2023-11-07 09:55:55 +08:00
parent 021859361f
commit 95cfc5eb46
1 changed files with 132 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -107,8 +107,6 @@ const char* compactFilteNameState(void* arg);
const char* compactFilteNameFill(void* arg); const char* compactFilteNameFill(void* arg);
const char* compactFilteNameFunc(void* arg); const char* compactFilteNameFunc(void* arg);
unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen,
char** newval, size_t* newvlen, unsigned char* value_changed); char** newval, size_t* newvlen, unsigned char* value_changed);
rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx);
@ -117,7 +115,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rock
rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx);
rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx);
typedef int (*EncodeFunc)(void* key, char* buf); typedef int (*EncodeFunc)(void* key, char* buf);
typedef int (*DecodeFunc)(void* key, char* buf); typedef int (*DecodeFunc)(void* key, char* buf);
typedef int (*ToStringFunc)(void* key, char* buf); typedef int (*ToStringFunc)(void* key, char* buf);
@ -149,7 +146,6 @@ typedef struct {
} SCfInit; } SCfInit;
void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg); void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg);
typedef struct { typedef struct {
void* funcName; void* funcName;
@ -205,32 +201,38 @@ void destroyFunc(void* arg);
int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest);
int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest);
static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len);
static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName,
rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt);
SCfInit ginitDict[] = { SCfInit ginitDict[] = {
{"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName,
destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory,
compactFilteFactoryName},
{"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc,
encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, compactFilteFactoryNameState}, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterState, destroyCompactFilteFactory,
compactFilteFactoryNameState},
{"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc,
encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory,compactFilteFactoryNameFill}, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory,
compactFilteFactoryNameFill},
{"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString,
compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess, destroyCompactFilteFactory,compactFilteFactoryNameSess}, compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess,
destroyCompactFilteFactory, compactFilteFactoryNameSess},
{"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc,
encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, compactFilteFactoryNameFunc}, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory,
compactFilteFactoryNameFunc},
{"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc,
encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory,
compactFilteFactoryName},
{"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc,
encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory,
compactFilteFactoryName},
}; };
const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"};
@ -316,7 +318,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) {
} }
} }
if (p->init == 0) { if (p->init == 0) {
void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL);
while (pIter) { while (pIter) {
size_t len; size_t len;
@ -351,7 +352,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) {
p->curChkpId = chkpId; p->curChkpId = chkpId;
} }
taosHashClear(p->pSstTbl[p->idx]); taosHashClear(p->pSstTbl[p->idx]);
p->idx = 1 - p->idx; p->idx = 1 - p->idx;
@ -359,7 +359,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) {
taosThreadRwlockUnlock(&p->rwLock); taosThreadRwlockUnlock(&p->rwLock);
return 0; return 0;
} }
SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) {
SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp)); SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp));
@ -398,7 +397,6 @@ void dbChkpDestroy(SDbChkp* pChkp) {
taosMemoryFree(pChkp->pCurrent); taosMemoryFree(pChkp->pCurrent);
taosMemoryFree(pChkp->pManifest); taosMemoryFree(pChkp->pManifest);
} }
int32_t dbChkpInit(SDbChkp* p) { int32_t dbChkpInit(SDbChkp* p) {
@ -487,7 +485,6 @@ _ERROR:
taosMemoryFree(srcDir); taosMemoryFree(srcDir);
taosMemoryFree(dstDir); taosMemoryFree(dstDir);
return code; return code;
} }
SBackendManager* bkdMgtCreate(char* path) { SBackendManager* bkdMgtCreate(char* path) {
SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager));
@ -550,11 +547,11 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) {
taosThreadRwlockUnlock(&bm->rwLock); taosThreadRwlockUnlock(&bm->rwLock);
return code; return code;
} }
void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg) { void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg) {
rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName); rocksdb_compactionfilterfactory_t* filterFactory = rocksdb_compactionfilterfactory_create(
arg, ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName);
rocksdb_options_set_compaction_filter_factory(opt, filterFactory); rocksdb_options_set_compaction_filter_factory(opt, filterFactory);
} }
@ -1357,7 +1354,6 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) {
// |key|-----value------| // |key|-----value------|
// |key|ttl|len|userData| // |key|ttl|len|userData|
int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) {
int len = aLen < bLen ? aLen : bLen; int len = aLen < bLen ? aLen : bLen;
int ret = memcmp(aBuf, bBuf, len); int ret = memcmp(aBuf, bBuf, len);
@ -1777,7 +1773,6 @@ const char* compactFilteFactoryNameFunc(void* arg) {
return "stream_compact_filter_func"; return "stream_compact_filter_func";
} }
void destroyCompactFilte(void* arg) { (void)arg; } void destroyCompactFilte(void* arg) { (void)arg; }
unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen,
char** newval, size_t* newvlen, unsigned char* value_changed) { char** newval, size_t* newvlen, unsigned char* value_changed) {
@ -1817,7 +1812,6 @@ unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t kle
// return streamStateValueIsStale((char*)val) ? 1 : 0; // return streamStateValueIsStale((char*)val) ? 1 : 0;
} }
rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) {
SCompactFilteFactory* state = arg; SCompactFilteFactory* state = arg;
rocksdb_compactionfilter_t* filter = rocksdb_compactionfilter_t* filter =
@ -1849,8 +1843,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocks
return filter; return filter;
} }
int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) {
int32_t code = -1; int32_t code = -1;
char* err = NULL; char* err = NULL;
@ -2096,7 +2088,6 @@ void taskDbDestroy(void* pBackend) {
rocksdb_env_destroy(wrapper->env); rocksdb_env_destroy(wrapper->env);
rocksdb_cache_destroy(wrapper->cache); rocksdb_cache_destroy(wrapper->cache);
taosMemoryFree(wrapper->pCf); taosMemoryFree(wrapper->pCf);
for (int i = 0; i < nCf; i++) { for (int i = 0; i < nCf; i++) {
@ -2107,7 +2098,6 @@ void taskDbDestroy(void* pBackend) {
rocksdb_options_destroy(opt); rocksdb_options_destroy(opt);
rocksdb_comparator_destroy(compare); rocksdb_comparator_destroy(compare);
rocksdb_block_based_options_destroy(tblOpt); rocksdb_block_based_options_destroy(tblOpt);
} }
taosMemoryFree(wrapper->pCompares); taosMemoryFree(wrapper->pCompares);
taosMemoryFree(wrapper->pCfOpts); taosMemoryFree(wrapper->pCfOpts);
@ -2553,7 +2543,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
taosMemoryFree(err); \ taosMemoryFree(err); \
code = -1; \ code = -1; \
} else { \ } else { \
qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, ttlVLen, wrapper); \ qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \
ttlVLen, wrapper); \
} \ } \
taosMemoryFree(ttlV); \ taosMemoryFree(ttlV); \
} while (0); } while (0);
@ -2594,7 +2585,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
funcname); \ funcname); \
code = -1; \ code = -1; \
} else { \ } else { \
qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, wrapper); \ qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \
wrapper); \
} \ } \
taosMemoryFree(val); \ taosMemoryFree(val); \
if (vLen != NULL) *vLen = tlen; \ if (vLen != NULL) *vLen = tlen; \