From ee8c09667e955cbb67e12e2cfa3f6ad7570d2360 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Sep 2023 12:53:58 +0000 Subject: [PATCH 001/151] refactor backend --- source/libs/stream/src/streamMeta.c | 135 ++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 35 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 652ef7cde7..5c64619ca9 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -103,6 +103,76 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { return 0; } +typedef struct { + int64_t chkpId; + char* path; + char* taskId; + + SArray* pChkpSave; + SArray* pChkpInUse; + int8_t chkpCap; + void* backend; + +} StreamMetaTaskState; + +int32_t streamMetaOpenTdb(SStreamMeta* pMeta) { + if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { + return -1; + // goto _err; + } + + if (tdbTbOpen("task.db", STREAM_TASK_KEY_LEN, -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { + return -1; + } + + if (tdbTbOpen("checkpoint.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pCheckpointDb, 0) < 0) { + return -1; + } + return 0; +} + +// +// impl later +// +enum STREAM_STATE_VER { + STREAM_STATA_NO_COMPATIBLE, + STREAM_STATA_COMPATIBLE, + STREAM_STATA_NEED_CONVERT, +}; + +int32_t streamMetaCheckStateVer(SStreamMeta* pMeta) { + TBC* pCur = NULL; + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { + // no task info, no stream + return STREAM_STATA_COMPATIBLE; + } + + void* pKey = NULL; + int32_t kLen = 0; + void* pVal = NULL; + int32_t vLen = 0; + SDecoder decoder; + + tdbTbcMoveToFirst(pCur); + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + if (pVal == NULL || vLen == 0) { + break; + } + SCheckpointInfo info; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + if (tDecodeStreamTaskChkInfo(&decoder, &info) < 0) { + continue; + } + tDecoderClear(&decoder); + } + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + + return STREAM_STATA_NEED_CONVERT; +} + +int32_t streamMetaDoStateDataConvert(SStreamMeta* pMeta) { return 0; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); @@ -118,15 +188,11 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; - if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { + if (streamMetaOpenTdb(pMeta) < 0) { goto _err; } - if (tdbTbOpen("task.db", STREAM_TASK_KEY_LEN, -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("checkpoint.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pCheckpointDb, 0) < 0) { + if (streamMetaDoStateDataConvert(pMeta) < 0) { goto _err; } @@ -134,8 +200,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); - pMeta->pTasks = taosHashInit(64, fp, true, HASH_NO_LOCK); + pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); if (pMeta->pTasks == NULL) { goto _err; } @@ -153,41 +218,43 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->expandFunc = expandFunc; pMeta->stage = stage; - // send heartbeat every 5sec. - pMeta->rid = taosAddRef(streamMetaId, pMeta); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); + pMeta->rid = taosAddRef(streamMetaId, pMeta); *pRid = pMeta->rid; metaRefMgtAdd(pMeta->vgId, pRid); + // send heartbeat every 5sec. + // TODO: start hb later pMeta->hbInfo.hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); pMeta->hbInfo.tickCounter = 0; pMeta->hbInfo.stopFlag = 0; - pMeta->pTaskBackendUnique = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t)); - pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t)); - pMeta->chkpCap = 8; - taosInitRWLatch(&pMeta->chkpDirLock); + // start backend + // taosInitRWLatch(&pMeta->chkpDirLock); - pMeta->chkpId = streamGetLatestCheckpointId(pMeta); - pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - while (pMeta->streamBackend == NULL) { - taosMsleep(2 * 1000); - pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - if (pMeta->streamBackend == NULL) { - qError("vgId:%d failed to init stream backend", pMeta->vgId); - qInfo("vgId:%d retry to init stream backend", pMeta->vgId); - } - } - pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); + // pMeta->pTaskBackendUnique = + // taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + // pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t)); + // pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t)); + // pMeta->chkpCap = 8; - code = streamBackendLoadCheckpointInfo(pMeta); + // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); + // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); + // while (pMeta->streamBackend == NULL) { + // qError("vgId:%d failed to init stream backend", pMeta->vgId); + // taosMsleep(2 * 1000); + // qInfo("vgId:%d retry to init stream backend", pMeta->vgId); + // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); + // if (pMeta->streamBackend == NULL) { + // } + // } + // pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); + + // code = streamBackendLoadCheckpointInfo(pMeta); + // taosThreadMutexInit(&pMeta->backendMutex, NULL); taosInitRWLatch(&pMeta->lock); - taosThreadMutexInit(&pMeta->backendMutex, NULL); - pMeta->pauseTaskNum = 0; qInfo("vgId:%d open stream meta successfully, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, @@ -241,13 +308,11 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) { pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); while (pMeta->streamBackend == NULL) { + qError("vgId:%d failed to init stream backend", pMeta->vgId); taosMsleep(2 * 1000); + + qInfo("vgId:%d retry to init stream backend", pMeta->vgId); pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - if (pMeta->streamBackend == NULL) { - qError("vgId:%d failed to init stream backend", pMeta->vgId); - qInfo("vgId:%d retry to init stream backend", pMeta->vgId); - // return -1; - } } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); streamBackendLoadCheckpointInfo(pMeta); From 470849982e26eac46db1b7d714b60cb9589f244b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Sep 2023 09:41:30 +0000 Subject: [PATCH 002/151] add cvt func --- include/libs/stream/tstream.h | 32 ++-- source/libs/stream/inc/streamBackendRocksdb.h | 24 +++ source/libs/stream/src/streamBackendRocksdb.c | 178 +++++++++++++++++- source/libs/stream/src/streamMeta.c | 63 +++++-- source/libs/stream/src/streamTask.c | 5 +- 5 files changed, 270 insertions(+), 32 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 3eb624f932..29dc054d78 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -31,7 +31,10 @@ extern "C" { typedef struct SStreamTask SStreamTask; -#define SSTREAM_TASK_VER 2 +#define SSTREAM_TASK_VER 3 +#define SSTREAM_TASK_INCOMPATIBLE_VER 1 +#define SSTREAM_TASK_NEED_CONVERT_VER 2 + enum { STREAM_STATUS__NORMAL = 0, STREAM_STATUS__STOP, @@ -117,7 +120,7 @@ typedef struct { typedef struct { int8_t type; int64_t ver; - SArray* submits; // SArray + SArray* submits; // SArray } SStreamMergedSubmit; typedef struct { @@ -257,8 +260,9 @@ typedef struct SStreamTaskId { typedef struct SCheckpointInfo { int64_t checkpointId; - int64_t checkpointVer; // latest checkpointId version - int64_t nextProcessVer; // current offset in WAL, not serialize it + int64_t checkpointVer; // latest checkpointId version + int64_t nextProcessVer; // current offset in WAL, not serialize it + int64_t msgVer; } SCheckpointInfo; typedef struct SStreamStatus { @@ -283,8 +287,8 @@ typedef struct SSTaskBasicInfo { int32_t selfChildId; int32_t totalLevel; int8_t taskLevel; - int8_t fillHistory; // is fill history task or not - int64_t triggerParam; // in msec + int8_t fillHistory; // is fill history task or not + int64_t triggerParam; // in msec } SSTaskBasicInfo; typedef struct SDispatchMsgInfo { @@ -301,13 +305,13 @@ typedef struct STaskOutputInfo { } STaskOutputInfo; typedef struct STaskInputInfo { - int8_t status; + int8_t status; SStreamQueue* queue; } STaskInputInfo; typedef struct STaskSchedInfo { - int8_t status; - void* pTimer; + int8_t status; + void* pTimer; } STaskSchedInfo; typedef struct SSinkTaskRecorder { @@ -325,10 +329,10 @@ typedef struct { } STaskTimestamp; typedef struct STokenBucket { - int32_t capacity; // total capacity - int64_t fillTimestamp;// fill timestamp - int32_t numOfToken; // total available tokens - int32_t rate; // number of token per second + int32_t capacity; // total capacity + int64_t fillTimestamp; // fill timestamp + int32_t numOfToken; // total available tokens + int32_t rate; // number of token per second } STokenBucket; struct SStreamTask { @@ -650,7 +654,7 @@ bool streamTaskShouldStop(const SStreamStatus* pStatus); bool streamTaskShouldPause(const SStreamStatus* pStatus); bool streamTaskIsIdle(const SStreamTask* pTask); -void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); +void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 39854d1824..5bcc29287d 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -42,8 +42,31 @@ typedef struct { TdThreadMutex cfMutex; SHashObj* cfInst; int64_t defaultCfInit; + } SBackendWrapper; +typedef struct { + void* tableOpt; +} RocksdbCfParam; + +typedef struct { + rocksdb_t* db; + rocksdb_writeoptions_t* writeOpts; + rocksdb_readoptions_t* readOpts; + rocksdb_options_t* dbOpt; + rocksdb_env_t* env; + rocksdb_cache_t* cache; + + rocksdb_column_family_handle_t** pCf; + rocksdb_comparator_t** pCompares; + rocksdb_options_t** pCfOpts; + RocksdbCfParam* pCfParams; + + rocksdb_compactionfilterfactory_t* filterFactory; + TdThreadMutex cfMutex; + +} STaskBackendWrapper; + void* streamBackendInit(const char* path, int64_t chkpId); void streamBackendCleanup(void* arg); void streamBackendHandleCleanup(void* arg); @@ -51,6 +74,7 @@ int32_t streamBackendLoadCheckpointInfo(void* pMeta); int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); +int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 19f4ebbeea..7feb029e81 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -45,9 +45,6 @@ typedef struct SCompactFilteFactory { void* status; } SCompactFilteFactory; -typedef struct { - void* tableOpt; -} RocksdbCfParam; typedef struct { rocksdb_t* db; rocksdb_column_family_handle_t** pHandle; @@ -1455,6 +1452,181 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { taosMemoryFree(inst); } +STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { + int32_t code = 0; + + char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); + + STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(SBackendWrapper)); + rocksdb_env_t* env = rocksdb_create_default_env(); + + rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); + rocksdb_options_t* opts = rocksdb_options_create(); + // rocksdb_options_set_env(opts, env); + rocksdb_options_set_create_if_missing(opts, 1); + rocksdb_options_set_create_missing_column_families(opts, 1); + // rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); + rocksdb_options_set_recycle_log_file_num(opts, 6); + rocksdb_options_set_max_write_buffer_number(opts, 3); + rocksdb_options_set_info_log_level(opts, 1); + // rocksdb_options_set_db_write_buffer_size(opts, dbMemLimit); + // rocksdb_options_set_write_buffer_size(opts, dbMemLimit / 2); + rocksdb_options_set_atomic_flush(opts, 1); + + pTaskBackend->env = env; + pTaskBackend->dbOpt = opts; + pTaskBackend->cache = cache; + pTaskBackend->filterFactory = rocksdb_compactionfilterfactory_create( + NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); + rocksdb_options_set_compaction_filter_factory(pTaskBackend->dbOpt, pTaskBackend->filterFactory); + + char* err = NULL; + size_t nCf = 0; + + char** cfs = rocksdb_list_column_families(opts, taskPath, &nCf, &err); + if (nCf == 0 || nCf == 1 || err != NULL) { + taosMemoryFreeClear(err); + pTaskBackend->db = rocksdb_open(opts, taskPath, &err); + if (err != NULL) { + qError("failed to open rocksdb, path:%s, reason:%s", taskPath, err); + taosMemoryFreeClear(err); + } + } else { + ASSERT(0); + } + if (cfs != NULL) { + rocksdb_list_column_families_destroy(cfs, nCf); + } + qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); + + nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + pTaskBackend->pCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + pTaskBackend->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); + pTaskBackend->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + pTaskBackend->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + + for (int i = 0; i < nCf; i++) { + rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskBackend->dbOpt); + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, pTaskBackend->cache); + rocksdb_block_based_options_set_partition_filters(tableOpt, 1); + + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); + + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)opt, tableOpt); + + SCfInit* cfPara = &ginitDict[i]; + + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); + + pTaskBackend->pCompares[i] = compare; + pTaskBackend->pCfOpts[i] = opt; + pTaskBackend->pCfParams[i].tableOpt = tableOpt; + } + + return pTaskBackend; +} + +int8_t getCfIdx(const char* key) { + int idx = -1; + size_t len = strlen(key); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (len == ginitDict[i].len && strncmp(key, ginitDict[i].key, strlen(key)) == 0) { + idx = i; + break; + } + } + return idx; +} +rocksdb_column_family_handle_t* taskBackendOpenCf(STaskBackendWrapper* pBackend, const char* key) { + char* err = NULL; + int8_t idx = getCfIdx(key); + if (idx == -1) return NULL; + + rocksdb_column_family_handle_t* cf = + rocksdb_create_column_family(pBackend->db, pBackend->pCfOpts[idx], ginitDict[idx].key, &err); + if (err != NULL) { + qError("failed to open cf, key:%s, reason: %s", key, err); + taosMemoryFree(err); + return NULL; + } + return cf; +} +int32_t copyData(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { + int32_t WRITE_BATCH = 1024; + char* err = NULL; + int code = 0; + + rocksdb_readoptions_t* pRdOpt = rocksdb_readoptions_create(); + + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_iterator_t* pIter = rocksdb_create_iterator_cf(pSrc->db, pRdOpt, pSrc->pHandle[i]); + rocksdb_iter_seek_to_first(pIter); + while (rocksdb_iter_valid(pIter)) { + if (rocksdb_writebatch_count(wb) >= WRITE_BATCH) { + rocksdb_write(pDst->db, pDst->writeOpts, wb, &err); + if (err != NULL) { + code = -1; + goto _EXIT; + } + rocksdb_writebatch_clear(wb); + } + + size_t klen = 0, vlen = 0; + char* key = (char*)rocksdb_iter_key(pIter, &klen); + char* val = (char*)rocksdb_iter_value(pIter, &vlen); + + rocksdb_writebatch_put_cf(wb, pDst->pCf[i], key, klen, val, vlen); + rocksdb_iter_next(pIter); + } + + if (rocksdb_writebatch_count(wb) > 0) { + rocksdb_write(pDst->db, pDst->writeOpts, wb, &err); + if (err != NULL) { + code = -1; + goto _EXIT; + } + } + +_EXIT: + rocksdb_iter_destroy(pIter); + rocksdb_readoptions_destroy(pRdOpt); + taosMemoryFree(err); + + return code; +} + +int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst) { + int nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + + int32_t code = 0; + RocksdbCfInst* CfInst = cfInst; + + STaskBackendWrapper* pTaskBackend = streamStateOpenTaskBackend(path, key); + RocksdbCfInst* pSrcBackend = cfInst; + + for (int i = 0; i < nCf; i++) { + rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; + if (pSrcCf == NULL) continue; + + rocksdb_column_family_handle_t* pDstCf = taskBackendOpenCf(pTaskBackend, ginitDict[i].key); + if (pDstCf == NULL) { + return -1; + } + + code = copyData(pSrcBackend, pTaskBackend, i); + if (code != 0) { + return -1; + } + + pTaskBackend->pCf[i] = pDstCf; + } + return 0; +} int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { SBackendWrapper* handle = backend; char* err = NULL; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 5c64619ca9..f47961d026 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -140,39 +140,78 @@ enum STREAM_STATE_VER { STREAM_STATA_NEED_CONVERT, }; -int32_t streamMetaCheckStateVer(SStreamMeta* pMeta) { - TBC* pCur = NULL; +int32_t streamMetaCheckStateCompatible(SStreamMeta* pMeta) { + int8_t ret = STREAM_STATA_COMPATIBLE; + TBC* pCur = NULL; + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream - return STREAM_STATA_COMPATIBLE; + return ret; } - - void* pKey = NULL; - int32_t kLen = 0; - void* pVal = NULL; - int32_t vLen = 0; - SDecoder decoder; + void* pKey = NULL; + int32_t kLen = 0; + void* pVal = NULL; + int32_t vLen = 0; tdbTbcMoveToFirst(pCur); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { if (pVal == NULL || vLen == 0) { break; } + SDecoder decoder; SCheckpointInfo info; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); if (tDecodeStreamTaskChkInfo(&decoder, &info) < 0) { continue; } + if (info.msgVer <= SSTREAM_TASK_INCOMPATIBLE_VER) { + ret = STREAM_STATA_NO_COMPATIBLE; + } else if (info.msgVer == SSTREAM_TASK_NEED_CONVERT_VER) { + ret = STREAM_STATA_NEED_CONVERT; + } else if (info.msgVer == SSTREAM_TASK_VER) { + ret = STREAM_STATA_COMPATIBLE; + } tDecoderClear(&decoder); + break; } tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); - - return STREAM_STATA_NEED_CONVERT; + return ret; } -int32_t streamMetaDoStateDataConvert(SStreamMeta* pMeta) { return 0; } +int32_t streamMetaDoStateDataConvertImpl(SStreamMeta* pMeta) { + int64_t chkpId = streamGetLatestCheckpointId(pMeta); + SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId); + + void* pIter = taosHashIterate(pBackend->cfInst, NULL); + while (pIter) { + size_t len = 0; + void* key = taosHashGetKey(pIter, &len); + + pIter = taosHashIterate(pBackend->cfInst, pIter); + } + + // streamBackendCleanup(); + + return 0; +} +int32_t streamMetaDoStateDataConvert(SStreamMeta* pMeta) { + int8_t compatible = streamMetaCheckStateCompatible(pMeta); + if (compatible == STREAM_STATA_COMPATIBLE) { + return 0; + } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { + qError( + "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " + "manually", + tsDataDir); + return -1; + } else if (compatible == STREAM_STATA_NEED_CONVERT) { + qError("stream state need covert backend format"); + return streamMetaDoStateDataConvertImpl(pMeta); + } + return 0; +} SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 663deca171..837f787dd2 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -226,9 +226,8 @@ int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) SEpSet epSet; if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &ver) < 0) return -1; - - if (ver != SSTREAM_TASK_VER) return -1; + if (tDecodeI64(pDecoder, &pChkpInfo->msgVer) < 0) return -1; + // if (ver != SSTREAM_TASK_VER) return -1; if (tDecodeI64(pDecoder, &skip64) < 0) return -1; if (tDecodeI32(pDecoder, &skip32) < 0) return -1; From 19e52c364e8ecfc496686ea743fd69a2dc512709 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 22 Sep 2023 08:26:03 +0000 Subject: [PATCH 003/151] add cvt state --- source/libs/stream/inc/streamBackendRocksdb.h | 23 +-- source/libs/stream/src/streamBackendRocksdb.c | 133 ++++++++++++------ source/libs/stream/src/streamMeta.c | 20 ++- 3 files changed, 117 insertions(+), 59 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 5bcc29287d..3e07f2532e 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -51,8 +51,8 @@ typedef struct { typedef struct { rocksdb_t* db; - rocksdb_writeoptions_t* writeOpts; - rocksdb_readoptions_t* readOpts; + rocksdb_writeoptions_t* writeOpt; + rocksdb_readoptions_t* readOpt; rocksdb_options_t* dbOpt; rocksdb_env_t* env; rocksdb_cache_t* cache; @@ -63,18 +63,19 @@ typedef struct { RocksdbCfParam* pCfParams; rocksdb_compactionfilterfactory_t* filterFactory; - TdThreadMutex cfMutex; + TdThreadMutex mutex; } STaskBackendWrapper; -void* streamBackendInit(const char* path, int64_t chkpId); -void streamBackendCleanup(void* arg); -void streamBackendHandleCleanup(void* arg); -int32_t streamBackendLoadCheckpointInfo(void* pMeta); -int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); -SListNode* streamBackendAddCompare(void* backend, void* arg); -void streamBackendDelCompare(void* backend, void* arg); -int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); +void* streamBackendInit(const char* path, int64_t chkpId); +void streamBackendCleanup(void* arg); +void streamBackendHandleCleanup(void* arg); +int32_t streamBackendLoadCheckpointInfo(void* pMeta); +int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); +SListNode* streamBackendAddCompare(void* backend, void* arg); +void streamBackendDelCompare(void* backend, void* arg); +int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); +STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 7feb029e81..ff53ec6c51 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1457,6 +1457,15 @@ STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); + if (!taosDirExist(taskPath)) { + code = taosMkDir(taskPath); + if (code != 0) { + qError("failed to create dir: %s, reason:%s", taskPath, tstrerror(code)); + taosMemoryFree(taskPath); + return NULL; + } + } + char* err = NULL; STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(SBackendWrapper)); rocksdb_env_t* env = rocksdb_create_default_env(); @@ -1474,33 +1483,16 @@ STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { // rocksdb_options_set_write_buffer_size(opts, dbMemLimit / 2); rocksdb_options_set_atomic_flush(opts, 1); - pTaskBackend->env = env; pTaskBackend->dbOpt = opts; + pTaskBackend->env = env; pTaskBackend->cache = cache; pTaskBackend->filterFactory = rocksdb_compactionfilterfactory_create( NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); rocksdb_options_set_compaction_filter_factory(pTaskBackend->dbOpt, pTaskBackend->filterFactory); + pTaskBackend->readOpt = rocksdb_readoptions_create(); + pTaskBackend->writeOpt = rocksdb_writeoptions_create(); - char* err = NULL; - size_t nCf = 0; - - char** cfs = rocksdb_list_column_families(opts, taskPath, &nCf, &err); - if (nCf == 0 || nCf == 1 || err != NULL) { - taosMemoryFreeClear(err); - pTaskBackend->db = rocksdb_open(opts, taskPath, &err); - if (err != NULL) { - qError("failed to open rocksdb, path:%s, reason:%s", taskPath, err); - taosMemoryFreeClear(err); - } - } else { - ASSERT(0); - } - if (cfs != NULL) { - rocksdb_list_column_families_destroy(cfs, nCf); - } - qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); - - nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + size_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); pTaskBackend->pCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); pTaskBackend->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); pTaskBackend->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); @@ -1528,8 +1520,65 @@ STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { pTaskBackend->pCfParams[i].tableOpt = tableOpt; } + char** cfs = rocksdb_list_column_families(opts, taskPath, &nCf, &err); + if (nCf == 0 || nCf == 1 || err != NULL) { + taosMemoryFreeClear(err); + pTaskBackend->db = rocksdb_open(opts, taskPath, &err); + if (err != NULL) { + qError("failed to open rocksdb, path:%s, reason:%s", taskPath, err); + taosMemoryFreeClear(err); + } + } else { + } + if (cfs != NULL) { + rocksdb_list_column_families_destroy(cfs, nCf); + } + + taosThreadMutexInit(&pTaskBackend->mutex, NULL); + taosMemoryFree(taskPath); + + qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); return pTaskBackend; } +void streamMeteCloseTaskBackend(STaskBackendWrapper* wrapper) { + if (wrapper == NULL) return; + + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flushOpt, 1); + + int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + char* err = NULL; + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); + if (err != NULL) { + qError("failed to flush cf:%s, reason:%s", ginitDict[i].key, err); + taosMemoryFreeClear(err); + } + rocksdb_flushoptions_destroy(flushOpt); + } + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + } + } + rocksdb_options_destroy(wrapper->dbOpt); + rocksdb_readoptions_destroy(wrapper->readOpt); + rocksdb_writeoptions_destroy(wrapper->writeOpt); + rocksdb_env_destroy(wrapper->env); + rocksdb_cache_destroy(wrapper->cache); + + taosMemoryFree(wrapper->pCf); + taosMemoryFree(wrapper->pCompares); + taosMemoryFree(wrapper->pCfOpts); + taosMemoryFree(wrapper->pCfParams); + + taosThreadMutexDestroy(&wrapper->mutex); + + rocksdb_close(wrapper->db); + taosMemoryFree(wrapper); + + return; +} int8_t getCfIdx(const char* key) { int idx = -1; @@ -1542,21 +1591,25 @@ int8_t getCfIdx(const char* key) { } return idx; } -rocksdb_column_family_handle_t* taskBackendOpenCf(STaskBackendWrapper* pBackend, const char* key) { - char* err = NULL; - int8_t idx = getCfIdx(key); - if (idx == -1) return NULL; +int32_t taskBackendOpenCf(STaskBackendWrapper* pBackend, const char* key) { + int32_t code = 0; + char* err = NULL; + int8_t idx = getCfIdx(key); + if (idx == -1) return -1; rocksdb_column_family_handle_t* cf = rocksdb_create_column_family(pBackend->db, pBackend->pCfOpts[idx], ginitDict[idx].key, &err); if (err != NULL) { qError("failed to open cf, key:%s, reason: %s", key, err); taosMemoryFree(err); - return NULL; + code = -1; + return code; } - return cf; + + pBackend->pCf[idx] = cf; + return code; } -int32_t copyData(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { +int32_t copyDataAt(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { int32_t WRITE_BATCH = 1024; char* err = NULL; int code = 0; @@ -1568,7 +1621,7 @@ int32_t copyData(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { rocksdb_iter_seek_to_first(pIter); while (rocksdb_iter_valid(pIter)) { if (rocksdb_writebatch_count(wb) >= WRITE_BATCH) { - rocksdb_write(pDst->db, pDst->writeOpts, wb, &err); + rocksdb_write(pDst->db, pDst->writeOpt, wb, &err); if (err != NULL) { code = -1; goto _EXIT; @@ -1585,7 +1638,7 @@ int32_t copyData(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { } if (rocksdb_writebatch_count(wb) > 0) { - rocksdb_write(pDst->db, pDst->writeOpts, wb, &err); + rocksdb_write(pDst->db, pDst->writeOpt, wb, &err); if (err != NULL) { code = -1; goto _EXIT; @@ -1613,19 +1666,17 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst) { rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; if (pSrcCf == NULL) continue; - rocksdb_column_family_handle_t* pDstCf = taskBackendOpenCf(pTaskBackend, ginitDict[i].key); - if (pDstCf == NULL) { - return -1; - } + code = taskBackendOpenCf(pTaskBackend, ginitDict[i].key); + if (code != 0) goto _EXIT; - code = copyData(pSrcBackend, pTaskBackend, i); - if (code != 0) { - return -1; - } - - pTaskBackend->pCf[i] = pDstCf; + code = copyDataAt(pSrcBackend, pTaskBackend, i); + if (code != 0) goto _EXIT; } - return 0; + +_EXIT: + streamMeteCloseTaskBackend(pTaskBackend); + + return code; } int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { SBackendWrapper* handle = backend; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f47961d026..d1f59d0a8f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -181,17 +181,20 @@ int32_t streamMetaCheckStateCompatible(SStreamMeta* pMeta) { } int32_t streamMetaDoStateDataConvertImpl(SStreamMeta* pMeta) { + int32_t code = 0; int64_t chkpId = streamGetLatestCheckpointId(pMeta); SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId); void* pIter = taosHashIterate(pBackend->cfInst, NULL); while (pIter) { - size_t len = 0; - void* key = taosHashGetKey(pIter, &len); - + void* key = taosHashGetKey(pIter, NULL); + code = streamStateConvertDataFormat(pMeta->path, key, *(void**)pIter); + if (code != 0) { + // continue + } + pIter = taosHashIterate(pBackend->cfInst, pIter); } - // streamBackendCleanup(); return 0; @@ -200,18 +203,21 @@ int32_t streamMetaDoStateDataConvert(SStreamMeta* pMeta) { int8_t compatible = streamMetaCheckStateCompatible(pMeta); if (compatible == STREAM_STATA_COMPATIBLE) { return 0; + } else if (compatible == STREAM_STATA_NEED_CONVERT) { + qInfo("stream state need covert backend format"); + + return streamMetaDoStateDataConvertImpl(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { qError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " "manually", tsDataDir); + return -1; - } else if (compatible == STREAM_STATA_NEED_CONVERT) { - qError("stream state need covert backend format"); - return streamMetaDoStateDataConvertImpl(pMeta); } return 0; } + SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); From 173604ee1afcbdca4fc32124a84418c499a4d8d9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 22 Sep 2023 08:32:14 +0000 Subject: [PATCH 004/151] add cvt state --- source/libs/stream/src/streamBackendRocksdb.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ff53ec6c51..5df33949ef 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1653,14 +1653,13 @@ _EXIT: return code; } -int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst) { +int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { int nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); - int32_t code = 0; - RocksdbCfInst* CfInst = cfInst; + int32_t code = 0; STaskBackendWrapper* pTaskBackend = streamStateOpenTaskBackend(path, key); - RocksdbCfInst* pSrcBackend = cfInst; + RocksdbCfInst* pSrcBackend = pCfInst; for (int i = 0; i < nCf; i++) { rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; From 970b1cb84065ed9bb240d97267b0617b749f496d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 22 Sep 2023 08:38:41 +0000 Subject: [PATCH 005/151] add cvt state --- source/libs/stream/src/streamMeta.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d1f59d0a8f..aa29ea8159 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -180,7 +180,7 @@ int32_t streamMetaCheckStateCompatible(SStreamMeta* pMeta) { return ret; } -int32_t streamMetaDoStateDataConvertImpl(SStreamMeta* pMeta) { +int32_t streamMetaDoStateBackendConvertImpl(SStreamMeta* pMeta) { int32_t code = 0; int64_t chkpId = streamGetLatestCheckpointId(pMeta); SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId); @@ -190,23 +190,25 @@ int32_t streamMetaDoStateDataConvertImpl(SStreamMeta* pMeta) { void* key = taosHashGetKey(pIter, NULL); code = streamStateConvertDataFormat(pMeta->path, key, *(void**)pIter); if (code != 0) { - // continue + qError("failed to cvt data"); + goto _EXIT; } pIter = taosHashIterate(pBackend->cfInst, pIter); } - // streamBackendCleanup(); - return 0; +_EXIT: + streamBackendCleanup((void*)pBackend); + return code; } -int32_t streamMetaDoStateDataConvert(SStreamMeta* pMeta) { +int32_t streamMetaMayDoStateBackendConvert(SStreamMeta* pMeta) { int8_t compatible = streamMetaCheckStateCompatible(pMeta); if (compatible == STREAM_STATA_COMPATIBLE) { return 0; } else if (compatible == STREAM_STATA_NEED_CONVERT) { qInfo("stream state need covert backend format"); - return streamMetaDoStateDataConvertImpl(pMeta); + return streamMetaDoStateBackendConvertImpl(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { qError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " @@ -237,7 +239,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - if (streamMetaDoStateDataConvert(pMeta) < 0) { + if (streamMetaMayDoStateBackendConvert(pMeta) < 0) { goto _err; } From 11da631bb5291e157176e9a7a8be7ef6ec073671 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 22 Sep 2023 09:08:42 +0000 Subject: [PATCH 006/151] add cvt state --- include/libs/stream/tstream.h | 6 +++++- source/dnode/vnode/src/tq/tq.c | 32 +++++++++++++++-------------- source/libs/stream/src/streamMeta.c | 16 +++++++++++++-- source/libs/stream/src/streamTask.c | 2 ++ 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 29dc054d78..e09ffb416c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -31,7 +31,7 @@ extern "C" { typedef struct SStreamTask SStreamTask; -#define SSTREAM_TASK_VER 3 +#define SSTREAM_TASK_VER 3 #define SSTREAM_TASK_INCOMPATIBLE_VER 1 #define SSTREAM_TASK_NEED_CONVERT_VER 2 @@ -382,6 +382,7 @@ struct SStreamTask { int32_t transferStateAlignCnt; struct SStreamMeta* pMeta; SSHashObj* pNameMap; + void* pBackend; char reserve[256]; }; @@ -422,6 +423,8 @@ typedef struct SStreamMeta { int32_t chkpCap; SRWLatch chkpDirLock; int32_t pauseTaskNum; + + // SHashObj* pTaskBackend; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -722,6 +725,7 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3aeb679eb7..e136b9b265 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -582,9 +582,9 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosWLockLatch(&pTq->lock); bool exec = tqIsHandleExec(pHandle); - if(exec){ + if (exec) { tqInfo("vgId:%d, topic:%s, subscription is executing, delete wait for 10ms and retry, pHandle:%p", vgId, - pHandle->subKey, pHandle); + pHandle->subKey, pHandle); taosWUnLockLatch(&pTq->lock); taosMsleep(10); continue; @@ -699,12 +699,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } ret = tqMetaSaveHandle(pTq, req.subKey, &handle); } else { - while(1){ + while (1) { taosWLockLatch(&pTq->lock); bool exec = tqIsHandleExec(pHandle); - if(exec){ - tqInfo("vgId:%d, topic:%s, subscription is executing, sub wait for 10ms and retry, pHandle:%p", pTq->pVnode->config.vgId, - pHandle->subKey, pHandle); + if (exec) { + tqInfo("vgId:%d, topic:%s, subscription is executing, sub wait for 10ms and retry, pHandle:%p", + pTq->pVnode->config.vgId, pHandle->subKey, pHandle); taosWUnLockLatch(&pTq->lock); taosMsleep(10); continue; @@ -713,7 +713,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d no switch consumer:0x%" PRIx64 " remains, because redo wal log", req.vgId, req.newConsumerId); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, - req.newConsumerId); + req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_store_32(&pHandle->epoch, 0); tqUnregisterPushHandle(pTq, pHandle); @@ -736,6 +736,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { tqDebug("s-task:0x%x start to expand task", pTask->id.taskId); int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, ver); + + pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr); + if (code != TSDB_CODE_SUCCESS) { return code; } @@ -859,8 +862,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } - tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", + tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.fillHistory, pTask->info.triggerParam); @@ -1004,7 +1007,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms bool restored = pTq->pVnode->restored; if (p != NULL && restored) { p->tsInfo.init = taosGetTimestampMs(); - tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->tsInfo.init); + tqDebug("s-task:%s set the init ts:%" PRId64, p->id.idStr, p->tsInfo.init); streamTaskCheckDownstream(p); } else if (!restored) { @@ -1443,7 +1446,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, return 0; } - int8_t status = pTask->status.taskStatus; + int8_t status = pTask->status.taskStatus; if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { // no lock needs to secure the access of the version if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { @@ -1622,12 +1625,13 @@ int32_t tqProcessStreamCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg) { // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. if (pTask->status.downstreamReady != 1) { qError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 - ", set it failure", pTask->id.idStr, req.checkpointId); + ", set it failure", + pTask->id.idStr, req.checkpointId); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } @@ -1784,7 +1788,6 @@ _end: tDecoderClear(&decoder); if (allStopped) { - if (!pTq->pVnode->restored) { tqDebug("vgId:%d vnode restore not completed, not restart the tasks", vgId); } else { @@ -1816,4 +1819,3 @@ _end: return rsp.code; } - diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index aa29ea8159..e516678941 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -220,6 +220,17 @@ int32_t streamMetaMayDoStateBackendConvert(SStreamMeta* pMeta) { return 0; } +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key) { + void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); + if (ppBackend != NULL && *ppBackend != NULL) { + // add ref later + return *ppBackend; + } + void* pBackend = streamStateOpenTaskBackend(pMeta->path, key); + taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); + + return pBackend; +} SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); @@ -277,11 +288,12 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->hbInfo.tickCounter = 0; pMeta->hbInfo.stopFlag = 0; + pMeta->pTaskBackendUnique = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + // start backend // taosInitRWLatch(&pMeta->chkpDirLock); - // pMeta->pTaskBackendUnique = - // taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); // pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t)); // pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t)); // pMeta->chkpCap = 8; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 837f787dd2..9a76f661f8 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -388,6 +388,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); + //pTask->pBackend = streamStateOpenTaskBackend(pMeta->path, (char*)pTask->id.idStr); + return TSDB_CODE_SUCCESS; } From 8890f5c950e4133b6d1169c300cae1c71499e479 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 22 Sep 2023 12:51:29 +0000 Subject: [PATCH 007/151] add cvt state --- source/libs/stream/src/streamMeta.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index e516678941..2a76f178a7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -140,7 +140,7 @@ enum STREAM_STATE_VER { STREAM_STATA_NEED_CONVERT, }; -int32_t streamMetaCheckStateCompatible(SStreamMeta* pMeta) { +int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { int8_t ret = STREAM_STATA_COMPATIBLE; TBC* pCur = NULL; @@ -180,7 +180,7 @@ int32_t streamMetaCheckStateCompatible(SStreamMeta* pMeta) { return ret; } -int32_t streamMetaDoStateBackendConvertImpl(SStreamMeta* pMeta) { +int32_t streamMetaConvertBackendFormat(SStreamMeta* pMeta) { int32_t code = 0; int64_t chkpId = streamGetLatestCheckpointId(pMeta); SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId); @@ -201,14 +201,14 @@ _EXIT: streamBackendCleanup((void*)pBackend); return code; } -int32_t streamMetaMayDoStateBackendConvert(SStreamMeta* pMeta) { - int8_t compatible = streamMetaCheckStateCompatible(pMeta); +int32_t streamMetaMayConvertBackendFormat(SStreamMeta* pMeta) { + int8_t compatible = streamMetaCheckBackendCompatible(pMeta); if (compatible == STREAM_STATA_COMPATIBLE) { return 0; } else if (compatible == STREAM_STATA_NEED_CONVERT) { qInfo("stream state need covert backend format"); - return streamMetaDoStateBackendConvertImpl(pMeta); + return streamMetaConvertBackendFormat(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { qError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " @@ -250,7 +250,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - if (streamMetaMayDoStateBackendConvert(pMeta) < 0) { + if (streamMetaMayConvertBackendFormat(pMeta) < 0) { goto _err; } From ed172a939cc08075887713b0662b9dccfb18e008 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 25 Sep 2023 18:09:35 +0800 Subject: [PATCH 008/151] rename cfg default name --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5df33949ef..fcdb945aae 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1540,7 +1540,7 @@ STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); return pTaskBackend; } -void streamMeteCloseTaskBackend(STaskBackendWrapper* wrapper) { +void streamStateCloseTaskBackend(STaskBackendWrapper* wrapper) { if (wrapper == NULL) return; rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); @@ -1673,7 +1673,7 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { } _EXIT: - streamMeteCloseTaskBackend(pTaskBackend); + streamStateCloseTaskBackend(pTaskBackend); return code; } From d4f87378fd491895cfe2ede6bb01dc853e51f7d4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 27 Sep 2023 14:44:09 +0800 Subject: [PATCH 009/151] avoid first tag index conflict --- source/dnode/snode/src/snode.c | 52 ++++++++++++---------- source/libs/stream/src/streamMeta.c | 4 +- source/libs/stream/src/streamState.c | 64 ++++++++++++++-------------- 3 files changed, 64 insertions(+), 56 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 2b1885fb0e..2dba51c150 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -50,7 +50,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { FAIL: if (pMsg->info.handle == NULL) return; - SRpcMsg rsp = { .code = code, .info = pMsg->info}; + SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -62,6 +62,8 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { if (code != TSDB_CODE_SUCCESS) { return code; } + pTask->pBackend = NULL; + ASSERT(0); streamTaskOpenAllUpstreamInput(pTask); @@ -73,8 +75,11 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { qDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } - int32_t numOfChildEp = taosArrayGetSize(pTask->pUpstreamInfoList); - SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState, .fillHistory = pTask->info.fillHistory }; + int32_t numOfChildEp = taosArrayGetSize(pTask->pUpstreamInfoList); + SReadHandle handle = {.vnode = NULL, + .numOfVgroups = numOfChildEp, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory}; initStreamStateAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0, pTask->id.taskId); @@ -83,23 +88,23 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); - SCheckpointInfo* pChkInfo = &pTask->chkInfo; + SCheckpointInfo *pChkInfo = &pTask->chkInfo; // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; - qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, + pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } else { if (pTask->chkInfo.nextProcessVer == -1) { pTask->chkInfo.nextProcessVer = 0; } } - qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", + qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), - pTask->info.fillHistory, pTask->info.triggerParam); + pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), + pTask->info.fillHistory, pTask->info.triggerParam); return 0; } @@ -179,8 +184,8 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); taosWUnLockLatch(&pSnode->pMeta->lock); - qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr, - streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks); + qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, + pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks); streamTaskCheckDownstream(pTask); return 0; @@ -190,7 +195,7 @@ int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; qDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); - SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); if (pTask == NULL) { qError("vgId:%d failed to acquire s-task:0x%x when dropping it", pSnode->pMeta->vgId, pReq->taskId); return 0; @@ -355,12 +360,13 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); streamMetaReleaseTask(pSnode->pMeta, pTask); - const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + const char *pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", - pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; - qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", + qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp status %d", taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } @@ -388,15 +394,15 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } -int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { - char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); +int32_t sndProcessStreamTaskCheckRsp(SSnode *pSnode, SRpcMsg *pMsg) { + char *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t code; SStreamTaskCheckRsp rsp; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)pReq, len); + tDecoderInit(&decoder, (uint8_t *)pReq, len); code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); if (code < 0) { @@ -405,13 +411,13 @@ int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { } tDecoderClear(&decoder); - qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); - SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { qError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, - pSnode->pMeta->vgId); + pSnode->pMeta->vgId); return -1; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2a76f178a7..99b3b9cca1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -221,14 +221,16 @@ int32_t streamMetaMayConvertBackendFormat(SStreamMeta* pMeta) { } void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key) { + taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { // add ref later + taosThreadMutexUnlock(&pMeta->backendMutex); return *ppBackend; } void* pBackend = streamStateOpenTaskBackend(pMeta->path, key); taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); - + taosThreadMutexLock(&pMeta->backendMutex); return pBackend; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 44c7b4f2e0..f51181b0b4 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -106,43 +106,43 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz } SStreamTask* pStreamTask = pTask; - char statePath[1024]; - if (!specPath) { - sprintf(statePath, "%s%s%d", path, TD_DIRSEP, pStreamTask->id.taskId); - } else { - memset(statePath, 0, 1024); - tstrncpy(statePath, path, 1024); - } + // char statePath[1024]; + // if (!specPath) { + // sprintf(statePath, "%s%s%d", path, TD_DIRSEP, pStreamTask->id.taskId); + // } else { + // memset(statePath, 0, 1024); + // tstrncpy(statePath, path, 1024); + // } pState->taskId = pStreamTask->id.taskId; pState->streamId = pStreamTask->id.streamId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); #ifdef USE_ROCKSDB - SStreamMeta* pMeta = pStreamTask->pMeta; - pState->streamBackendRid = pMeta->streamBackendRid; - // taosWLockLatch(&pMeta->lock); - taosThreadMutexLock(&pMeta->backendMutex); - void* uniqueId = - taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - if (uniqueId == NULL) { - int code = streamStateOpenBackend(pMeta->streamBackend, pState); - if (code == -1) { - taosThreadMutexUnlock(&pMeta->backendMutex); - taosMemoryFree(pState); - return NULL; - } - taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, - &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); - } else { - int64_t id = *(int64_t*)uniqueId; - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); - // already exist stream task for - qInfo("already exist stream-state for %s", pState->pTdbState->idstr); - // taosAcquireRef(streamBackendId, pState->streamBackendRid); - } - taosThreadMutexUnlock(&pMeta->backendMutex); + // SStreamMeta* pMeta = pStreamTask->pMeta; + // pState->streamBackendRid = pMeta->streamBackendRid; + // taosWLockLatch(&pMeta->lock); + // taosThreadMutexLock(&pMeta->backendMutex); + // void* uniqueId = + // taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + // if (uniqueId == NULL) { + // int code = streamStateOpenBackend(pMeta->streamBackend, pState); + // if (code == -1) { + // taosThreadMutexUnlock(&pMeta->backendMutex); + // taosMemoryFree(pState); + // return NULL; + // } + // taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, + // &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); + // } else { + // int64_t id = *(int64_t*)uniqueId; + // pState->pTdbState->backendCfWrapperId = id; + // pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); + // // already exist stream task for + // qInfo("already exist stream-state for %s", pState->pTdbState->idstr); + // // taosAcquireRef(streamBackendId, pState->streamBackendRid); + // } + // taosThreadMutexUnlock(&pMeta->backendMutex); pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; @@ -1125,7 +1125,7 @@ int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal void streamStateDestroy(SStreamState* pState, bool remove) { #ifdef USE_ROCKSDB streamFileStateDestroy(pState->pFileState); - streamStateDestroy_rocksdb(pState, remove); + // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong #endif From 54e3ac2c1ea26c274c22ad8531ee1b958e235e98 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Sep 2023 15:46:12 +0800 Subject: [PATCH 010/151] refact task backend --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tq.c | 6 +- source/libs/stream/inc/streamBackendRocksdb.h | 6 +- source/libs/stream/src/streamBackendRocksdb.c | 178 ++++++++++++------ source/libs/stream/src/streamMeta.c | 5 +- source/libs/stream/src/streamState.c | 10 +- source/libs/stream/src/streamTask.c | 2 +- 7 files changed, 136 insertions(+), 72 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e09ffb416c..553e8f8217 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -383,6 +383,7 @@ struct SStreamTask { struct SStreamMeta* pMeta; SSHashObj* pNameMap; void* pBackend; + int64_t backendRefId; char reserve[256]; }; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e136b9b265..12b89dbd9c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -736,12 +736,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { tqDebug("s-task:0x%x start to expand task", pTask->id.taskId); int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, ver); + if (code != TSDB_CODE_SUCCESS) return code; pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pTask->pBackend == NULL) return -1; streamTaskOpenAllUpstreamInput(pTask); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 3e07f2532e..a5533ef2f2 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -65,6 +65,8 @@ typedef struct { rocksdb_compactionfilterfactory_t* filterFactory; TdThreadMutex mutex; + int64_t refId; + } STaskBackendWrapper; void* streamBackendInit(const char* path, int64_t chkpId); @@ -75,7 +77,9 @@ int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointI SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); -STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key); +STaskBackendWrapper* taskBackendOpen(char* path, char* key); + +void taskBackendAddRef(void* pTaskBackend); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index fcdb945aae..eed65d3f3a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -629,7 +629,7 @@ void streamBackendHandleCleanup(void* arg) { taosThreadRwlockDestroy(&wrapper->rwLock); wrapper->rocksdb = NULL; - taosReleaseRef(streamBackendId, wrapper->backendId); + // taosReleaseRef(streamBackendId, wrapper->backendId); qDebug("end to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); taosMemoryFree(wrapper); @@ -776,7 +776,10 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* int64_t id = *(int64_t*)pIter; SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); - if (wrapper == NULL) continue; + if (wrapper == NULL) { + pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + continue; + } taosThreadRwlockRdlock(&wrapper->rwLock); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { @@ -788,7 +791,6 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* taosThreadRwlockUnlock(&wrapper->rwLock); taosArrayPush(refs, &id); - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); } int32_t nCf = taosArrayGetSize(pHandle); @@ -1452,35 +1454,72 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { taosMemoryFree(inst); } -STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { - int32_t code = 0; - - char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); - sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); - if (!taosDirExist(taskPath)) { - code = taosMkDir(taskPath); - if (code != 0) { - qError("failed to create dir: %s, reason:%s", taskPath, tstrerror(code)); - taosMemoryFree(taskPath); - return NULL; +int32_t getCfIdx(const char* cfName) { + int idx = -1; + size_t len = strlen(cfName); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (len == ginitDict[i].len && strncmp(cfName, ginitDict[i].key, strlen(cfName)) == 0) { + idx = i; + break; } } - char* err = NULL; + return idx; +} +int32_t taskBackendOpenCfs(STaskBackendWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { + int32_t code = -1; + char* err = NULL; - STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(SBackendWrapper)); - rocksdb_env_t* env = rocksdb_create_default_env(); + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + + for (int i = 0; i < nCf; i++) { + int32_t idx = getCfIdx(pCfNames[i]); + cfOpts[i] = pTask->pCfOpts[idx]; + } + + rocksdb_t* db = rocksdb_open_column_families(pTask->dbOpt, path, nCf, (const char* const*)pCfNames, + (const rocksdb_options_t* const*)cfOpts, cfHandle, &err); + + if (err != NULL) { + qError("failed to open cf path: %s", err); + taosMemoryFree(err); + goto _EXIT; + } + + for (int i = 0; i < nCf; i++) { + int32_t idx = getCfIdx(pCfNames[i]); + pTask->pCf[idx] = cfHandle[i]; + } + + pTask->db = db; + code = 0; + +_EXIT: + taosMemoryFree(cfOpts); + taosMemoryFree(cfHandle); + return code; +} +void taskBackendAddRef(void* pTaskBackend) { + STaskBackendWrapper* pBackend = pTaskBackend; + taosAcquireRef(streamBackendCfWrapperId, pBackend->refId); + return; +} +void taskBackendDestroy(STaskBackendWrapper* wrapper); + +void taskBackendInitOpt(STaskBackendWrapper* pTaskBackend) { + rocksdb_env_t* env = rocksdb_create_default_env(); rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); rocksdb_options_t* opts = rocksdb_options_create(); - // rocksdb_options_set_env(opts, env); + rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); // rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 3); rocksdb_options_set_info_log_level(opts, 1); - // rocksdb_options_set_db_write_buffer_size(opts, dbMemLimit); - // rocksdb_options_set_write_buffer_size(opts, dbMemLimit / 2); + rocksdb_options_set_db_write_buffer_size(opts, 64 << 20); + rocksdb_options_set_write_buffer_size(opts, 32 << 20); rocksdb_options_set_atomic_flush(opts, 1); pTaskBackend->dbOpt = opts; @@ -1519,46 +1558,86 @@ STaskBackendWrapper* streamStateOpenTaskBackend(char* path, char* key) { pTaskBackend->pCfOpts[i] = opt; pTaskBackend->pCfParams[i].tableOpt = tableOpt; } + return; +} +int32_t taskBackendBuildFullPath(char* path, char* key, char** fullPath) { + int32_t code = 0; + char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); + if (!taosDirExist(taskPath)) { + code = taosMkDir(taskPath); + if (code != 0) { + qError("failed to create dir: %s, reason:%s", taskPath, tstrerror(code)); + taosMemoryFree(taskPath); + return code; + } + } + *fullPath = taskPath; + return 0; +} +STaskBackendWrapper* taskBackendOpen(char* path, char* key) { + char* taskPath = NULL; + char* err = NULL; - char** cfs = rocksdb_list_column_families(opts, taskPath, &nCf, &err); + int32_t code = taskBackendBuildFullPath(path, key, &taskPath); + if (code != 0) return NULL; + + size_t nCf = 0; + + STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); + taskBackendInitOpt(pTaskBackend); + + char** cfs = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); if (nCf == 0 || nCf == 1 || err != NULL) { taosMemoryFreeClear(err); - pTaskBackend->db = rocksdb_open(opts, taskPath, &err); + pTaskBackend->db = rocksdb_open(pTaskBackend->dbOpt, taskPath, &err); if (err != NULL) { qError("failed to open rocksdb, path:%s, reason:%s", taskPath, err); taosMemoryFreeClear(err); + code = -1; + goto _EXIT; } } else { + code = taskBackendOpenCfs(pTaskBackend, taskPath, cfs, nCf); + if (code != 0) goto _EXIT; } - if (cfs != NULL) { - rocksdb_list_column_families_destroy(cfs, nCf); - } + if (cfs != NULL) rocksdb_list_column_families_destroy(cfs, nCf); taosThreadMutexInit(&pTaskBackend->mutex, NULL); taosMemoryFree(taskPath); qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); + + pTaskBackend->refId = taosAddRef(streamBackendCfWrapperId, pTaskBackend); + return pTaskBackend; +_EXIT: + + taskBackendDestroy(pTaskBackend); + return NULL; } -void streamStateCloseTaskBackend(STaskBackendWrapper* wrapper) { +void taskBackendDestroy(STaskBackendWrapper* wrapper) { if (wrapper == NULL) return; - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - rocksdb_flushoptions_set_wait(flushOpt, 1); + if (wrapper->db && wrapper->pCf) { + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flushOpt, 1); - int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); - char* err = NULL; - for (int i = 0; i < nCf; i++) { - if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); - if (err != NULL) { - qError("failed to flush cf:%s, reason:%s", ginitDict[i].key, err); - taosMemoryFreeClear(err); + int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); + char* err = NULL; + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); + if (err != NULL) { + qError("failed to flush cf:%s, reason:%s", ginitDict[i].key, err); + taosMemoryFreeClear(err); + } } rocksdb_flushoptions_destroy(flushOpt); - } - for (int i = 0; i < nCf; i++) { - if (wrapper->pCf[i] != NULL) { - rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + } } } rocksdb_options_destroy(wrapper->dbOpt); @@ -1574,24 +1653,13 @@ void streamStateCloseTaskBackend(STaskBackendWrapper* wrapper) { taosThreadMutexDestroy(&wrapper->mutex); - rocksdb_close(wrapper->db); + if (wrapper->db) rocksdb_close(wrapper->db); taosMemoryFree(wrapper); return; } -int8_t getCfIdx(const char* key) { - int idx = -1; - size_t len = strlen(key); - for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - if (len == ginitDict[i].len && strncmp(key, ginitDict[i].key, strlen(key)) == 0) { - idx = i; - break; - } - } - return idx; -} -int32_t taskBackendOpenCf(STaskBackendWrapper* pBackend, const char* key) { +int32_t taskBackendOpenCfByKey(STaskBackendWrapper* pBackend, const char* key) { int32_t code = 0; char* err = NULL; int8_t idx = getCfIdx(key); @@ -1658,14 +1726,14 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { int32_t code = 0; - STaskBackendWrapper* pTaskBackend = streamStateOpenTaskBackend(path, key); + STaskBackendWrapper* pTaskBackend = taskBackendOpen(path, key); RocksdbCfInst* pSrcBackend = pCfInst; for (int i = 0; i < nCf; i++) { rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; if (pSrcCf == NULL) continue; - code = taskBackendOpenCf(pTaskBackend, ginitDict[i].key); + code = taskBackendOpenCfByKey(pTaskBackend, ginitDict[i].key); if (code != 0) goto _EXIT; code = copyDataAt(pSrcBackend, pTaskBackend, i); @@ -1673,7 +1741,7 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { } _EXIT: - streamStateCloseTaskBackend(pTaskBackend); + taskBackendDestroy(pTaskBackend); return code; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 99b3b9cca1..2da4b2f370 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -224,11 +224,12 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key) { taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { - // add ref later + taskBackendAddRef(*ppBackend); taosThreadMutexUnlock(&pMeta->backendMutex); return *ppBackend; } - void* pBackend = streamStateOpenTaskBackend(pMeta->path, key); + void* pBackend = taskBackendOpen(pMeta->path, key); + taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexLock(&pMeta->backendMutex); return pBackend; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index f51181b0b4..d4ac1838d0 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -106,20 +106,12 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz } SStreamTask* pStreamTask = pTask; - // char statePath[1024]; - // if (!specPath) { - // sprintf(statePath, "%s%s%d", path, TD_DIRSEP, pStreamTask->id.taskId); - // } else { - // memset(statePath, 0, 1024); - // tstrncpy(statePath, path, 1024); - // } - pState->taskId = pStreamTask->id.taskId; pState->streamId = pStreamTask->id.streamId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); #ifdef USE_ROCKSDB - // SStreamMeta* pMeta = pStreamTask->pMeta; + SStreamMeta* pMeta = pStreamTask->pMeta; // pState->streamBackendRid = pMeta->streamBackendRid; // taosWLockLatch(&pMeta->lock); // taosThreadMutexLock(&pMeta->backendMutex); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9a76f661f8..e01f87788f 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -388,7 +388,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); - //pTask->pBackend = streamStateOpenTaskBackend(pMeta->path, (char*)pTask->id.idStr); + // pTask->pBackend = taskBackendOpen(pMeta->path, (char*)pTask->id.idStr); return TSDB_CODE_SUCCESS; } From bb265887e6545e4179e0053919f1b3b6b0d24f5b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Sep 2023 17:07:19 +0800 Subject: [PATCH 011/151] refact task backend --- source/dnode/vnode/src/tq/tq.c | 1 - source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/streamMeta.c | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 12b89dbd9c..e95114fcfb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -990,7 +990,6 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms if (code < 0) { tqError("vgId:%d failed to add s-task:0x%x, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); - tFreeStreamTask(pTask); return -1; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index eed65d3f3a..6384d85eb6 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1565,7 +1565,7 @@ int32_t taskBackendBuildFullPath(char* path, char* key, char** fullPath) { char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); if (!taosDirExist(taskPath)) { - code = taosMkDir(taskPath); + code = taosMulMkDir(taskPath); if (code != 0) { qError("failed to create dir: %s, reason:%s", taskPath, tstrerror(code)); taosMemoryFree(taskPath); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2da4b2f370..ad3ff82ec3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -229,9 +229,12 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key) { return *ppBackend; } void* pBackend = taskBackendOpen(pMeta->path, key); + if (pBackend == NULL) { + taosThreadMutexUnlock(&pMeta->backendMutex); + return NULL; + } taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); - taosThreadMutexLock(&pMeta->backendMutex); return pBackend; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { From c5bb37a5cbc1b50403603c6938f1cd6e52ced7c4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Sep 2023 17:30:49 +0800 Subject: [PATCH 012/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6384d85eb6..76b9b7474c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1604,9 +1604,9 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { if (cfs != NULL) rocksdb_list_column_families_destroy(cfs, nCf); taosThreadMutexInit(&pTaskBackend->mutex, NULL); - taosMemoryFree(taskPath); qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); + taosMemoryFree(taskPath); pTaskBackend->refId = taosAddRef(streamBackendCfWrapperId, pTaskBackend); From 4f66dc538fdb476c2fbf482d2801a038abf7ee57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Sep 2023 11:52:18 +0000 Subject: [PATCH 013/151] add cvt state --- source/libs/stream/inc/streamBackendRocksdb.h | 4 +- source/libs/stream/src/streamBackendRocksdb.c | 253 +++++++++--------- 2 files changed, 127 insertions(+), 130 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index a5533ef2f2..2c5eeb1fbe 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -64,8 +64,8 @@ typedef struct { rocksdb_compactionfilterfactory_t* filterFactory; TdThreadMutex mutex; - - int64_t refId; + char* idstr; + int64_t refId; } STaskBackendWrapper; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 76b9b7474c..80b385d072 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2000,26 +2000,25 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { } } if (pState != NULL && idx != -1) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_column_family_handle_t* cf = NULL; - taosThreadRwlockWrlock(&wrapper->rwLock); - cf = wrapper->pHandle[idx]; - if (cf == NULL) { - char buf[128] = {0}; - GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key); - char* err = NULL; - cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err); + taosThreadMutexLock(&wrapper->mutex); + + cf = wrapper->pCf[idx]; + if (cf == NULL) { + char* err = NULL; + cf = rocksdb_create_column_family(wrapper->db, wrapper->pCfOpts[idx], ginitDict[idx].key, &err); if (err != NULL) { idx = -1; qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { qDebug("succ to to open cf, %p %s_%s", pState, wrapper->idstr, funcName); - wrapper->pHandle[idx] = cf; + wrapper->pCf[idx] = cf; } } - taosThreadRwlockUnlock(&wrapper->rwLock); + taosThreadMutexUnlock(&wrapper->mutex); } return idx; @@ -2040,15 +2039,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe *readOpt = rocksdb_readoptions_create(); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (snapshot != NULL) { - *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->rocksdb); + *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->db); rocksdb_readoptions_set_snapshot(*readOpt, *snapshot); rocksdb_readoptions_set_fill_cache(*readOpt, 0); } - return rocksdb_create_iterator_cf(wrapper->rocksdb, *readOpt, - ((rocksdb_column_family_handle_t**)wrapper->pHandle)[idx]); + return rocksdb_create_iterator_cf(wrapper->db, *readOpt, ((rocksdb_column_family_handle_t**)wrapper->pCf)[idx]); } #define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ @@ -2062,13 +2060,13 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe code = -1; \ break; \ } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ char* ttlV = NULL; \ int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ @@ -2082,75 +2080,75 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_readoptions_t* opts = wrapper->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - qError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + qError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ } while (0); -#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ - code = -1; \ - break; \ - } \ - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->rocksdb; \ - rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ - rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ - if (err != NULL) { \ - qError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFree(err); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ - } \ +#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + code = -1; \ + break; \ + } \ + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ + rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ + if (err != NULL) { \ + qError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + } \ } while (0); // state cf @@ -2176,7 +2174,7 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -2186,10 +2184,9 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - if (wrapper->pHandle[1] != NULL) { + if (wrapper->pCf[1] != NULL) { char* err = NULL; - rocksdb_delete_range_cf(wrapper->rocksdb, wrapper->writeOpts, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, - &err); + rocksdb_delete_range_cf(wrapper->db, wrapper->writeOpt, wrapper->pCf[1], sKeyStr, sLen, eKeyStr, eLen, &err); if (err != NULL) { char toStringStart[128] = {0}; char toStringEnd[128] = {0}; @@ -2199,7 +2196,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); taosMemoryFree(err); } else { - rocksdb_compact_range_cf(wrapper->rocksdb, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + rocksdb_compact_range_cf(wrapper->db, wrapper->pCf[1], sKeyStr, sLen, eKeyStr, eLen); } } @@ -2300,9 +2297,9 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin if (pCur == NULL) { return NULL; } - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2356,7 +2353,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK if (pCur == NULL) return NULL; pCur->number = pState->number; - pCur->db = ((SBackendCfWrapper*)pState->pTdbState->pBackendCfWrapper)->rocksdb; + pCur->db = ((STaskBackendWrapper*)pState->pTdbState->pOwner->pBackend)->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2467,13 +2464,13 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2507,12 +2504,12 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2545,12 +2542,12 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2647,12 +2644,12 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (pCur == NULL) return NULL; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2707,13 +2704,13 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (!pCur) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2745,13 +2742,13 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2783,12 +2780,12 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const } int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { qDebug("streamStateSessionGetKeyByRange_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return -1; } - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -3015,7 +3012,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int code = 0; char* err = NULL; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); @@ -3048,16 +3045,16 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co } rocksdb_iter_next(pIter); } - rocksdb_release_snapshot(wrapper->rocksdb, snapshot); + rocksdb_release_snapshot(wrapper->db, snapshot); rocksdb_readoptions_destroy(readopts); rocksdb_iter_destroy(pIter); return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -3106,7 +3103,7 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -3120,7 +3117,7 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb char* ttlV = NULL; int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, ttl, &ttlV); - rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[i].idx]; + rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[i].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); taosMemoryFree(ttlV); @@ -3139,8 +3136,8 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb char* ttlV = tmpBuf; int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[cfIdx].idx]; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); if (tmpBuf == NULL) { @@ -3155,15 +3152,15 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb return 0; } int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { - char* err = NULL; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - rocksdb_write(wrapper->rocksdb, wrapper->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); + char* err = NULL; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { qError("streamState failed to write batch, err:%s", err); taosMemoryFree(err); return -1; } else { - qDebug("write batch to backend:%p", wrapper->pBackend); + qDebug("write batch to backend:%p", wrapper->db); } return 0; } From eac16b9d1b12e4cda30ad9da0f98c4ac066f4278 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Sep 2023 11:59:53 +0000 Subject: [PATCH 014/151] add cvt state --- source/libs/stream/src/streamBackendRocksdb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 80b385d072..ecf93c577e 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2374,12 +2374,12 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -2469,6 +2469,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta if (pCur == NULL) { return NULL; } + pCur->number = pState->number; pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, From b8e265e502ed6357acaaa59f674835bd3041e29c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 30 Sep 2023 09:05:28 +0000 Subject: [PATCH 015/151] add cvt state --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ecf93c577e..0d34970cb3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2065,8 +2065,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->db; \ rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ + rocksdb_t* db = wrapper->db; \ char* ttlV = NULL; \ int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ From fa702212382d2a3edb0abd9f8b1597191b5f0826 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 12:17:55 +0800 Subject: [PATCH 016/151] refact task backend --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/stream/inc/streamBackendRocksdb.h | 23 +- source/libs/stream/inc/streamInt.h | 12 +- source/libs/stream/src/streamBackendRocksdb.c | 381 +++++++++--------- source/libs/stream/src/streamMeta.c | 7 +- source/libs/stream/src/streamTask.c | 6 +- 7 files changed, 230 insertions(+), 203 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 553e8f8217..9bc4a91f1c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -726,7 +726,7 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key); +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e95114fcfb..7cfe776378 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -738,7 +738,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, ver); if (code != TSDB_CODE_SUCCESS) return code; - pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr); + pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr, &pTask->backendRefId); if (pTask->pBackend == NULL) return -1; streamTaskOpenAllUpstreamInput(pTask); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 2c5eeb1fbe..f5c303b809 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -69,17 +69,20 @@ typedef struct { } STaskBackendWrapper; -void* streamBackendInit(const char* path, int64_t chkpId); -void streamBackendCleanup(void* arg); -void streamBackendHandleCleanup(void* arg); -int32_t streamBackendLoadCheckpointInfo(void* pMeta); -int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); -SListNode* streamBackendAddCompare(void* backend, void* arg); -void streamBackendDelCompare(void* backend, void* arg); -int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); -STaskBackendWrapper* taskBackendOpen(char* path, char* key); +void* streamBackendInit(const char* path, int64_t chkpId); +void streamBackendCleanup(void* arg); +void streamBackendHandleCleanup(void* arg); +int32_t streamBackendLoadCheckpointInfo(void* pMeta); +int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); +SListNode* streamBackendAddCompare(void* backend, void* arg); +void streamBackendDelCompare(void* backend, void* arg); +int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); -void taskBackendAddRef(void* pTaskBackend); +STaskBackendWrapper* taskBackendOpen(char* path, char* key); +void taskBackendDestroy(void* pBackend); + +void* taskBackendAddRef(void* pTaskBackend); +void taskBackendRemoveRef(void* pTaskBackend); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index bbb7595e5a..54aefc8962 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -18,9 +18,9 @@ #include "executor.h" #include "query.h" -#include "tstream.h" #include "streamBackendRocksdb.h" #include "trpc.h" +#include "tstream.h" #ifdef __cplusplus extern "C" { @@ -41,8 +41,9 @@ typedef struct { } SStreamContinueExecInfo; extern SStreamGlobalEnv streamEnv; -extern int32_t streamBackendId; -extern int32_t streamBackendCfWrapperId; +extern int32_t streamBackendId; +extern int32_t streamBackendCfWrapperId; +extern int32_t taskBackendWrapperId; const char* streamGetBlockTypeStr(int32_t type); void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration); @@ -68,10 +69,11 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); -int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks); +int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks); SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); -int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen); +int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, + int32_t* pLen); int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq); int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 0d34970cb3..5589932fb8 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -770,39 +770,40 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { } int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { - SArray* pHandle = taosArrayInit(16, POINTER_BYTES); - void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); - while (pIter) { - int64_t id = *(int64_t*)pIter; + return 0; + // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); + // void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + // while (pIter) { + // int64_t id = *(int64_t*)pIter; - SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); - if (wrapper == NULL) { - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); - continue; - } + // SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); + // if (wrapper == NULL) { + // pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + // continue; + // } - taosThreadRwlockRdlock(&wrapper->rwLock); - for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - if (wrapper->pHandle[i]) { - rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; - taosArrayPush(pHandle, &p); - } - } - taosThreadRwlockUnlock(&wrapper->rwLock); + // taosThreadRwlockRdlock(&wrapper->rwLock); + // for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + // if (wrapper->pHandle[i]) { + // rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; + // taosArrayPush(pHandle, &p); + // } + // } + // taosThreadRwlockUnlock(&wrapper->rwLock); - taosArrayPush(refs, &id); - } + // taosArrayPush(refs, &id); + // } - int32_t nCf = taosArrayGetSize(pHandle); + // int32_t nCf = taosArrayGetSize(pHandle); - rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); - for (int i = 0; i < nCf; i++) { - ppCf[i] = taosArrayGetP(pHandle, i); - } - taosArrayDestroy(pHandle); + // rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + // for (int i = 0; i < nCf; i++) { + // ppCf[i] = taosArrayGetP(pHandle, i); + // } + // taosArrayDestroy(pHandle); - *ppHandle = ppCf; - return nCf; + // *ppHandle = ppCf; + // return nCf; } int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) { int32_t code = -1; @@ -826,172 +827,176 @@ _ERROR: return code; } int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32_t nCf) { - int code = 0; - char* err = NULL; + return 0; + // int code = 0; + // char* err = NULL; - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - rocksdb_flushoptions_set_wait(flushOpt, 1); + // rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + // rocksdb_flushoptions_set_wait(flushOpt, 1); - rocksdb_flush_cfs(db, flushOpt, cf, nCf, &err); - if (err != NULL) { - qError("failed to flush db before streamBackend clean up, reason:%s", err); - taosMemoryFree(err); - code = -1; - } - rocksdb_flushoptions_destroy(flushOpt); - return code; + // rocksdb_flush_cfs(db, flushOpt, cf, nCf, &err); + // if (err != NULL) { + // qError("failed to flush db before streamBackend clean up, reason:%s", err); + // taosMemoryFree(err); + // code = -1; + // } + // rocksdb_flushoptions_destroy(flushOpt); + // return code; } int32_t chkpPreCheckDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { - int32_t code = 0; - char* pChkpDir = taosMemoryCalloc(1, 256); - char* pChkpIdDir = taosMemoryCalloc(1, 256); - - sprintf(pChkpDir, "%s%s%s", path, TD_DIRSEP, "checkpoints"); - code = taosMulModeMkDir(pChkpDir, 0755, true); - if (code != 0) { - qError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code)); - taosMemoryFree(pChkpDir); - taosMemoryFree(pChkpIdDir); - code = -1; - return code; - } - - sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); - if (taosIsDir(pChkpIdDir)) { - qInfo("stream rm exist checkpoint%s", pChkpIdDir); - taosRemoveFile(pChkpIdDir); - } - *chkpDir = pChkpDir; - *chkpIdDir = pChkpIdDir; - return 0; + // int32_t code = 0; + // char* pChkpDir = taosMemoryCalloc(1, 256); + // char* pChkpIdDir = taosMemoryCalloc(1, 256); + + // sprintf(pChkpDir, "%s%s%s", path, TD_DIRSEP, "checkpoints"); + // code = taosMulModeMkDir(pChkpDir, 0755, true); + // if (code != 0) { + // qError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code)); + // taosMemoryFree(pChkpDir); + // taosMemoryFree(pChkpIdDir); + // code = -1; + // return code; + // } + + // sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); + // if (taosIsDir(pChkpIdDir)) { + // qInfo("stream rm exist checkpoint%s", pChkpIdDir); + // taosRemoveFile(pChkpIdDir); + // } + // *chkpDir = pChkpDir; + // *chkpIdDir = pChkpIdDir; + + // return 0; } int32_t streamBackendTriggerChkp(void* arg, char* dst) { - SStreamMeta* pMeta = arg; - int64_t backendRid = pMeta->streamBackendRid; - int32_t code = -1; + return 0; + // SStreamMeta* pMeta = arg; + // int64_t backendRid = pMeta->streamBackendRid; + // int32_t code = -1; - SArray* refs = taosArrayInit(16, sizeof(int64_t)); - rocksdb_column_family_handle_t** ppCf = NULL; + // SArray* refs = taosArrayInit(16, sizeof(int64_t)); + // rocksdb_column_family_handle_t** ppCf = NULL; - int64_t st = taosGetTimestampMs(); - SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); + // int64_t st = taosGetTimestampMs(); + // SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); - if (pHandle == NULL || pHandle->db == NULL) { - goto _ERROR; - } - int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, dst, nCf); + // if (pHandle == NULL || pHandle->db == NULL) { + // goto _ERROR; + // } + // int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); + // qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, dst, nCf); - code = chkpPreFlushDb(pHandle->db, ppCf, nCf); - if (code == 0) { - code = chkpDoDbCheckpoint(pHandle->db, dst); - if (code != 0) { - qError("stream backend:%p failed to do checkpoint at:%s", pHandle, dst); - } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, dst, - taosGetTimestampMs() - st); - } - } else { - qError("stream backend:%p failed to flush db at:%s", pHandle, dst); - } + // code = chkpPreFlushDb(pHandle->db, ppCf, nCf); + // if (code == 0) { + // code = chkpDoDbCheckpoint(pHandle->db, dst); + // if (code != 0) { + // qError("stream backend:%p failed to do checkpoint at:%s", pHandle, dst); + // } else { + // qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, dst, + // taosGetTimestampMs() - st); + // } + // } else { + // qError("stream backend:%p failed to flush db at:%s", pHandle, dst); + // } - // release all ref to cfWrapper; - for (int i = 0; i < taosArrayGetSize(refs); i++) { - int64_t id = *(int64_t*)taosArrayGet(refs, i); - taosReleaseRef(streamBackendCfWrapperId, id); - } + // // release all ref to cfWrapper; + // // for (int i = 0; i < taosArrayGetSize(refs); i++) { + // // int64_t id = *(int64_t*)taosArrayGet(refs, i); + // // taosReleaseRef(streamBackendCfWrapperId, id); + // // } -_ERROR: - taosReleaseRef(streamBackendId, backendRid); - taosArrayDestroy(refs); - return code; + // _ERROR: + // // taosReleaseRef(streamBackendId, backendRid); + // // taosArrayDestroy(refs); + // return code; } int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { - if (arg == NULL) return 0; + // if (arg == NULL) return 0; - SStreamMeta* pMeta = arg; - taosWLockLatch(&pMeta->chkpDirLock); - taosArrayPush(pMeta->chkpInUse, &chkpId); - taosWUnLockLatch(&pMeta->chkpDirLock); + // SStreamMeta* pMeta = arg; + // taosWLockLatch(&pMeta->chkpDirLock); + // taosArrayPush(pMeta->chkpInUse, &chkpId); + // taosWUnLockLatch(&pMeta->chkpDirLock); return 0; } int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { - if (arg == NULL) return 0; - - SStreamMeta* pMeta = arg; - taosWLockLatch(&pMeta->chkpDirLock); - if (taosArrayGetSize(pMeta->chkpInUse) > 0) { - int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0); - if (id == chkpId) { - taosArrayPopFrontBatch(pMeta->chkpInUse, 1); - } - } - taosWUnLockLatch(&pMeta->chkpDirLock); return 0; + // if (arg == NULL) return 0; + + // SStreamMeta* pMeta = arg; + // taosWLockLatch(&pMeta->chkpDirLock); + // if (taosArrayGetSize(pMeta->chkpInUse) > 0) { + // int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0); + // if (id == chkpId) { + // taosArrayPopFrontBatch(pMeta->chkpInUse, 1); + // } + // } + // taosWUnLockLatch(&pMeta->chkpDirLock); } int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) { - SStreamMeta* pMeta = arg; - int64_t backendRid = pMeta->streamBackendRid; - int64_t st = taosGetTimestampMs(); - int32_t code = -1; + return 0; + // SStreamMeta* pMeta = arg; + // int64_t backendRid = pMeta->streamBackendRid; + // int64_t st = taosGetTimestampMs(); + // int32_t code = -1; - SArray* refs = taosArrayInit(16, sizeof(int64_t)); + // SArray* refs = taosArrayInit(16, sizeof(int64_t)); - rocksdb_column_family_handle_t** ppCf = NULL; + // rocksdb_column_family_handle_t** ppCf = NULL; - char* pChkpDir = NULL; - char* pChkpIdDir = NULL; - if (chkpPreCheckDir(pMeta->path, checkpointId, &pChkpDir, &pChkpIdDir) != 0) { - taosArrayDestroy(refs); - return code; - } + // char* pChkpDir = NULL; + // char* pChkpIdDir = NULL; + // if (chkpPreCheckDir(pMeta->path, checkpointId, &pChkpDir, &pChkpIdDir) != 0) { + // taosArrayDestroy(refs); + // return code; + // } - SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); - if (pHandle == NULL || pHandle->db == NULL) { - goto _ERROR; - } + // SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); + // if (pHandle == NULL || pHandle->db == NULL) { + // goto _ERROR; + // } - // Get all cf and acquire cfWrappter - int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, pChkpIdDir, nCf); + // // Get all cf and acquire cfWrappter + // int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); + // qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, pChkpIdDir, nCf); - code = chkpPreFlushDb(pHandle->db, ppCf, nCf); - if (code == 0) { - code = chkpDoDbCheckpoint(pHandle->db, pChkpIdDir); - if (code != 0) { - qError("stream backend:%p failed to do checkpoint at:%s", pHandle, pChkpIdDir); - } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, pChkpIdDir, - taosGetTimestampMs() - st); - } - } else { - qError("stream backend:%p failed to flush db at:%s", pHandle, pChkpIdDir); - } - // release all ref to cfWrapper; - for (int i = 0; i < taosArrayGetSize(refs); i++) { - int64_t id = *(int64_t*)taosArrayGet(refs, i); - taosReleaseRef(streamBackendCfWrapperId, id); - } - if (code == 0) { - taosWLockLatch(&pMeta->chkpDirLock); - taosArrayPush(pMeta->chkpSaved, &checkpointId); - taosWUnLockLatch(&pMeta->chkpDirLock); + // code = chkpPreFlushDb(pHandle->db, ppCf, nCf); + // if (code == 0) { + // code = chkpDoDbCheckpoint(pHandle->db, pChkpIdDir); + // if (code != 0) { + // qError("stream backend:%p failed to do checkpoint at:%s", pHandle, pChkpIdDir); + // } else { + // qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, pChkpIdDir, + // taosGetTimestampMs() - st); + // } + // } else { + // qError("stream backend:%p failed to flush db at:%s", pHandle, pChkpIdDir); + // } + // // release all ref to cfWrapper; + // for (int i = 0; i < taosArrayGetSize(refs); i++) { + // int64_t id = *(int64_t*)taosArrayGet(refs, i); + // taosReleaseRef(streamBackendCfWrapperId, id); + // } + // if (code == 0) { + // taosWLockLatch(&pMeta->chkpDirLock); + // taosArrayPush(pMeta->chkpSaved, &checkpointId); + // taosWUnLockLatch(&pMeta->chkpDirLock); - // delete obsolte checkpoint - delObsoleteCheckpoint(arg, pChkpDir); - pMeta->chkpId = checkpointId; - } + // // delete obsolte checkpoint + // delObsoleteCheckpoint(arg, pChkpDir); + // pMeta->chkpId = checkpointId; + // } -_ERROR: - taosReleaseRef(streamBackendId, backendRid); - taosArrayDestroy(refs); - taosMemoryFree(ppCf); - taosMemoryFree(pChkpDir); - taosMemoryFree(pChkpIdDir); - return code; + // _ERROR: + // taosReleaseRef(streamBackendId, backendRid); + // taosArrayDestroy(refs); + // taosMemoryFree(ppCf); + // taosMemoryFree(pChkpDir); + // taosMemoryFree(pChkpIdDir); + // return code; } SListNode* streamBackendAddCompare(void* backend, void* arg) { @@ -1499,12 +1504,15 @@ _EXIT: taosMemoryFree(cfHandle); return code; } -void taskBackendAddRef(void* pTaskBackend) { +void* taskBackendAddRef(void* pTaskBackend) { STaskBackendWrapper* pBackend = pTaskBackend; - taosAcquireRef(streamBackendCfWrapperId, pBackend->refId); - return; + return taosAcquireRef(taskBackendWrapperId, pBackend->refId); } -void taskBackendDestroy(STaskBackendWrapper* wrapper); +void taskBackendRemoveRef(void* pTaskBackend) { + // STaskBackendWrapper* pBackend = pTaskBackend; + // taosReleaseRef(taskBackendWrapperId, pBackend->refId); +} +// void taskBackendDestroy(STaskBackendWrapper* wrapper); void taskBackendInitOpt(STaskBackendWrapper* pTaskBackend) { rocksdb_env_t* env = rocksdb_create_default_env(); @@ -1579,44 +1587,49 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { char* taskPath = NULL; char* err = NULL; - int32_t code = taskBackendBuildFullPath(path, key, &taskPath); - if (code != 0) return NULL; - + char** cfNames = NULL; size_t nCf = 0; + if (0 != taskBackendBuildFullPath(path, key, &taskPath)) return NULL; + STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); taskBackendInitOpt(pTaskBackend); - char** cfs = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); - if (nCf == 0 || nCf == 1 || err != NULL) { - taosMemoryFreeClear(err); - pTaskBackend->db = rocksdb_open(pTaskBackend->dbOpt, taskPath, &err); - if (err != NULL) { - qError("failed to open rocksdb, path:%s, reason:%s", taskPath, err); - taosMemoryFreeClear(err); - code = -1; - goto _EXIT; - } - } else { - code = taskBackendOpenCfs(pTaskBackend, taskPath, cfs, nCf); - if (code != 0) goto _EXIT; + cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); + if (nCf == 0) { + pTaskBackend->db = rocksdb_open(pTaskBackend->pCfOpts[0], taskPath, &err); + rocksdb_close(pTaskBackend->db); + + if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); + taosMemoryFree(err); + } + + cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); + ASSERT(err != NULL); + + if (0 != taskBackendOpenCfs(pTaskBackend, taskPath, cfNames, nCf)) { + goto _EXIT; } - if (cfs != NULL) rocksdb_list_column_families_destroy(cfs, nCf); taosThreadMutexInit(&pTaskBackend->mutex, NULL); qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); taosMemoryFree(taskPath); - pTaskBackend->refId = taosAddRef(streamBackendCfWrapperId, pTaskBackend); + if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); return pTaskBackend; _EXIT: taskBackendDestroy(pTaskBackend); + + if (err != NULL) taosMemoryFree(err); + if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); return NULL; } -void taskBackendDestroy(STaskBackendWrapper* wrapper) { +void taskBackendDestroy(void* pBackend) { + STaskBackendWrapper* wrapper = pBackend; + if (wrapper == NULL) return; if (wrapper->db && wrapper->pCf) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ad3ff82ec3..1892f58b9e 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -30,6 +30,7 @@ static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; int32_t streamMetaId = 0; +int32_t taskBackendWrapperId = 0; static int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta); static void metaHbToMnode(void* param, void* tmrId); @@ -52,6 +53,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); static void streamMetaEnvInit() { streamBackendId = taosOpenRef(64, streamBackendCleanup); streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); + taskBackendWrapperId = taosOpenRef(64, taskBackendDestroy); streamMetaId = taosOpenRef(64, streamMetaCloseImpl); @@ -220,19 +222,22 @@ int32_t streamMetaMayConvertBackendFormat(SStreamMeta* pMeta) { return 0; } -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key) { +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) { taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { taskBackendAddRef(*ppBackend); + *ref = ((STaskBackendWrapper*)*ppBackend)->refId; taosThreadMutexUnlock(&pMeta->backendMutex); return *ppBackend; } + void* pBackend = taskBackendOpen(pMeta->path, key); if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); return NULL; } + *ref = taosAddRef(taskBackendWrapperId, pBackend); taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); return pBackend; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index e01f87788f..152b426628 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -354,7 +354,11 @@ void tFreeStreamTask(SStreamTask* pTask) { taosArrayDestroyEx(pTask->pUpstreamInfoList, freeUpstreamItem); pTask->pUpstreamInfoList = NULL; } - + if (pTask->pBackend) { + taskBackendRemoveRef(pTask->pBackend); + + pTask->pBackend = NULL; + } taosThreadMutexDestroy(&pTask->lock); taosMemoryFree(pTask); From 82e024288904227e6c43800c3770ae475c860e18 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 13:43:52 +0800 Subject: [PATCH 017/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5589932fb8..49dada00d2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1618,9 +1618,10 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); + pTaskBackend->idstr = taosStrdup(key); return pTaskBackend; -_EXIT: +_EXIT: taskBackendDestroy(pTaskBackend); if (err != NULL) taosMemoryFree(err); @@ -1667,6 +1668,8 @@ void taskBackendDestroy(void* pBackend) { taosThreadMutexDestroy(&wrapper->mutex); if (wrapper->db) rocksdb_close(wrapper->db); + + taosMemoryFree(wrapper->idstr); taosMemoryFree(wrapper); return; @@ -2024,10 +2027,10 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { cf = rocksdb_create_column_family(wrapper->db, wrapper->pCfOpts[idx], ginitDict[idx].key, &err); if (err != NULL) { idx = -1; - qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); + qError("failed to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - qDebug("succ to to open cf, %p %s_%s", pState, wrapper->idstr, funcName); + qDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); wrapper->pCf[idx] = cf; } } From 66315fe4058332b1805221d1378cb88133719f0d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 13:53:44 +0800 Subject: [PATCH 018/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 49dada00d2..9e1a7ceeec 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1590,7 +1590,9 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { char** cfNames = NULL; size_t nCf = 0; - if (0 != taskBackendBuildFullPath(path, key, &taskPath)) return NULL; + if (taskBackendBuildFullPath(path, key, &taskPath) != 0) { + return NULL; + } STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); taskBackendInitOpt(pTaskBackend); @@ -1607,18 +1609,19 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); ASSERT(err != NULL); - if (0 != taskBackendOpenCfs(pTaskBackend, taskPath, cfNames, nCf)) { + if (taskBackendOpenCfs(pTaskBackend, taskPath, cfNames, nCf) != 0) { goto _EXIT; } - taosThreadMutexInit(&pTaskBackend->mutex, NULL); + if (cfNames != NULL) { + rocksdb_list_column_families_destroy(cfNames, nCf); + } qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); taosMemoryFree(taskPath); - if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); - pTaskBackend->idstr = taosStrdup(key); + taosThreadMutexInit(&pTaskBackend->mutex, NULL); return pTaskBackend; _EXIT: @@ -1628,6 +1631,7 @@ _EXIT: if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); return NULL; } + void taskBackendDestroy(void* pBackend) { STaskBackendWrapper* wrapper = pBackend; From c146327da843b4a4eb8ac361f2ef2865e2607838 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 16:48:36 +0800 Subject: [PATCH 019/151] refact task backend --- include/libs/stream/tstream.h | 2 +- source/libs/stream/inc/streamBackendRocksdb.h | 7 + source/libs/stream/src/streamBackendRocksdb.c | 257 ++++++++++++++---- 3 files changed, 213 insertions(+), 53 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9bc4a91f1c..a9e593eaad 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -419,9 +419,9 @@ typedef struct SStreamMeta { int64_t rid; int64_t chkpId; + int32_t chkpCap; SArray* chkpSaved; SArray* chkpInUse; - int32_t chkpCap; SRWLatch chkpDirLock; int32_t pauseTaskNum; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index f5c303b809..b79979df67 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -66,6 +66,13 @@ typedef struct { TdThreadMutex mutex; char* idstr; int64_t refId; + char* path; + + int64_t chkpId; + SArray* chkpSaved; + SArray* chkpInUse; + int32_t chkpCap; + TdThreadRwlock chkpDirLock; } STaskBackendWrapper; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 9e1a7ceeec..4edb166982 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -709,6 +709,60 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { taosArrayDestroy(chkpDel); return 0; } +/* + * checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--| + * chkpInUse: |--cp2--|--cp4--| + * chkpInUse is doing translation, cannot del until + * replication is finished + */ +int32_t chkpDelObsolete(void* arg, char* path) { + STaskBackendWrapper* pBackend = arg; + taosThreadRwlockWrlock(&pBackend->chkpDirLock); + + SArray* chkpDel = taosArrayInit(8, sizeof(int64_t)); + SArray* chkpDup = taosArrayInit(8, sizeof(int64_t)); + + int64_t firsId = 0; + if (taosArrayGetSize(pBackend->chkpInUse) >= 1) { + firsId = *(int64_t*)taosArrayGet(pBackend->chkpInUse, 0); + + for (int i = 0; i < taosArrayGetSize(pBackend->chkpSaved); i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + if (id >= firsId) { + taosArrayPush(chkpDup, &id); + } else { + taosArrayPush(chkpDel, &id); + } + } + } else { + int32_t sz = taosArrayGetSize(pBackend->chkpSaved); + int32_t dsz = sz - pBackend->chkpCap; // del size + + for (int i = 0; i < dsz; i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + taosArrayPush(chkpDel, &id); + } + for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) { + int64_t id = *(int64_t*)taosArrayGet(pBackend->chkpSaved, i); + taosArrayPush(chkpDup, &id); + } + } + taosArrayDestroy(pBackend->chkpSaved); + pBackend->chkpSaved = chkpDup; + + taosThreadRwlockUnlock(&pBackend->chkpDirLock); + + for (int i = 0; i < taosArrayGetSize(chkpDel); i++) { + int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); + char tbuf[256] = {0}; + sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); + if (taosIsDir(tbuf)) { + taosRemoveDir(tbuf); + } + } + taosArrayDestroy(chkpDel); + return 0; +} static int32_t compareCheckpoint(const void* a, const void* b) { int64_t x = *(int64_t*)a; @@ -805,6 +859,27 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* // *ppHandle = ppCf; // return nCf; } + +int32_t chkpGetAllDbCfHandle2(STaskBackendWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { + SArray* pHandle = taosArrayInit(8, POINTER_BYTES); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (pBackend->pCf[i]) { + rocksdb_column_family_handle_t* p = pBackend->pCf[i]; + taosArrayPush(pHandle, &p); + } + } + int32_t nCf = taosArrayGetSize(pHandle); + + rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + for (int i = 0; i < nCf; i++) { + ppCf[i] = taosArrayGetP(pHandle, i); + } + taosArrayDestroy(pHandle); + + *ppHandle = ppCf; + return nCf; +} + int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) { int32_t code = -1; char* err = NULL; @@ -827,47 +902,45 @@ _ERROR: return code; } int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32_t nCf) { - return 0; - // int code = 0; - // char* err = NULL; + int code = 0; + char* err = NULL; - // rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - // rocksdb_flushoptions_set_wait(flushOpt, 1); + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flushOpt, 1); - // rocksdb_flush_cfs(db, flushOpt, cf, nCf, &err); - // if (err != NULL) { - // qError("failed to flush db before streamBackend clean up, reason:%s", err); - // taosMemoryFree(err); - // code = -1; - // } - // rocksdb_flushoptions_destroy(flushOpt); - // return code; + rocksdb_flush_cfs(db, flushOpt, cf, nCf, &err); + if (err != NULL) { + qError("failed to flush db before streamBackend clean up, reason:%s", err); + taosMemoryFree(err); + code = -1; + } + rocksdb_flushoptions_destroy(flushOpt); + return code; } int32_t chkpPreCheckDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { + int32_t code = 0; + char* pChkpDir = taosMemoryCalloc(1, 256); + char* pChkpIdDir = taosMemoryCalloc(1, 256); + + sprintf(pChkpDir, "%s%s%s", path, TD_DIRSEP, "checkpoints"); + code = taosMulModeMkDir(pChkpDir, 0755, true); + if (code != 0) { + qError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code)); + taosMemoryFree(pChkpDir); + taosMemoryFree(pChkpIdDir); + code = -1; + return code; + } + + sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); + if (taosIsDir(pChkpIdDir)) { + qInfo("stream rm exist checkpoint%s", pChkpIdDir); + taosRemoveFile(pChkpIdDir); + } + *chkpDir = pChkpDir; + *chkpIdDir = pChkpIdDir; + return 0; - // int32_t code = 0; - // char* pChkpDir = taosMemoryCalloc(1, 256); - // char* pChkpIdDir = taosMemoryCalloc(1, 256); - - // sprintf(pChkpDir, "%s%s%s", path, TD_DIRSEP, "checkpoints"); - // code = taosMulModeMkDir(pChkpDir, 0755, true); - // if (code != 0) { - // qError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code)); - // taosMemoryFree(pChkpDir); - // taosMemoryFree(pChkpIdDir); - // code = -1; - // return code; - // } - - // sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); - // if (taosIsDir(pChkpIdDir)) { - // qInfo("stream rm exist checkpoint%s", pChkpIdDir); - // taosRemoveFile(pChkpIdDir); - // } - // *chkpDir = pChkpDir; - // *chkpIdDir = pChkpIdDir; - - // return 0; } int32_t streamBackendTriggerChkp(void* arg, char* dst) { @@ -936,6 +1009,49 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { // taosWUnLockLatch(&pMeta->chkpDirLock); } +int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { + STaskBackendWrapper* pBackend = arg; + int64_t st = taosGetTimestampMs(); + int32_t code = -1; + int64_t refId = pBackend->refId; + + if (taosAcquireRef(taskBackendWrapperId, refId) == NULL) { + return -1; + } + + char* pChkpDir = NULL; + char* pChkpIdDir = NULL; + if (chkpPreCheckDir(pBackend->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + goto _EXIT; + } + // Get all cf and acquire cfWrappter + rocksdb_column_family_handle_t** ppCf = NULL; + + int32_t nCf = chkpGetAllDbCfHandle2(pBackend, &ppCf); + qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pBackend, pChkpIdDir, nCf); + + if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0) { + if ((code = chkpDoDbCheckpoint(pBackend->db, pChkpIdDir)) != 0) { + qError("stream backend:%p failed to do checkpoint at:%s", pBackend, pChkpIdDir); + } else { + qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pBackend, pChkpIdDir, + taosGetTimestampMs() - st); + } + } else { + qError("stream backend:%p failed to flush db at:%s", pBackend, pChkpIdDir); + } + + code = chkpDelObsolete(pBackend, pChkpDir); + taosReleaseRef(taskBackendWrapperId, refId); + + return code; + +_EXIT: + taosMemoryFree(pChkpDir); + taosMemoryFree(pChkpIdDir); + taosReleaseRef(taskBackendWrapperId, refId); + return -1; +} int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) { return 0; // SStreamMeta* pMeta = arg; @@ -1514,7 +1630,7 @@ void taskBackendRemoveRef(void* pTaskBackend) { } // void taskBackendDestroy(STaskBackendWrapper* wrapper); -void taskBackendInitOpt(STaskBackendWrapper* pTaskBackend) { +void taskBackendInitDBOpt(STaskBackendWrapper* pTaskBackend) { rocksdb_env_t* env = rocksdb_create_default_env(); rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); @@ -1568,10 +1684,26 @@ void taskBackendInitOpt(STaskBackendWrapper* pTaskBackend) { } return; } -int32_t taskBackendBuildFullPath(char* path, char* key, char** fullPath) { +void taskBackendInitChkpOpt(STaskBackendWrapper* pBackend) { + pBackend->chkpId = -1; + pBackend->chkpCap = 4; + pBackend->chkpSaved = taosArrayInit(4, sizeof(int64_t)); + pBackend->chkpInUse = taosArrayInit(4, sizeof(int64_t)); + + taosThreadRwlockInit(&pBackend->chkpDirLock, NULL); +} + +void taskBackendDestroyChkpOpt(STaskBackendWrapper* pBackend) { + taosArrayDestroy(pBackend->chkpSaved); + taosArrayDestroy(pBackend->chkpInUse); + taosThreadRwlockDestroy(&pBackend->chkpDirLock); +} + +int32_t taskBackendBuildFullPath(char* path, char* key, char** dbFullPath, char** taskFullPath) { int32_t code = 0; - char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); - sprintf(taskPath, "%s%s%s%s%s", path, TD_DIRSEP, "state", TD_DIRSEP, key); + + char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(taskPath, "%s%s%s", path, TD_DIRSEP, key); if (!taosDirExist(taskPath)) { code = taosMulMkDir(taskPath); if (code != 0) { @@ -1580,36 +1712,53 @@ int32_t taskBackendBuildFullPath(char* path, char* key, char** fullPath) { return code; } } - *fullPath = taskPath; + + char* dbPath = taosMemoryCalloc(1, strlen(taskPath) + 128); + sprintf(dbPath, "%s%s%s", taskPath, TD_DIRSEP, "state"); + if (!taosDirExist(dbPath)) { + code = taosMulMkDir(dbPath); + if (code != 0) { + qError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); + taosMemoryFree(taskPath); + taosMemoryFree(dbPath); + return code; + } + } + + *dbFullPath = dbPath; + *taskFullPath = taskPath; return 0; } STaskBackendWrapper* taskBackendOpen(char* path, char* key) { - char* taskPath = NULL; - char* err = NULL; - + char* taskPath = NULL; + char* err = NULL; + char* dbPath = NULL; char** cfNames = NULL; size_t nCf = 0; - if (taskBackendBuildFullPath(path, key, &taskPath) != 0) { + if (taskBackendBuildFullPath(path, key, &dbPath, &taskPath) != 0) { return NULL; } STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); - taskBackendInitOpt(pTaskBackend); - cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); + pTaskBackend->path = taskPath; + taskBackendInitChkpOpt(pTaskBackend); + taskBackendInitDBOpt(pTaskBackend); + + cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { - pTaskBackend->db = rocksdb_open(pTaskBackend->pCfOpts[0], taskPath, &err); + pTaskBackend->db = rocksdb_open(pTaskBackend->pCfOpts[0], dbPath, &err); rocksdb_close(pTaskBackend->db); if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); taosMemoryFree(err); } - cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, taskPath, &nCf, &err); + cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); ASSERT(err != NULL); - if (taskBackendOpenCfs(pTaskBackend, taskPath, cfNames, nCf) != 0) { + if (taskBackendOpenCfs(pTaskBackend, dbPath, cfNames, nCf) != 0) { goto _EXIT; } @@ -1617,11 +1766,12 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { rocksdb_list_column_families_destroy(cfNames, nCf); } - qDebug("succ to init stream backend at %s, backend:%p", taskPath, pTaskBackend); - taosMemoryFree(taskPath); + qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskBackend); + taosMemoryFree(dbPath); pTaskBackend->idstr = taosStrdup(key); taosThreadMutexInit(&pTaskBackend->mutex, NULL); + return pTaskBackend; _EXIT: @@ -1673,7 +1823,10 @@ void taskBackendDestroy(void* pBackend) { if (wrapper->db) rocksdb_close(wrapper->db); + taskBackendDestroyChkpOpt(pBackend); + taosMemoryFree(wrapper->idstr); + taosMemoryFree(wrapper->path); taosMemoryFree(wrapper); return; From 3ebc7eef5f32193398b5afc51137a9d2a52b488a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 17:03:12 +0800 Subject: [PATCH 020/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4edb166982..07249862fb 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -715,9 +715,10 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { * chkpInUse is doing translation, cannot del until * replication is finished */ -int32_t chkpDelObsolete(void* arg, char* path) { +int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { STaskBackendWrapper* pBackend = arg; taosThreadRwlockWrlock(&pBackend->chkpDirLock); + taosArrayPush(pBackend->chkpSaved, &chkpId); SArray* chkpDel = taosArrayInit(8, sizeof(int64_t)); SArray* chkpDup = taosArrayInit(8, sizeof(int64_t)); @@ -1041,9 +1042,9 @@ int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { qError("stream backend:%p failed to flush db at:%s", pBackend, pChkpIdDir); } - code = chkpDelObsolete(pBackend, pChkpDir); - taosReleaseRef(taskBackendWrapperId, refId); + code = chkpMayDelObsolete(pBackend, chkpId, pChkpDir); + taosReleaseRef(taskBackendWrapperId, refId); return code; _EXIT: From e41da13d5bdde89f84299281d8c9dfe6f7022e4a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 17:07:26 +0800 Subject: [PATCH 021/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 63 +------------------ source/libs/stream/src/streamCheckpoint.c | 24 +++---- 2 files changed, 14 insertions(+), 73 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 07249862fb..97d6f0634a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1053,68 +1053,7 @@ _EXIT: taosReleaseRef(taskBackendWrapperId, refId); return -1; } -int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) { - return 0; - // SStreamMeta* pMeta = arg; - // int64_t backendRid = pMeta->streamBackendRid; - // int64_t st = taosGetTimestampMs(); - // int32_t code = -1; - - // SArray* refs = taosArrayInit(16, sizeof(int64_t)); - - // rocksdb_column_family_handle_t** ppCf = NULL; - - // char* pChkpDir = NULL; - // char* pChkpIdDir = NULL; - // if (chkpPreCheckDir(pMeta->path, checkpointId, &pChkpDir, &pChkpIdDir) != 0) { - // taosArrayDestroy(refs); - // return code; - // } - - // SBackendWrapper* pHandle = taosAcquireRef(streamBackendId, backendRid); - // if (pHandle == NULL || pHandle->db == NULL) { - // goto _ERROR; - // } - - // // Get all cf and acquire cfWrappter - // int32_t nCf = chkpGetAllDbCfHandle(pMeta, &ppCf, refs); - // qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pHandle, pChkpIdDir, nCf); - - // code = chkpPreFlushDb(pHandle->db, ppCf, nCf); - // if (code == 0) { - // code = chkpDoDbCheckpoint(pHandle->db, pChkpIdDir); - // if (code != 0) { - // qError("stream backend:%p failed to do checkpoint at:%s", pHandle, pChkpIdDir); - // } else { - // qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, pChkpIdDir, - // taosGetTimestampMs() - st); - // } - // } else { - // qError("stream backend:%p failed to flush db at:%s", pHandle, pChkpIdDir); - // } - // // release all ref to cfWrapper; - // for (int i = 0; i < taosArrayGetSize(refs); i++) { - // int64_t id = *(int64_t*)taosArrayGet(refs, i); - // taosReleaseRef(streamBackendCfWrapperId, id); - // } - // if (code == 0) { - // taosWLockLatch(&pMeta->chkpDirLock); - // taosArrayPush(pMeta->chkpSaved, &checkpointId); - // taosWUnLockLatch(&pMeta->chkpDirLock); - - // // delete obsolte checkpoint - // delObsoleteCheckpoint(arg, pChkpDir); - // pMeta->chkpId = checkpointId; - // } - - // _ERROR: - // taosReleaseRef(streamBackendId, backendRid); - // taosArrayDestroy(refs); - // taosMemoryFree(ppCf); - // taosMemoryFree(pChkpDir); - // taosMemoryFree(pChkpIdDir); - // return code; -} +int32_t streamBackendDoCheckpoint(void* arg, uint64_t chkpId) { return taskBackendDoCheckpoint(arg, chkpId); } SListNode* streamBackendAddCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index cc93d25fd5..63e3b94561 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -120,7 +120,7 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint pBlock->info.rows = 1; pBlock->info.childId = pTask->info.selfChildId; - pChkpoint->blocks = taosArrayInit(4, sizeof(SSDataBlock));//pBlock; + pChkpoint->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock; taosArrayPush(pChkpoint->blocks, pBlock); taosMemoryFree(pBlock); @@ -166,10 +166,10 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); - int64_t checkpointId = pDataBlock->info.version; + int64_t checkpointId = pDataBlock->info.version; const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; // set the task status pTask->checkpointingId = checkpointId; @@ -177,7 +177,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc // set task status pTask->status.taskStatus = TASK_STATUS__CK; - { // todo: remove this when the pipeline checkpoint generating is used. + { // todo: remove this when the pipeline checkpoint generating is used. SStreamMeta* pMeta = pTask->pMeta; taosWLockLatch(&pMeta->lock); @@ -189,10 +189,11 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc taosWUnLockLatch(&pMeta->lock); } - //todo fix race condition: set the status and append checkpoint block + // todo fix race condition: set the status and append checkpoint block int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { - if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || + pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { qDebug("s-task:%s set childIdx:%d, and add checkpoint block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info @@ -223,7 +224,8 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc } else { qDebug( "s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, dispatch checkpoint msg " - "downstream", id, num); + "downstream", + id, num); // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure @@ -288,11 +290,11 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { // save the task streamMetaSaveTask(pMeta, p); - streamTaskOpenAllUpstreamInput(p); // open inputQ for all upstream tasks + streamTaskOpenAllUpstreamInput(p); // open inputQ for all upstream tasks qDebug("vgId:%d s-task:%s level:%d commit task status after checkpoint completed, checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, - streamGetTaskStatusStr(prev)); + pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, + p->chkInfo.nextProcessVer, streamGetTaskStatusStr(prev)); } if (streamMetaCommit(pMeta) < 0) { @@ -320,7 +322,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { qDebug("s-task:%s is ready for checkpoint", pTask->id.idStr); pMeta->totalTasks = 0; - streamBackendDoCheckpoint(pMeta, pTask->checkpointingId); + streamBackendDoCheckpoint(pTask->pBackend, pTask->checkpointingId); streamSaveAllTaskStatus(pMeta, pTask->checkpointingId); qDebug("vgId:%d vnode wide checkpoint completed, save all tasks status, checkpointId:%" PRId64, pMeta->vgId, pTask->checkpointingId); From bdc4afec8323e251e3606f189c2efdd19d81f476 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 17:17:30 +0800 Subject: [PATCH 022/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 28 +++++++++---------- source/libs/stream/src/streamMeta.c | 2 ++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 97d6f0634a..539844a4e8 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1639,50 +1639,50 @@ void taskBackendDestroyChkpOpt(STaskBackendWrapper* pBackend) { taosThreadRwlockDestroy(&pBackend->chkpDirLock); } -int32_t taskBackendBuildFullPath(char* path, char* key, char** dbFullPath, char** taskFullPath) { +int32_t taskBackendBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { int32_t code = 0; - char* taskPath = taosMemoryCalloc(1, strlen(path) + 128); - sprintf(taskPath, "%s%s%s", path, TD_DIRSEP, key); - if (!taosDirExist(taskPath)) { - code = taosMulMkDir(taskPath); + char* statePath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(statePath, "%s%s%s", path, TD_DIRSEP, key); + if (!taosDirExist(statePath)) { + code = taosMulMkDir(statePath); if (code != 0) { - qError("failed to create dir: %s, reason:%s", taskPath, tstrerror(code)); - taosMemoryFree(taskPath); + qError("failed to create dir: %s, reason:%s", statePath, tstrerror(code)); + taosMemoryFree(statePath); return code; } } - char* dbPath = taosMemoryCalloc(1, strlen(taskPath) + 128); - sprintf(dbPath, "%s%s%s", taskPath, TD_DIRSEP, "state"); + char* dbPath = taosMemoryCalloc(1, strlen(statePath) + 128); + sprintf(dbPath, "%s%s%s", statePath, TD_DIRSEP, "state"); if (!taosDirExist(dbPath)) { code = taosMulMkDir(dbPath); if (code != 0) { qError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); - taosMemoryFree(taskPath); + taosMemoryFree(statePath); taosMemoryFree(dbPath); return code; } } *dbFullPath = dbPath; - *taskFullPath = taskPath; + *stateFullPath = statePath; return 0; } STaskBackendWrapper* taskBackendOpen(char* path, char* key) { - char* taskPath = NULL; + char* statePath = NULL; char* err = NULL; char* dbPath = NULL; char** cfNames = NULL; size_t nCf = 0; - if (taskBackendBuildFullPath(path, key, &dbPath, &taskPath) != 0) { + if (taskBackendBuildFullPath(path, key, &dbPath, &statePath) != 0) { return NULL; } STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); - pTaskBackend->path = taskPath; + pTaskBackend->path = statePath; taskBackendInitChkpOpt(pTaskBackend); taskBackendInitDBOpt(pTaskBackend); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 1892f58b9e..8cce4dc2f9 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -237,9 +237,11 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) taosThreadMutexUnlock(&pMeta->backendMutex); return NULL; } + *ref = taosAddRef(taskBackendWrapperId, pBackend); taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); + taosThreadMutexUnlock(&pMeta->backendMutex); return pBackend; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { From 98099ebbae54151c24fad76912d3ec7f50806673 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Oct 2023 18:41:27 +0800 Subject: [PATCH 023/151] fix mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 539844a4e8..19dfe3cf8d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -870,6 +870,10 @@ int32_t chkpGetAllDbCfHandle2(STaskBackendWrapper* pBackend, rocksdb_column_fami } } int32_t nCf = taosArrayGetSize(pHandle); + if (nCf == 0) { + taosArrayDestroy(pHandle); + return nCf; + } rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); for (int i = 0; i < nCf; i++) { @@ -1031,14 +1035,14 @@ int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { int32_t nCf = chkpGetAllDbCfHandle2(pBackend, &ppCf); qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pBackend, pChkpIdDir, nCf); - if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0) { + if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0 && nCf != 0) { if ((code = chkpDoDbCheckpoint(pBackend->db, pChkpIdDir)) != 0) { qError("stream backend:%p failed to do checkpoint at:%s", pBackend, pChkpIdDir); } else { qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pBackend, pChkpIdDir, taosGetTimestampMs() - st); } - } else { + } else if (nCf != 0) { qError("stream backend:%p failed to flush db at:%s", pBackend, pChkpIdDir); } From 2fa91341a7c4a0d79cbae3248db25319335f9583 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Oct 2023 10:35:49 +0800 Subject: [PATCH 024/151] refact task backend --- source/libs/stream/src/streamBackendRocksdb.c | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 19dfe3cf8d..775eaebf89 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1685,22 +1685,27 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { } STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); - + pTaskBackend->idstr = taosStrdup(key); pTaskBackend->path = statePath; + taosThreadMutexInit(&pTaskBackend->mutex, NULL); taskBackendInitChkpOpt(pTaskBackend); taskBackendInitDBOpt(pTaskBackend); + statePath = NULL; cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { + // pre create db pTaskBackend->db = rocksdb_open(pTaskBackend->pCfOpts[0], dbPath, &err); rocksdb_close(pTaskBackend->db); - if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); + if (cfNames != NULL) { + rocksdb_list_column_families_destroy(cfNames, nCf); + } taosMemoryFree(err); - } - cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); - ASSERT(err != NULL); + cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); + ASSERT(err != NULL); + } if (taskBackendOpenCfs(pTaskBackend, dbPath, cfNames, nCf) != 0) { goto _EXIT; @@ -1713,16 +1718,15 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskBackend); taosMemoryFree(dbPath); - pTaskBackend->idstr = taosStrdup(key); - taosThreadMutexInit(&pTaskBackend->mutex, NULL); - return pTaskBackend; _EXIT: taskBackendDestroy(pTaskBackend); - if (err != NULL) taosMemoryFree(err); - if (cfNames != NULL) rocksdb_list_column_families_destroy(cfNames, nCf); + if (err) taosMemoryFree(err); + if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); + if (dbPath) taosMemoryFree(dbPath); + if (statePath) taosMemoryFree(statePath); return NULL; } From f49a2cbeb058c92fd6c5eb4a579aab9277e0f932 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Oct 2023 18:14:17 +0800 Subject: [PATCH 025/151] refact task backend --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 6 ++---- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 1 + source/libs/stream/src/streamBackendRocksdb.c | 5 +++-- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamSnapshot.c | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 4a1b3961cd..53f9cc3c92 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -168,10 +168,8 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) } int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); - int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta, chkpId); - if (code == 0) { - code = streamStateLoadTasks(pWriter); - } + // int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta, chkpId); + int32_t code = streamStateLoadTasks(pWriter); tqDebug("vgId:%d, vnode %s succ to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); taosMemoryFree(pWriter); return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f19068ea88..35c8a4102e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -256,6 +256,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (code) { pReader->streamStateDone = 1; pReader->pStreamStateReader = NULL; + code = 0; goto _err; } } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 775eaebf89..ba05d9968b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -922,7 +922,8 @@ int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32 rocksdb_flushoptions_destroy(flushOpt); return code; } -int32_t chkpPreCheckDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { + +int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpIdDir) { int32_t code = 0; char* pChkpDir = taosMemoryCalloc(1, 256); char* pChkpIdDir = taosMemoryCalloc(1, 256); @@ -1026,7 +1027,7 @@ int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { char* pChkpDir = NULL; char* pChkpIdDir = NULL; - if (chkpPreCheckDir(pBackend->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + if (chkpPreBuildDir(pBackend->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { goto _EXIT; } // Get all cf and acquire cfWrappter diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8cce4dc2f9..72399865e6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -806,7 +806,7 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - // do duplicate task check. + // do duplicate task checke int64_t keys[2] = {pTask->id.streamId, pTask->id.taskId}; void* p = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); if (p == NULL) { diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 8a4500dd86..42d77fc4c0 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -270,7 +270,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { if (handle->checkpointId == 0) { // del tmp dir - if (taosIsDir(pFile->path)) { + if (pFile && taosIsDir(pFile->path)) { taosRemoveDir(pFile->path); } } else { From bffa6387fe09d623fb2a4a8664858afcdf472634 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Oct 2023 18:16:45 +0800 Subject: [PATCH 026/151] refact task backend --- source/dnode/vnode/src/tq/tq.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7cfe776378..3546df7e91 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1792,12 +1792,12 @@ _end: taosWLockLatch(&pMeta->lock); terrno = 0; - int32_t code = streamMetaReopen(pMeta, 0); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - taosWUnLockLatch(&pMeta->lock); - return -1; - } + // int32_t code = streamMetaReopen(pMeta, 0); + // if (code != 0) { + // tqError("vgId:%d failed to reopen stream meta", vgId); + // taosWUnLockLatch(&pMeta->lock); + // return -1; + // } if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { tqError("vgId:%d failed to load stream tasks", vgId); From 0b92a6b1eaf0013469a0d99b1cd465ec70c49394 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 9 Oct 2023 11:16:16 +0800 Subject: [PATCH 027/151] fix mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ba05d9968b..f936b569ae 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -717,7 +717,9 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { */ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { STaskBackendWrapper* pBackend = arg; + taosThreadRwlockWrlock(&pBackend->chkpDirLock); + taosArrayPush(pBackend->chkpSaved, &chkpId); SArray* chkpDel = taosArrayInit(8, sizeof(int64_t)); @@ -1028,6 +1030,7 @@ int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { char* pChkpDir = NULL; char* pChkpIdDir = NULL; if (chkpPreBuildDir(pBackend->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + code = -1; goto _EXIT; } // Get all cf and acquire cfWrappter @@ -1036,27 +1039,24 @@ int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { int32_t nCf = chkpGetAllDbCfHandle2(pBackend, &ppCf); qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pBackend, pChkpIdDir, nCf); - if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0 && nCf != 0) { + if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0) { if ((code = chkpDoDbCheckpoint(pBackend->db, pChkpIdDir)) != 0) { qError("stream backend:%p failed to do checkpoint at:%s", pBackend, pChkpIdDir); } else { qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pBackend, pChkpIdDir, taosGetTimestampMs() - st); } - } else if (nCf != 0) { + } else { qError("stream backend:%p failed to flush db at:%s", pBackend, pChkpIdDir); } code = chkpMayDelObsolete(pBackend, chkpId, pChkpDir); - taosReleaseRef(taskBackendWrapperId, refId); - return code; - _EXIT: taosMemoryFree(pChkpDir); taosMemoryFree(pChkpIdDir); taosReleaseRef(taskBackendWrapperId, refId); - return -1; + return code; } int32_t streamBackendDoCheckpoint(void* arg, uint64_t chkpId) { return taskBackendDoCheckpoint(arg, chkpId); } From addb1107d264228308b150e276f70072c0f0631b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 9 Oct 2023 15:11:59 +0800 Subject: [PATCH 028/151] fix mem leak --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 35c8a4102e..f19068ea88 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -256,7 +256,6 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (code) { pReader->streamStateDone = 1; pReader->pStreamStateReader = NULL; - code = 0; goto _err; } } From 2aa88dfe9b9bbe06a7ad595b486abba8de310242 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 9 Oct 2023 15:53:42 +0800 Subject: [PATCH 029/151] fix mem leak --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 1 + source/libs/stream/inc/streamBackendRocksdb.h | 2 + source/libs/stream/src/streamBackendRocksdb.c | 9 ++- source/libs/stream/src/streamSnapshot.c | 77 +++++++++++-------- 4 files changed, 56 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 53f9cc3c92..38cd1e1b76 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -104,6 +104,7 @@ int32_t streamStateSnapRead(SStreamStateReader* pReader, uint8_t** ppData) { pHdr->type = SNAP_DATA_STREAM_STATE_BACKEND; pHdr->size = len; memcpy(pHdr->data, rowData, len); + taosMemoryFree(rowData); tqDebug("vgId:%d, vnode stream-state snapshot read data success", TD_VID(pReader->pTq->pVnode)); return code; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index b79979df67..c8d5797dad 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -181,5 +181,7 @@ int32_t streamBackendTriggerChkp(void* pMeta, char* dst); int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); +int32_t taskBackendBuildSnap(void* arg, int64_t chkpId); + // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f936b569ae..5e48a345b1 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -950,7 +950,14 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return 0; } - +int32_t taskBackendBuildSnap(void* arg, int64_t chkpId) { + SStreamMeta* pMeta = arg; + void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + while (pIter) { + pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + } + return 0; +} int32_t streamBackendTriggerChkp(void* arg, char* dst) { return 0; // SStreamMeta* pMeta = arg; diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 42d77fc4c0..88790f2511 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -108,44 +108,57 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { - // impl later - int len = strlen(path); - char* tdir = taosMemoryCalloc(1, len + 256); - memcpy(tdir, path, len); +int32_t streamStateSnapBuild(void* arg, char* path, int64_t chkpId) { + return taskBackendBuildSnap(arg, chkpId); + // int32_t code = 0; + // int8_t validChkp = 0; - int32_t code = 0; + // int len = strlen(path); + // char* tpath = taosMemoryCalloc(1, len + 256); + // memcpy(tpath, path, len); - int8_t validChkp = 0; - if (chkpId != 0) { - sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", TD_DIRSEP, - chkpId); - if (taosIsDir(tdir)) { - validChkp = 1; - qInfo("%s start to read snap %s", STREAM_STATE_TRANSFER, tdir); - streamBackendAddInUseChkp(pMeta, chkpId); - } else { - qWarn("%s failed to read from %s, reason: dir not exist,retry to default state dir", STREAM_STATE_TRANSFER, tdir); - } - } + // SStreamMeta *pMeta = arg; + // if (chkpId != 0) { + // sprintf(tpath, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", + // TD_DIRSEP, + // chkpId); + // if (taosIsDir(tpath)) { + // validChkp = 1; + // qInfo("%s start to read snap %s", STREAM_STATE_TRANSFER, tpath); + // streamBackendAddInUseChkp(pMeta, chkpId); + // } else { + // qWarn("%s failed to read from %s, reason: dir not exist,retry to default state dir", STREAM_STATE_TRANSFER, + // tpath); + // } + // } // no checkpoint specified or not exists invalid checkpoint, do checkpoint at default path and translate it - if (validChkp == 0) { - sprintf(tdir, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state"); - char* chkpdir = taosMemoryCalloc(1, len + 256); - sprintf(chkpdir, "%s%s%s", tdir, TD_DIRSEP, "tmp"); - taosMemoryFree(tdir); + // if (validChkp == 0) { + // sprintf(tpath, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state"); + // char* chkpdir = taosMemoryCalloc(1, len + 256); + // sprintf(chkpdir, "%s%s%s", tpath, TD_DIRSEP, "tmp"); + // taosMemoryFree(tpath); - tdir = chkpdir; - qInfo("%s start to trigger checkpoint on %s", STREAM_STATE_TRANSFER, tdir); + // tpath = chkpdir; + // qInfo("%s start to trigger checkpoint on %s", STREAM_STATE_TRANSFER, tpath); - code = streamBackendTriggerChkp(pMeta, tdir); - if (code != 0) { - qError("%s failed to trigger chekckpoint at %s", STREAM_STATE_TRANSFER, tdir); - taosMemoryFree(tdir); - return code; - } - chkpId = 0; + // code = streamBackendTriggerChkp(arg, tpath); + // if (code != 0) { + // qError("%s failed to trigger chekckpoint at %s", STREAM_STATE_TRANSFER, tpath); + // taosMemoryFree(tpath); + // return code; + // } + // chkpId = 0; + // } + + //*dstPath = tpath; +} +int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { + // impl later + char* tdir = NULL; + int32_t code = streamStateSnapBuild(pMeta, path, chkpId); + if (code != 0) { + return -1; } qInfo("%s start to read dir: %s", STREAM_STATE_TRANSFER, tdir); From 28421cd35e37a0423fe69ce2cd147fe83ad4d72b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Oct 2023 20:51:41 +0800 Subject: [PATCH 030/151] refactor backend --- source/libs/stream/inc/streamBackendRocksdb.h | 3 +- source/libs/stream/inc/streamInt.h | 5 + source/libs/stream/src/streamBackendRocksdb.c | 19 +- source/libs/stream/src/streamSnapshot.c | 339 ++++++++++-------- 4 files changed, 204 insertions(+), 162 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index c8d5797dad..9aa616e190 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -80,13 +80,14 @@ void* streamBackendInit(const char* path, int64_t chkpId); void streamBackendCleanup(void* arg); void streamBackendHandleCleanup(void* arg); int32_t streamBackendLoadCheckpointInfo(void* pMeta); -int32_t streamBackendDoCheckpoint(void* pMeta, uint64_t checkpointId); +int32_t streamBackendDoCheckpoint(void* pMeta, int64_t checkpointId); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); STaskBackendWrapper* taskBackendOpen(char* path, char* key); void taskBackendDestroy(void* pBackend); +int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId); void* taskBackendAddRef(void* pTaskBackend); void taskBackendRemoveRef(void* pTaskBackend); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 54aefc8962..f200c714ab 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -40,6 +40,11 @@ typedef struct { SRpcMsg msg; } SStreamContinueExecInfo; +typedef struct { + int64_t streamId; + int64_t taskId; + int64_t chkpId; +} SStreamTaskSnap; extern SStreamGlobalEnv streamEnv; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5e48a345b1..917bc0f796 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -953,7 +953,15 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI int32_t taskBackendBuildSnap(void* arg, int64_t chkpId) { SStreamMeta* pMeta = arg; void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + int32_t code = 0; + while (pIter) { + STaskBackendWrapper* pBackend = *(STaskBackendWrapper**)pIter; + taskBackendAddRef(pBackend); + + code = taskBackendDoCheckpoint((STaskBackendWrapper*)pBackend, chkpId); + + taskBackendRemoveRef(pBackend); pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); } return 0; @@ -1024,7 +1032,10 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { // taosWUnLockLatch(&pMeta->chkpDirLock); } -int32_t taskBackendDoCheckpoint(void* arg, uint64_t chkpId) { +/* + 0 +*/ +int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId) { STaskBackendWrapper* pBackend = arg; int64_t st = taosGetTimestampMs(); int32_t code = -1; @@ -1065,7 +1076,7 @@ _EXIT: taosReleaseRef(taskBackendWrapperId, refId); return code; } -int32_t streamBackendDoCheckpoint(void* arg, uint64_t chkpId) { return taskBackendDoCheckpoint(arg, chkpId); } +int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskBackendDoCheckpoint(arg, chkpId); } SListNode* streamBackendAddCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; @@ -1577,8 +1588,8 @@ void* taskBackendAddRef(void* pTaskBackend) { return taosAcquireRef(taskBackendWrapperId, pBackend->refId); } void taskBackendRemoveRef(void* pTaskBackend) { - // STaskBackendWrapper* pBackend = pTaskBackend; - // taosReleaseRef(taskBackendWrapperId, pBackend->refId); + STaskBackendWrapper* pBackend = pTaskBackend; + taosReleaseRef(taskBackendWrapperId, pBackend->refId); } // void taskBackendDestroy(STaskBackendWrapper* wrapper); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 88790f2511..a4a507fc13 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -17,6 +17,7 @@ #include "query.h" #include "rocksdb/c.h" #include "streamBackendRocksdb.h" +#include "streamInt.h" #include "tcommon.h" enum SBackendFileType { @@ -39,7 +40,26 @@ typedef struct SBackendFile { SArray* pSst; char* pCheckpointMeta; char* path; + } SBanckendFile; + +typedef struct SBackendSnapFiles2 { + char* pCurrent; + char* pMainfest; + char* pOptions; + SArray* pSst; + char* pCheckpointMeta; + char* path; + + int64_t checkpointId; + int64_t seraial; + int64_t offset; + TdFilePtr fd; + int8_t filetype; + SArray* pFileList; + int32_t currFileIdx; + +} SBackendSnapFile2; struct SStreamSnapHandle { void* handle; SBanckendFile* pBackendFile; @@ -50,6 +70,9 @@ struct SStreamSnapHandle { int8_t filetype; SArray* pFileList; int32_t currFileIdx; + + SArray* pBackendSnapSet; + int32_t currIdx; }; struct SStreamSnapBlockHdr { int8_t type; @@ -108,202 +131,204 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamStateSnapBuild(void* arg, char* path, int64_t chkpId) { - return taskBackendBuildSnap(arg, chkpId); - // int32_t code = 0; - // int8_t validChkp = 0; +int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return taskBackendBuildSnap(arg, chkpId); } - // int len = strlen(path); - // char* tpath = taosMemoryCalloc(1, len + 256); - // memcpy(tpath, path, len); +void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { + char* buf = taosMemoryCalloc(1, 512); + sprintf(buf, "[current: %s,", pSnapFile->pCurrent); + sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); + sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); - // SStreamMeta *pMeta = arg; - // if (chkpId != 0) { - // sprintf(tpath, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", - // TD_DIRSEP, - // chkpId); - // if (taosIsDir(tpath)) { - // validChkp = 1; - // qInfo("%s start to read snap %s", STREAM_STATE_TRANSFER, tpath); - // streamBackendAddInUseChkp(pMeta, chkpId); - // } else { - // qWarn("%s failed to read from %s, reason: dir not exist,retry to default state dir", STREAM_STATE_TRANSFER, - // tpath); - // } - // } + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* name = taosArrayGetP(pSnapFile->pSst, i); + sprintf(buf + strlen(buf), "%s,", name); + } + sprintf(buf + strlen(buf) - 1, "]"); - // no checkpoint specified or not exists invalid checkpoint, do checkpoint at default path and translate it - // if (validChkp == 0) { - // sprintf(tpath, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state"); - // char* chkpdir = taosMemoryCalloc(1, len + 256); - // sprintf(chkpdir, "%s%s%s", tpath, TD_DIRSEP, "tmp"); - // taosMemoryFree(tpath); - - // tpath = chkpdir; - // qInfo("%s start to trigger checkpoint on %s", STREAM_STATE_TRANSFER, tpath); - - // code = streamBackendTriggerChkp(arg, tpath); - // if (code != 0) { - // qError("%s failed to trigger chekckpoint at %s", STREAM_STATE_TRANSFER, tpath); - // taosMemoryFree(tpath); - // return code; - // } - // chkpId = 0; - // } - - //*dstPath = tpath; + qInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); + taosMemoryFree(buf); } -int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { - // impl later - char* tdir = NULL; - int32_t code = streamStateSnapBuild(pMeta, path, chkpId); - if (code != 0) { + +int32_t snapFileCvtMeta(SBackendSnapFile2* pSnapFile) { + SBackendFileItem item; + // current + item.name = pSnapFile->pCurrent; + item.type = ROCKSDB_CURRENT_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); + + // mainfest + item.name = pSnapFile->pMainfest; + item.type = ROCKSDB_MAINFEST_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); + + // options + item.name = pSnapFile->pOptions; + item.type = ROCKSDB_OPTIONS_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); + // sst + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* sst = taosArrayGetP(pSnapFile->pSst, i); + item.name = sst; + item.type = ROCKSDB_SST_TYPE; + streamGetFileSize(pSnapFile->path, item.name, &item.size); + taosArrayPush(pSnapFile->pFileList, &item); + } + // meta + item.name = pSnapFile->pCheckpointMeta; + item.type = ROCKSDB_CHECKPOINT_META_TYPE; + if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { + taosArrayPush(pSnapFile->pFileList, &item); + } + return 0; +} +int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { + TdDirPtr pDir = taosOpenDir(pSnapFile->path); + if (NULL == pDir) { + qError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); return -1; } - qInfo("%s start to read dir: %s", STREAM_STATE_TRANSFER, tdir); - - TdDirPtr pDir = taosOpenDir(tdir); - if (NULL == pDir) { - qError("%s failed to open %s", STREAM_STATE_TRANSFER, tdir); - goto _err; - } - - SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); - pHandle->pBackendFile = pFile; - pHandle->checkpointId = chkpId; - pHandle->seraial = 0; - - pFile->path = tdir; - pFile->pSst = taosArrayInit(16, sizeof(void*)); - TdDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(pDirEntry); if (strlen(name) >= strlen(ROCKSDB_CURRENT) && 0 == strncmp(name, ROCKSDB_CURRENT, strlen(ROCKSDB_CURRENT))) { - pFile->pCurrent = taosStrdup(name); + pSnapFile->pCurrent = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_MAINFEST) && 0 == strncmp(name, ROCKSDB_MAINFEST, strlen(ROCKSDB_MAINFEST))) { - pFile->pMainfest = taosStrdup(name); + pSnapFile->pMainfest = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_OPTIONS) && 0 == strncmp(name, ROCKSDB_OPTIONS, strlen(ROCKSDB_OPTIONS))) { - pFile->pOptions = taosStrdup(name); + pSnapFile->pOptions = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_CHECKPOINT_META) && 0 == strncmp(name, ROCKSDB_CHECKPOINT_META, strlen(ROCKSDB_CHECKPOINT_META))) { - pFile->pCheckpointMeta = taosStrdup(name); + pSnapFile->pCheckpointMeta = taosStrdup(name); continue; } if (strlen(name) >= strlen(ROCKSDB_SST) && 0 == strncmp(name + strlen(name) - strlen(ROCKSDB_SST), ROCKSDB_SST, strlen(ROCKSDB_SST))) { char* sst = taosStrdup(name); - taosArrayPush(pFile->pSst, &sst); + taosArrayPush(pSnapFile->pSst, &sst); } } - { - char* buf = taosMemoryCalloc(1, 512); - sprintf(buf, "[current: %s,", pFile->pCurrent); - sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest); - sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions); - - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* name = taosArrayGetP(pFile->pSst, i); - sprintf(buf + strlen(buf), "%s,", name); - } - sprintf(buf + strlen(buf) - 1, "]"); - - qInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); - taosMemoryFree(buf); - } - taosCloseDir(&pDir); - - if (pFile->pCurrent == NULL) { - qError("%s failed to open %s, reason: no valid file", STREAM_STATE_TRANSFER, tdir); - code = -1; - tdir = NULL; - goto _err; - } - SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); - - SBackendFileItem item; - // current - item.name = pFile->pCurrent; - item.type = ROCKSDB_CURRENT_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - - // mainfest - item.name = pFile->pMainfest; - item.type = ROCKSDB_MAINFEST_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - - // options - item.name = pFile->pOptions; - item.type = ROCKSDB_OPTIONS_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - // sst - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* sst = taosArrayGetP(pFile->pSst, i); - item.name = sst; - item.type = ROCKSDB_SST_TYPE; - streamGetFileSize(pFile->path, item.name, &item.size); - taosArrayPush(list, &item); - } - // meta - item.name = pFile->pCheckpointMeta; - item.type = ROCKSDB_CHECKPOINT_META_TYPE; - if (streamGetFileSize(pFile->path, item.name, &item.size) == 0) { - taosArrayPush(list, &item); - } - - pHandle->pBackendFile = pFile; - - pHandle->currFileIdx = 0; - pHandle->pFileList = list; - pHandle->seraial = 0; - pHandle->offset = 0; - pHandle->handle = pMeta; return 0; +} +int32_t streamBackendSnapInitFile(char* path, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { + // SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); + int32_t code = -1; + + char* snapPath = taosMemoryCalloc(1, strlen(path) + 256); + sprintf(snapPath, "%s%s%" PRId64 "_%" PRId64 "%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, pSnap->streamId, + pSnap->taskId, TD_DIRSEP, "state", TD_DIRSEP, "checkpoints", TD_DIRSEP, pSnap->chkpId); + if (taosIsDir(snapPath)) { + goto _ERROR; + } + + pSnapFile->pSst = taosArrayInit(16, sizeof(void*)); + pSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); + pSnapFile->path = snapPath; + if ((code = snapFileReadMeta(pSnapFile)) != 0) { + goto _ERROR; + } + if ((code = snapFileCvtMeta(pSnapFile)) != 0) { + goto _ERROR; + } + + snapFileDebugInfo(pSnapFile); + + code = 0; + +_ERROR: + taosMemoryFree(snapPath); + return code; +} +void snapFileDestroy(SBackendSnapFile2* pSnap) { + taosMemoryFree(pSnap->pCheckpointMeta); + taosMemoryFree(pSnap->pCurrent); + taosMemoryFree(pSnap->pMainfest); + taosMemoryFree(pSnap->pOptions); + taosMemoryFree(pSnap->path); + for (int i = 0; i < taosArrayGetSize(pSnap->pSst); i++) { + char* sst = taosArrayGetP(pSnap->pSst, i); + taosMemoryFree(sst); + } + taosArrayDestroy(pSnap->pFileList); + taosArrayDestroy(pSnap->pSst); + taosCloseFile(&pSnap->fd); + + return; +} +int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { + // impl later + + SArray* pSnapSet = NULL; + int32_t code = streamBackendGetSnapInfo(pMeta, path, chkpId); + if (code != 0) { + return -1; + } + + SArray* pBdSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + + for (int i = 0; i < taosArrayGetSize(pSnapSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); + + SBackendSnapFile2 snapFile = {0}; + code = streamBackendSnapInitFile(path, pSnap, &snapFile); + ASSERT(code == 0); + taosArrayPush(pBdSnapSet, &snapFile); + } + + pHandle->pBackendSnapSet = pBdSnapSet; + pHandle->currIdx = 0; + return 0; + _err: streamSnapHandleDestroy(pHandle); - taosMemoryFreeClear(tdir); code = -1; return code; } void streamSnapHandleDestroy(SStreamSnapHandle* handle) { - SBanckendFile* pFile = handle->pBackendFile; + // SBanckendFile* pFile = handle->pBackendFile; + if (handle->pBackendSnapSet) { + for (int i = 0; i < taosArrayGetSize(handle->pBackendSnapSet); i++) { + SBackendSnapFile2* pSnapFile = taosArrayGet(handle->pBackendSnapSet, i); + snapFileDestroy(pSnapFile); + } + taosArrayDestroy(handle->pBackendSnapSet); + } - if (handle->checkpointId == 0) { - // del tmp dir - if (pFile && taosIsDir(pFile->path)) { - taosRemoveDir(pFile->path); - } - } else { - streamBackendDelInUseChkp(handle->handle, handle->checkpointId); - } - if (pFile) { - taosMemoryFree(pFile->pCheckpointMeta); - taosMemoryFree(pFile->pCurrent); - taosMemoryFree(pFile->pMainfest); - taosMemoryFree(pFile->pOptions); - taosMemoryFree(pFile->path); - for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - char* sst = taosArrayGetP(pFile->pSst, i); - taosMemoryFree(sst); - } - taosArrayDestroy(pFile->pSst); - taosMemoryFree(pFile); - } - taosArrayDestroy(handle->pFileList); - taosCloseFile(&handle->fd); + // if (handle->checkpointId == 0) { + // // del tmp dir + // if (pFile && taosIsDir(pFile->path)) { + // taosRemoveDir(pFile->path); + // } + // } else { + // streamBackendDelInUseChkp(handle->handle, handle->checkpointId); + // } + // if (pFile) { + // taosMemoryFree(pFile->pCheckpointMeta); + // taosMemoryFree(pFile->pCurrent); + // taosMemoryFree(pFile->pMainfest); + // taosMemoryFree(pFile->pOptions); + // taosMemoryFree(pFile->path); + // for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { + // char* sst = taosArrayGetP(pFile->pSst, i); + // taosMemoryFree(sst); + // } + // taosArrayDestroy(pFile->pSst); + // taosMemoryFree(pFile); + // } + // taosArrayDestroy(handle->pFileList); + // taosCloseFile(&handle->fd); return; } From 4cff121a38ccffbd4f38645d9bf395713155bbf8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 14:46:41 +0800 Subject: [PATCH 031/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 98 +++++++++++++++---------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index a4a507fc13..64ff7b7569 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -51,13 +51,14 @@ typedef struct SBackendSnapFiles2 { char* pCheckpointMeta; char* path; - int64_t checkpointId; - int64_t seraial; - int64_t offset; - TdFilePtr fd; - int8_t filetype; - SArray* pFileList; - int32_t currFileIdx; + int64_t checkpointId; + int64_t seraial; + int64_t offset; + TdFilePtr fd; + int8_t filetype; + SArray* pFileList; + int32_t currFileIdx; + SStreamTaskSnap snapInfo; } SBackendSnapFile2; struct SStreamSnapHandle { @@ -78,10 +79,13 @@ struct SStreamSnapBlockHdr { int8_t type; int8_t flag; int64_t index; - char name[128]; - int64_t totalSize; - int64_t size; - uint8_t data[]; + // int64_t streamId; + // int64_t taskId; + SStreamTaskSnap snapInfo; + char name[128]; + int64_t totalSize; + int64_t size; + uint8_t data[]; }; struct SStreamSnapReader { void* pMeta; @@ -234,6 +238,7 @@ int32_t streamBackendSnapInitFile(char* path, SStreamTaskSnap* pSnap, SBackendSn pSnapFile->pSst = taosArrayInit(16, sizeof(void*)); pSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); pSnapFile->path = snapPath; + pSnapFile->snapInfo = *pSnap; if ((code = snapFileReadMeta(pSnapFile)) != 0) { goto _ERROR; } @@ -355,31 +360,43 @@ int32_t streamSnapReaderClose(SStreamSnapReader* pReader) { taosMemoryFree(pReader); return 0; } + int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* size) { // impl later int32_t code = 0; SStreamSnapHandle* pHandle = &pReader->handle; - SBanckendFile* pFile = pHandle->pBackendFile; + int32_t idx = pHandle->currIdx; + SBackendSnapFile2* pSnapFile = taosArrayGet(pHandle->pBackendSnapSet, idx); + SBackendFileItem* item = NULL; - SBackendFileItem* item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); +_NEXT: + + if (pSnapFile->fd == NULL) { + if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { + if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pBackendSnapSet)) { + pHandle->currIdx += 1; + + pSnapFile = taosArrayGet(pHandle->pBackendSnapSet, pHandle->currIdx); + goto _NEXT; + } else { + *ppData = NULL; + *size = 0; + return 0; + } - if (pHandle->fd == NULL) { - if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) { - // finish - *ppData = NULL; - *size = 0; - return 0; } else { - pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ); + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, item->name, TD_FILE_READ); qDebug("%s open file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); + item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); } } qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); + item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); + uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); - int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); + int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); if (nread == -1) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, @@ -388,43 +405,44 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si } else if (nread > 0 && nread <= kBlockSize) { // left bytes less than kBlockSize qDebug("%s read file %s, current offset:%" PRId64 ",size:% " PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); - pHandle->offset += nread; - if (pHandle->offset >= item->size || nread < kBlockSize) { - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); + pSnapFile->offset += nread; + if (pSnapFile->offset >= item->size || nread < kBlockSize) { + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; } } else { qDebug("%s no data read, close file no.%d, move to next file, open and read", STREAM_STATE_TRANSFER, - pHandle->currFileIdx); - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + pSnapFile->currFileIdx); + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; - if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) { + if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { // finish *ppData = NULL; *size = 0; return 0; } - item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); - pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ); + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, item->name, TD_FILE_READ); - nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); - pHandle->offset += nread; + nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); + pSnapFile->offset += nread; qDebug("%s open file and read file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d", - STREAM_STATE_TRANSFER, item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); + STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); } SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)buf; pHdr->size = nread; pHdr->type = item->type; pHdr->totalSize = item->size; + pHdr->snapInfo = pSnapFile->snapInfo; memcpy(pHdr->name, item->name, strlen(item->name)); - pHandle->seraial += nread; + pSnapFile->seraial += nread; *ppData = buf; *size = sizeof(SStreamSnapBlockHdr) + nread; From 0baef335d70fdcd53b86b9600ad19ebd45b5f629 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 16:03:04 +0800 Subject: [PATCH 032/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 125 +++++++++++++++++------- 1 file changed, 89 insertions(+), 36 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 64ff7b7569..a4363f46b1 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -59,6 +59,7 @@ typedef struct SBackendSnapFiles2 { SArray* pFileList; int32_t currFileIdx; SStreamTaskSnap snapInfo; + int8_t inited; } SBackendSnapFile2; struct SStreamSnapHandle { @@ -71,6 +72,7 @@ struct SStreamSnapHandle { int8_t filetype; SArray* pFileList; int32_t currFileIdx; + char* metaPath; SArray* pBackendSnapSet; int32_t currIdx; @@ -310,6 +312,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { } taosArrayDestroy(handle->pBackendSnapSet); } + taosMemoryFree(handle->metaPath); // if (handle->checkpointId == 0) { // // del tmp dir @@ -455,76 +458,126 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path if (pWriter == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + SBackendSnapFile2 snapFile = {0}; + SStreamSnapHandle* pHandle = &pWriter->handle; + pHandle->pBackendSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); - SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); - pFile->path = taosStrdup(path); - SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); + taosArrayPush(pHandle->pBackendSnapSet, &snapFile); + pHandle->currIdx = 0; + pHandle->metaPath = taosStrdup(path); - SBackendFileItem item; - item.name = taosStrdup((char*)ROCKSDB_CURRENT); - item.type = ROCKSDB_CURRENT_TYPE; - taosArrayPush(list, &item); + // SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); + // pFile->path = taosStrdup(path); + // SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); - pHandle->pBackendFile = pFile; + // SBackendFileItem item; + // item.name = taosStrdup((char*)ROCKSDB_CURRENT); + // item.type = ROCKSDB_CURRENT_TYPE; + // taosArrayPush(list, &item); - pHandle->pFileList = list; - pHandle->currFileIdx = 0; - pHandle->offset = 0; + // pHandle->pBackendFile = pFile; + + // pHandle->pFileList = list; + // pHandle->currFileIdx = 0; + // pHandle->offset = 0; *ppWriter = pWriter; return 0; } -int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; +int32_t snapInfoEqual(SStreamTaskSnap* a, SStreamTaskSnap* b) { + if (a->streamId != b->streamId || a->taskId != b->taskId || a->chkpId != b->chkpId) { + return 0; + } + return 1; +} +int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData, + SBackendSnapFile2* pBackendFile) { + int code = -1; SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData; SStreamSnapHandle* pHandle = &pWriter->handle; - SBanckendFile* pFile = pHandle->pBackendFile; - SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); + SStreamTaskSnap snapInfo = pHdr->snapInfo; - if (pHandle->fd == NULL) { - pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pHandle->fd == NULL) { + SStreamTaskSnap* pSnapInfo = &pBackendFile->snapInfo; + + SBackendFileItem* pItem = taosArrayGet(pBackendFile->pFileList, pBackendFile->currFileIdx); + + if (pBackendFile->fd == 0) { + pBackendFile->fd = streamOpenFile(pHandle->metaPath, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pBackendFile->fd == NULL) { code = TAOS_SYSTEM_ERROR(terrno); - qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pFile->path, TD_DIRSEP, pHdr->name, - tstrerror(code)); + qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pHandle->metaPath, TD_DIRSEP, + pHdr->name, tstrerror(code)); } } - if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { - int64_t bytes = taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset); + int64_t bytes = taosPWriteFile(pBackendFile->fd, pHdr->data, pHdr->size, pBackendFile->offset); if (bytes != pHdr->size) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; } - pHandle->offset += bytes; + pBackendFile->offset += bytes; } else { - taosCloseFile(&pHandle->fd); - pHandle->offset = 0; - pHandle->currFileIdx += 1; + taosCloseFile(&pBackendFile->fd); + pBackendFile->offset = 0; + pBackendFile->currFileIdx += 1; SBackendFileItem item; item.name = taosStrdup(pHdr->name); item.type = pHdr->type; - taosArrayPush(pHandle->pFileList, &item); + taosArrayPush(pBackendFile->pFileList, &item); - SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); - pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pHandle->fd == NULL) { + SBackendFileItem* pItem = taosArrayGet(pBackendFile->pFileList, pBackendFile->currFileIdx); + pBackendFile->fd = streamOpenFile(pHandle->metaPath, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pBackendFile->fd == NULL) { code = TAOS_SYSTEM_ERROR(terrno); - qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pFile->path, TD_DIRSEP, pHdr->name, - tstrerror(code)); + qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pBackendFile->path, TD_DIRSEP, + pHdr->name, tstrerror(code)); } - taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset); - pHandle->offset += pHdr->size; + taosPWriteFile(pBackendFile->fd, pHdr->data, pHdr->size, pBackendFile->offset); + pBackendFile->offset += pHdr->size; } + code = 0; +_EXIT: + return code; +} +int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + int32_t code = 0; - // impl later - return 0; + SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData; + SStreamSnapHandle* pHandle = &pWriter->handle; + SStreamTaskSnap snapInfo = pHdr->snapInfo; + + SBackendSnapFile2* pBackendFile = taosArrayGet(pHandle->pBackendSnapSet, pHandle->currIdx); + if (pBackendFile->inited == 0) { + pBackendFile->snapInfo = snapInfo; + pBackendFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); + pBackendFile->currFileIdx = 0; + pBackendFile->offset = 0; + + SBackendFileItem item; + item.name = taosStrdup((char*)ROCKSDB_CURRENT); + item.type = ROCKSDB_CURRENT_TYPE; + taosArrayPush(pBackendFile->pFileList, &item); + + pBackendFile->inited = 1; + return streamSnapWriteImpl(pWriter, pData, nData, pBackendFile); + } else { + if (snapInfoEqual(&snapInfo, &pBackendFile->snapInfo)) { + return streamSnapWriteImpl(pWriter, pData, nData, pBackendFile); + } else { + SBackendSnapFile2 snapFile = {0}; + taosArrayPush(pHandle->pBackendSnapSet, &snapFile); + pHandle->currIdx += 1; + + return streamSnapWrite(pWriter, pData, nData); + } + } + return code; } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { SStreamSnapHandle* handle = &pWriter->handle; From 722d72659b5c2b38680f5045ffc6054935c18a7b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 16:10:16 +0800 Subject: [PATCH 033/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 58 ++++++++++++------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index a4363f46b1..e7bfca8252 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -140,19 +140,21 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return taskBackendBuildSnap(arg, chkpId); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { - char* buf = taosMemoryCalloc(1, 512); - sprintf(buf, "[current: %s,", pSnapFile->pCurrent); - sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); - sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); + if (qDebugFlag & DEBUG_DEBUG) { + char* buf = taosMemoryCalloc(1, 512); + sprintf(buf, "[current: %s,", pSnapFile->pCurrent); + sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); + sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); - for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { - char* name = taosArrayGetP(pSnapFile->pSst, i); - sprintf(buf + strlen(buf), "%s,", name); + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* name = taosArrayGetP(pSnapFile->pSst, i); + sprintf(buf + strlen(buf), "%s,", name); + } + sprintf(buf + strlen(buf) - 1, "]"); + + qInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); + taosMemoryFree(buf); } - sprintf(buf + strlen(buf) - 1, "]"); - - qInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); - taosMemoryFree(buf); } int32_t snapFileCvtMeta(SBackendSnapFile2* pSnapFile) { @@ -308,6 +310,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { if (handle->pBackendSnapSet) { for (int i = 0; i < taosArrayGetSize(handle->pBackendSnapSet); i++) { SBackendSnapFile2* pSnapFile = taosArrayGet(handle->pBackendSnapSet, i); + snapFileDebugInfo(pSnapFile); snapFileDestroy(pSnapFile); } taosArrayDestroy(handle->pBackendSnapSet); @@ -581,25 +584,20 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { SStreamSnapHandle* handle = &pWriter->handle; - if (qDebugFlag & DEBUG_DEBUG) { - char* buf = (char*)taosMemoryMalloc(1024); - int n = sprintf(buf, "["); - for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { - SBackendFileItem* item = taosArrayGet(handle->pFileList, i); - if (i != taosArrayGetSize(handle->pFileList) - 1) { - n += sprintf(buf + n, "%s %" PRId64 ",", item->name, item->size); - } else { - n += sprintf(buf + n, "%s %" PRId64 "]", item->name, item->size); - } - } - qDebug("%s snap get file list, %s", STREAM_STATE_TRANSFER, buf); - taosMemoryFree(buf); - } - - for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { - SBackendFileItem* item = taosArrayGet(handle->pFileList, i); - taosMemoryFree(item->name); - } + // if (qDebugFlag & DEBUG_DEBUG) { + // char* buf = (char*)taosMemoryMalloc(1024); + // int n = sprintf(buf, "["); + // for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { + // SBackendFileItem* item = taosArrayGet(handle->pFileList, i); + // if (i != taosArrayGetSize(handle->pFileList) - 1) { + // n += sprintf(buf + n, "%s %" PRId64 ",", item->name, item->size); + // } else { + // n += sprintf(buf + n, "%s %" PRId64 "]", item->name, item->size); + // } + // } + // qDebug("%s snap get file list, %s", STREAM_STATE_TRANSFER, buf); + // taosMemoryFree(buf); + // } streamSnapHandleDestroy(handle); taosMemoryFree(pWriter); From 4f471625122ee1c2abd84e36a7491a2a434c5ef2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 16:36:43 +0800 Subject: [PATCH 034/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 27 +++++++------------------ 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index e7bfca8252..25c228e97d 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -145,14 +145,16 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { sprintf(buf, "[current: %s,", pSnapFile->pCurrent); sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); - - for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { - char* name = taosArrayGetP(pSnapFile->pSst, i); - sprintf(buf + strlen(buf), "%s,", name); + if (pSnapFile->pSst) { + for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* name = taosArrayGetP(pSnapFile->pSst, i); + sprintf(buf + strlen(buf), "%s,", name); + } } sprintf(buf + strlen(buf) - 1, "]"); - qInfo("%s get file list: %s", STREAM_STATE_TRANSFER, buf); + qInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, + pSnapFile->snapInfo.taskId, buf); taosMemoryFree(buf); } } @@ -584,21 +586,6 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { SStreamSnapHandle* handle = &pWriter->handle; - // if (qDebugFlag & DEBUG_DEBUG) { - // char* buf = (char*)taosMemoryMalloc(1024); - // int n = sprintf(buf, "["); - // for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { - // SBackendFileItem* item = taosArrayGet(handle->pFileList, i); - // if (i != taosArrayGetSize(handle->pFileList) - 1) { - // n += sprintf(buf + n, "%s %" PRId64 ",", item->name, item->size); - // } else { - // n += sprintf(buf + n, "%s %" PRId64 "]", item->name, item->size); - // } - // } - // qDebug("%s snap get file list, %s", STREAM_STATE_TRANSFER, buf); - // taosMemoryFree(buf); - // } - streamSnapHandleDestroy(handle); taosMemoryFree(pWriter); From 031506aef8b2088b5c03902ee7e40c718fd88a3b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 19:46:35 +0800 Subject: [PATCH 035/151] refactor backend --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tqStreamStateSnap.c | 5 +++-- source/libs/stream/src/streamMeta.c | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index a9e593eaad..859d7cea54 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -725,6 +725,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); +int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref); diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 38cd1e1b76..b2835f2198 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -169,14 +169,15 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) } int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); - // int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta, chkpId); int32_t code = streamStateLoadTasks(pWriter); tqDebug("vgId:%d, vnode %s succ to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); taosMemoryFree(pWriter); return code; } -int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) { return streamMetaLoadAllTasks(pWriter->pTq->pStreamMeta); } +int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) { + return streamMetaReloadAllTasks(pWriter->pTq->pStreamMeta); +} int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) { tqDebug("vgId:%d, vnode %s snapshot write data", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 72399865e6..83fe0b03c0 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -756,6 +756,22 @@ static void doClear(void* pKey, void* pVal, TBC* pCur, SArray* pRecycleList) { taosArrayDestroy(pRecycleList); } +int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta) { + if (pMeta == NULL) return 0; + + void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + while (pIter) { + STaskBackendWrapper* taskBackend = *(STaskBackendWrapper**)pIter; + if (taskBackend != NULL) { + taskBackendRemoveRef(taskBackend); + } + pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + } + taosHashClear(pMeta->pTaskBackendUnique); + + + return 0; +} int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { TBC* pCur = NULL; From 83b52a4c1d953e0873d5bb64b36cfd9e8141c9fd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 20:02:40 +0800 Subject: [PATCH 036/151] refactor backend --- source/libs/stream/src/streamMeta.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 83fe0b03c0..c398254548 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -759,18 +759,17 @@ static void doClear(void* pKey, void* pVal, TBC* pCur, SArray* pRecycleList) { int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta) { if (pMeta == NULL) return 0; - void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); - while (pIter) { - STaskBackendWrapper* taskBackend = *(STaskBackendWrapper**)pIter; - if (taskBackend != NULL) { - taskBackendRemoveRef(taskBackend); - } - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); - } - taosHashClear(pMeta->pTaskBackendUnique); + // void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + // while (pIter) { + // STaskBackendWrapper* taskBackend = *(STaskBackendWrapper**)pIter; + // if (taskBackend != NULL) { + // taskBackendRemoveRef(taskBackend); + // } + // pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + // } + // taosHashClear(pMeta->pTaskBackendUnique); - - return 0; + return streamMetaLoadAllTasks(pMeta); } int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { TBC* pCur = NULL; From f264ffdcd645dac5f3de780d8501362af0f0cd1d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 20:12:07 +0800 Subject: [PATCH 037/151] refactor backend --- include/libs/stream/tstream.h | 2 +- source/libs/stream/inc/streamBackendRocksdb.h | 12 +- source/libs/stream/src/streamBackendRocksdb.c | 190 +++++++++--------- source/libs/stream/src/streamMeta.c | 10 +- source/libs/stream/src/streamTask.c | 4 +- 5 files changed, 109 insertions(+), 109 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 859d7cea54..260609e697 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -425,7 +425,7 @@ typedef struct SStreamMeta { SRWLatch chkpDirLock; int32_t pauseTaskNum; - // SHashObj* pTaskBackend; + // SHashObj* pTaskDb; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 9aa616e190..cea948e2f1 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -74,7 +74,7 @@ typedef struct { int32_t chkpCap; TdThreadRwlock chkpDirLock; -} STaskBackendWrapper; +} STaskDbWrapper; void* streamBackendInit(const char* path, int64_t chkpId); void streamBackendCleanup(void* arg); @@ -85,12 +85,12 @@ SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); -STaskBackendWrapper* taskBackendOpen(char* path, char* key); -void taskBackendDestroy(void* pBackend); -int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId); +STaskDbWrapper* taskBackendOpen(char* path, char* key); +void taskDbDestroy(void* pBackend); +int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId); -void* taskBackendAddRef(void* pTaskBackend); -void taskBackendRemoveRef(void* pTaskBackend); +void* taskDbAddRef(void* pTaskDb); +void taskDbRemoveRef(void* pTaskDb); int streamStateOpenBackend(void* backend, SStreamState* pState); void streamStateCloseBackend(SStreamState* pState, bool remove); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 917bc0f796..d5450711a2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -716,7 +716,7 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { * replication is finished */ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { - STaskBackendWrapper* pBackend = arg; + STaskDbWrapper* pBackend = arg; taosThreadRwlockWrlock(&pBackend->chkpDirLock); @@ -863,7 +863,7 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* // return nCf; } -int32_t chkpGetAllDbCfHandle2(STaskBackendWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { +int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { SArray* pHandle = taosArrayInit(8, POINTER_BYTES); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { if (pBackend->pCf[i]) { @@ -956,12 +956,12 @@ int32_t taskBackendBuildSnap(void* arg, int64_t chkpId) { int32_t code = 0; while (pIter) { - STaskBackendWrapper* pBackend = *(STaskBackendWrapper**)pIter; - taskBackendAddRef(pBackend); + STaskDbWrapper* pBackend = *(STaskDbWrapper**)pIter; + taskDbAddRef(pBackend); - code = taskBackendDoCheckpoint((STaskBackendWrapper*)pBackend, chkpId); + code = taskBackendDoCheckpoint((STaskDbWrapper*)pBackend, chkpId); - taskBackendRemoveRef(pBackend); + taskDbRemoveRef(pBackend); pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); } return 0; @@ -1036,10 +1036,10 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { 0 */ int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId) { - STaskBackendWrapper* pBackend = arg; - int64_t st = taosGetTimestampMs(); - int32_t code = -1; - int64_t refId = pBackend->refId; + STaskDbWrapper* pBackend = arg; + int64_t st = taosGetTimestampMs(); + int32_t code = -1; + int64_t refId = pBackend->refId; if (taosAcquireRef(taskBackendWrapperId, refId) == NULL) { return -1; @@ -1549,7 +1549,7 @@ int32_t getCfIdx(const char* cfName) { } return idx; } -int32_t taskBackendOpenCfs(STaskBackendWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { +int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { int32_t code = -1; char* err = NULL; @@ -1583,17 +1583,17 @@ _EXIT: taosMemoryFree(cfHandle); return code; } -void* taskBackendAddRef(void* pTaskBackend) { - STaskBackendWrapper* pBackend = pTaskBackend; +void* taskDbAddRef(void* pTaskDb) { + STaskDbWrapper* pBackend = pTaskDb; return taosAcquireRef(taskBackendWrapperId, pBackend->refId); } -void taskBackendRemoveRef(void* pTaskBackend) { - STaskBackendWrapper* pBackend = pTaskBackend; +void taskDbRemoveRef(void* pTaskDb) { + STaskDbWrapper* pBackend = pTaskDb; taosReleaseRef(taskBackendWrapperId, pBackend->refId); } -// void taskBackendDestroy(STaskBackendWrapper* wrapper); +// void taskDbDestroy(STaskDbWrapper* wrapper); -void taskBackendInitDBOpt(STaskBackendWrapper* pTaskBackend) { +void taskDbInitDBOpt(STaskDbWrapper* pTaskDb) { rocksdb_env_t* env = rocksdb_create_default_env(); rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); @@ -1609,25 +1609,25 @@ void taskBackendInitDBOpt(STaskBackendWrapper* pTaskBackend) { rocksdb_options_set_write_buffer_size(opts, 32 << 20); rocksdb_options_set_atomic_flush(opts, 1); - pTaskBackend->dbOpt = opts; - pTaskBackend->env = env; - pTaskBackend->cache = cache; - pTaskBackend->filterFactory = rocksdb_compactionfilterfactory_create( + pTaskDb->dbOpt = opts; + pTaskDb->env = env; + pTaskDb->cache = cache; + pTaskDb->filterFactory = rocksdb_compactionfilterfactory_create( NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); - rocksdb_options_set_compaction_filter_factory(pTaskBackend->dbOpt, pTaskBackend->filterFactory); - pTaskBackend->readOpt = rocksdb_readoptions_create(); - pTaskBackend->writeOpt = rocksdb_writeoptions_create(); + rocksdb_options_set_compaction_filter_factory(pTaskDb->dbOpt, pTaskDb->filterFactory); + pTaskDb->readOpt = rocksdb_readoptions_create(); + pTaskDb->writeOpt = rocksdb_writeoptions_create(); size_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); - pTaskBackend->pCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); - pTaskBackend->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); - pTaskBackend->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); - pTaskBackend->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + pTaskDb->pCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + pTaskDb->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); + pTaskDb->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + pTaskDb->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); for (int i = 0; i < nCf; i++) { - rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskBackend->dbOpt); + rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskDb->dbOpt); rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); - rocksdb_block_based_options_set_block_cache(tableOpt, pTaskBackend->cache); + rocksdb_block_based_options_set_block_cache(tableOpt, pTaskDb->cache); rocksdb_block_based_options_set_partition_filters(tableOpt, 1); rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); @@ -1641,13 +1641,13 @@ void taskBackendInitDBOpt(STaskBackendWrapper* pTaskBackend) { rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); - pTaskBackend->pCompares[i] = compare; - pTaskBackend->pCfOpts[i] = opt; - pTaskBackend->pCfParams[i].tableOpt = tableOpt; + pTaskDb->pCompares[i] = compare; + pTaskDb->pCfOpts[i] = opt; + pTaskDb->pCfParams[i].tableOpt = tableOpt; } return; } -void taskBackendInitChkpOpt(STaskBackendWrapper* pBackend) { +void taskDbInitChkpOpt(STaskDbWrapper* pBackend) { pBackend->chkpId = -1; pBackend->chkpCap = 4; pBackend->chkpSaved = taosArrayInit(4, sizeof(int64_t)); @@ -1656,13 +1656,13 @@ void taskBackendInitChkpOpt(STaskBackendWrapper* pBackend) { taosThreadRwlockInit(&pBackend->chkpDirLock, NULL); } -void taskBackendDestroyChkpOpt(STaskBackendWrapper* pBackend) { +void taskDbDestroyChkpOpt(STaskDbWrapper* pBackend) { taosArrayDestroy(pBackend->chkpSaved); taosArrayDestroy(pBackend->chkpInUse); taosThreadRwlockDestroy(&pBackend->chkpDirLock); } -int32_t taskBackendBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { +int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { int32_t code = 0; char* statePath = taosMemoryCalloc(1, strlen(path) + 128); @@ -1692,41 +1692,41 @@ int32_t taskBackendBuildFullPath(char* path, char* key, char** dbFullPath, char* *stateFullPath = statePath; return 0; } -STaskBackendWrapper* taskBackendOpen(char* path, char* key) { +STaskDbWrapper* taskBackendOpen(char* path, char* key) { char* statePath = NULL; char* err = NULL; char* dbPath = NULL; char** cfNames = NULL; size_t nCf = 0; - if (taskBackendBuildFullPath(path, key, &dbPath, &statePath) != 0) { + if (taskDbBuildFullPath(path, key, &dbPath, &statePath) != 0) { return NULL; } - STaskBackendWrapper* pTaskBackend = taosMemoryCalloc(1, sizeof(STaskBackendWrapper)); - pTaskBackend->idstr = taosStrdup(key); - pTaskBackend->path = statePath; - taosThreadMutexInit(&pTaskBackend->mutex, NULL); - taskBackendInitChkpOpt(pTaskBackend); - taskBackendInitDBOpt(pTaskBackend); + STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); + pTaskDb->idstr = taosStrdup(key); + pTaskDb->path = statePath; + taosThreadMutexInit(&pTaskDb->mutex, NULL); + taskDbInitChkpOpt(pTaskDb); + taskDbInitDBOpt(pTaskDb); statePath = NULL; - cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); + cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { // pre create db - pTaskBackend->db = rocksdb_open(pTaskBackend->pCfOpts[0], dbPath, &err); - rocksdb_close(pTaskBackend->db); + pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); + rocksdb_close(pTaskDb->db); if (cfNames != NULL) { rocksdb_list_column_families_destroy(cfNames, nCf); } taosMemoryFree(err); - cfNames = rocksdb_list_column_families(pTaskBackend->dbOpt, dbPath, &nCf, &err); + cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); ASSERT(err != NULL); } - if (taskBackendOpenCfs(pTaskBackend, dbPath, cfNames, nCf) != 0) { + if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { goto _EXIT; } @@ -1734,13 +1734,13 @@ STaskBackendWrapper* taskBackendOpen(char* path, char* key) { rocksdb_list_column_families_destroy(cfNames, nCf); } - qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskBackend); + qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); taosMemoryFree(dbPath); - return pTaskBackend; + return pTaskDb; _EXIT: - taskBackendDestroy(pTaskBackend); + taskDbDestroy(pTaskDb); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); @@ -1749,8 +1749,8 @@ _EXIT: return NULL; } -void taskBackendDestroy(void* pBackend) { - STaskBackendWrapper* wrapper = pBackend; +void taskDbDestroy(void* pBackend) { + STaskDbWrapper* wrapper = pBackend; if (wrapper == NULL) return; @@ -1790,7 +1790,7 @@ void taskBackendDestroy(void* pBackend) { if (wrapper->db) rocksdb_close(wrapper->db); - taskBackendDestroyChkpOpt(pBackend); + taskDbDestroyChkpOpt(pBackend); taosMemoryFree(wrapper->idstr); taosMemoryFree(wrapper->path); @@ -1799,7 +1799,7 @@ void taskBackendDestroy(void* pBackend) { return; } -int32_t taskBackendOpenCfByKey(STaskBackendWrapper* pBackend, const char* key) { +int32_t taskDbOpenCfByKey(STaskDbWrapper* pBackend, const char* key) { int32_t code = 0; char* err = NULL; int8_t idx = getCfIdx(key); @@ -1817,7 +1817,7 @@ int32_t taskBackendOpenCfByKey(STaskBackendWrapper* pBackend, const char* key) { pBackend->pCf[idx] = cf; return code; } -int32_t copyDataAt(RocksdbCfInst* pSrc, STaskBackendWrapper* pDst, int8_t i) { +int32_t copyDataAt(RocksdbCfInst* pSrc, STaskDbWrapper* pDst, int8_t i) { int32_t WRITE_BATCH = 1024; char* err = NULL; int code = 0; @@ -1866,22 +1866,22 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { int32_t code = 0; - STaskBackendWrapper* pTaskBackend = taskBackendOpen(path, key); - RocksdbCfInst* pSrcBackend = pCfInst; + STaskDbWrapper* pTaskDb = taskBackendOpen(path, key); + RocksdbCfInst* pSrcBackend = pCfInst; for (int i = 0; i < nCf; i++) { rocksdb_column_family_handle_t* pSrcCf = pSrcBackend->pHandle[i]; if (pSrcCf == NULL) continue; - code = taskBackendOpenCfByKey(pTaskBackend, ginitDict[i].key); + code = taskDbOpenCfByKey(pTaskDb, ginitDict[i].key); if (code != 0) goto _EXIT; - code = copyDataAt(pSrcBackend, pTaskBackend, i); + code = copyDataAt(pSrcBackend, pTaskDb, i); if (code != 0) goto _EXIT; } _EXIT: - taskBackendDestroy(pTaskBackend); + taskDbDestroy(pTaskDb); return code; } @@ -2140,7 +2140,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { } } if (pState != NULL && idx != -1) { - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_column_family_handle_t* cf = NULL; taosThreadMutexLock(&wrapper->mutex); @@ -2179,7 +2179,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe *readOpt = rocksdb_readoptions_create(); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (snapshot != NULL) { *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->db); rocksdb_readoptions_set_snapshot(*readOpt, *snapshot); @@ -2200,8 +2200,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe code = -1; \ break; \ } \ - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ @@ -2231,8 +2231,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe code = -1; \ break; \ } \ - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ @@ -2274,8 +2274,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe code = -1; \ break; \ } \ - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ @@ -2314,7 +2314,7 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -2437,7 +2437,7 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin if (pCur == NULL) { return NULL; } - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->number = pState->number; pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, @@ -2493,7 +2493,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK if (pCur == NULL) return NULL; pCur->number = pState->number; - pCur->db = ((STaskBackendWrapper*)pState->pTdbState->pOwner->pBackend)->db; + pCur->db = ((STaskDbWrapper*)pState->pTdbState->pOwner->pBackend)->db; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -2514,7 +2514,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; @@ -2604,8 +2604,8 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } @@ -2645,8 +2645,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } @@ -2683,8 +2683,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } @@ -2785,8 +2785,8 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (pCur == NULL) return NULL; @@ -2845,8 +2845,8 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (!pCur) { return NULL; } @@ -2883,8 +2883,8 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } @@ -2921,8 +2921,8 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const } int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { qDebug("streamStateSessionGetKeyByRange_rocksdb"); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return -1; } @@ -3153,7 +3153,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int code = 0; char* err = NULL; - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); @@ -3192,8 +3192,8 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot, @@ -3244,7 +3244,7 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -3277,7 +3277,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb char* ttlV = tmpBuf; int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); @@ -3293,8 +3293,8 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb return 0; } int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { - char* err = NULL; - STaskBackendWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + char* err = NULL; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { qError("streamState failed to write batch, err:%s", err); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index c398254548..8d855d3513 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -53,7 +53,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); static void streamMetaEnvInit() { streamBackendId = taosOpenRef(64, streamBackendCleanup); streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); - taskBackendWrapperId = taosOpenRef(64, taskBackendDestroy); + taskBackendWrapperId = taosOpenRef(64, taskDbDestroy); streamMetaId = taosOpenRef(64, streamMetaCloseImpl); @@ -226,8 +226,8 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { - taskBackendAddRef(*ppBackend); - *ref = ((STaskBackendWrapper*)*ppBackend)->refId; + taskDbAddRef(*ppBackend); + *ref = ((STaskDbWrapper*)*ppBackend)->refId; taosThreadMutexUnlock(&pMeta->backendMutex); return *ppBackend; } @@ -761,9 +761,9 @@ int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta) { // void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); // while (pIter) { - // STaskBackendWrapper* taskBackend = *(STaskBackendWrapper**)pIter; + // STaskDbWrapper* taskBackend = *(STaskDbWrapper**)pIter; // if (taskBackend != NULL) { - // taskBackendRemoveRef(taskBackend); + // taskDbRemoveRef(taskBackend); // } // pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); // } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 152b426628..c0f9bc65ff 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -355,8 +355,8 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->pUpstreamInfoList = NULL; } if (pTask->pBackend) { - taskBackendRemoveRef(pTask->pBackend); - + taskDbRemoveRef(pTask->pBackend); + pTask->pBackend = NULL; } taosThreadMutexDestroy(&pTask->lock); From 4c94fbb44f223eba2c5781b703fb9b4a56923475 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Oct 2023 20:26:19 +0800 Subject: [PATCH 038/151] refactor backend --- source/libs/stream/inc/streamBackendRocksdb.h | 6 +- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 68 +++++++++---------- source/libs/stream/src/streamMeta.c | 8 +-- source/libs/stream/src/streamSnapshot.c | 2 +- source/libs/stream/src/streamTask.c | 2 +- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index cea948e2f1..0f97bda4c5 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -85,9 +85,9 @@ SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); -STaskDbWrapper* taskBackendOpen(char* path, char* key); +STaskDbWrapper* taskDbOpen(char* path, char* key); void taskDbDestroy(void* pBackend); -int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId); +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); void* taskDbAddRef(void* pTaskDb); void taskDbRemoveRef(void* pTaskDb); @@ -182,7 +182,7 @@ int32_t streamBackendTriggerChkp(void* pMeta, char* dst); int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); -int32_t taskBackendBuildSnap(void* arg, int64_t chkpId); +int32_t taskDbBuildSnap(void* arg, int64_t chkpId); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index f200c714ab..a5e7531f7e 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -48,7 +48,7 @@ typedef struct { extern SStreamGlobalEnv streamEnv; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; -extern int32_t taskBackendWrapperId; +extern int32_t taskDbWrapperId; const char* streamGetBlockTypeStr(int32_t type); void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d5450711a2..78f67d06d5 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -950,18 +950,18 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return 0; } -int32_t taskBackendBuildSnap(void* arg, int64_t chkpId) { +int32_t taskDbBuildSnap(void* arg, int64_t chkpId) { SStreamMeta* pMeta = arg; void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); int32_t code = 0; while (pIter) { - STaskDbWrapper* pBackend = *(STaskDbWrapper**)pIter; - taskDbAddRef(pBackend); + STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter; + taskDbAddRef(pTaskDb); - code = taskBackendDoCheckpoint((STaskDbWrapper*)pBackend, chkpId); + code = taskDbDoCheckpoint((STaskDbWrapper*)pTaskDb, chkpId); - taskDbRemoveRef(pBackend); + taskDbRemoveRef(pTaskDb); pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); } return 0; @@ -1035,48 +1035,48 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { /* 0 */ -int32_t taskBackendDoCheckpoint(void* arg, int64_t chkpId) { - STaskDbWrapper* pBackend = arg; +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { + STaskDbWrapper* pTaskDb = arg; int64_t st = taosGetTimestampMs(); int32_t code = -1; - int64_t refId = pBackend->refId; + int64_t refId = pTaskDb->refId; - if (taosAcquireRef(taskBackendWrapperId, refId) == NULL) { + if (taosAcquireRef(taskDbWrapperId, refId) == NULL) { return -1; } char* pChkpDir = NULL; char* pChkpIdDir = NULL; - if (chkpPreBuildDir(pBackend->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + if (chkpPreBuildDir(pTaskDb->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { code = -1; goto _EXIT; } // Get all cf and acquire cfWrappter rocksdb_column_family_handle_t** ppCf = NULL; - int32_t nCf = chkpGetAllDbCfHandle2(pBackend, &ppCf); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pBackend, pChkpIdDir, nCf); + int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); + qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); - if ((code = chkpPreFlushDb(pBackend->db, ppCf, nCf)) == 0) { - if ((code = chkpDoDbCheckpoint(pBackend->db, pChkpIdDir)) != 0) { - qError("stream backend:%p failed to do checkpoint at:%s", pBackend, pChkpIdDir); + if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { + if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { + qError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pBackend, pChkpIdDir, + qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, taosGetTimestampMs() - st); } } else { - qError("stream backend:%p failed to flush db at:%s", pBackend, pChkpIdDir); + qError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); } - code = chkpMayDelObsolete(pBackend, chkpId, pChkpDir); + code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir); _EXIT: taosMemoryFree(pChkpDir); taosMemoryFree(pChkpIdDir); - taosReleaseRef(taskBackendWrapperId, refId); + taosReleaseRef(taskDbWrapperId, refId); return code; } -int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskBackendDoCheckpoint(arg, chkpId); } +int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); } SListNode* streamBackendAddCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; @@ -1585,11 +1585,11 @@ _EXIT: } void* taskDbAddRef(void* pTaskDb) { STaskDbWrapper* pBackend = pTaskDb; - return taosAcquireRef(taskBackendWrapperId, pBackend->refId); + return taosAcquireRef(taskDbWrapperId, pBackend->refId); } void taskDbRemoveRef(void* pTaskDb) { STaskDbWrapper* pBackend = pTaskDb; - taosReleaseRef(taskBackendWrapperId, pBackend->refId); + taosReleaseRef(taskDbWrapperId, pBackend->refId); } // void taskDbDestroy(STaskDbWrapper* wrapper); @@ -1647,19 +1647,19 @@ void taskDbInitDBOpt(STaskDbWrapper* pTaskDb) { } return; } -void taskDbInitChkpOpt(STaskDbWrapper* pBackend) { - pBackend->chkpId = -1; - pBackend->chkpCap = 4; - pBackend->chkpSaved = taosArrayInit(4, sizeof(int64_t)); - pBackend->chkpInUse = taosArrayInit(4, sizeof(int64_t)); +void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { + pTaskDb->chkpId = -1; + pTaskDb->chkpCap = 4; + pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); + pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); - taosThreadRwlockInit(&pBackend->chkpDirLock, NULL); + taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL); } -void taskDbDestroyChkpOpt(STaskDbWrapper* pBackend) { - taosArrayDestroy(pBackend->chkpSaved); - taosArrayDestroy(pBackend->chkpInUse); - taosThreadRwlockDestroy(&pBackend->chkpDirLock); +void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { + taosArrayDestroy(pTaskDb->chkpSaved); + taosArrayDestroy(pTaskDb->chkpInUse); + taosThreadRwlockDestroy(&pTaskDb->chkpDirLock); } int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { @@ -1692,7 +1692,7 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta *stateFullPath = statePath; return 0; } -STaskDbWrapper* taskBackendOpen(char* path, char* key) { +STaskDbWrapper* taskDbOpen(char* path, char* key) { char* statePath = NULL; char* err = NULL; char* dbPath = NULL; @@ -1866,7 +1866,7 @@ int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { int32_t code = 0; - STaskDbWrapper* pTaskDb = taskBackendOpen(path, key); + STaskDbWrapper* pTaskDb = taskDbOpen(path, key); RocksdbCfInst* pSrcBackend = pCfInst; for (int i = 0; i < nCf; i++) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8d855d3513..f4c8021ac6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -30,7 +30,7 @@ static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; int32_t streamMetaId = 0; -int32_t taskBackendWrapperId = 0; +int32_t taskDbWrapperId = 0; static int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta); static void metaHbToMnode(void* param, void* tmrId); @@ -53,7 +53,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); static void streamMetaEnvInit() { streamBackendId = taosOpenRef(64, streamBackendCleanup); streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); - taskBackendWrapperId = taosOpenRef(64, taskDbDestroy); + taskDbWrapperId = taosOpenRef(64, taskDbDestroy); streamMetaId = taosOpenRef(64, streamMetaCloseImpl); @@ -232,13 +232,13 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) return *ppBackend; } - void* pBackend = taskBackendOpen(pMeta->path, key); + void* pBackend = taskDbOpen(pMeta->path, key); if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); return NULL; } - *ref = taosAddRef(taskBackendWrapperId, pBackend); + *ref = taosAddRef(taskDbWrapperId, pBackend); taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexUnlock(&pMeta->backendMutex); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 25c228e97d..d33f018a2c 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -137,7 +137,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return taskBackendBuildSnap(arg, chkpId); } +int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return taskDbBuildSnap(arg, chkpId); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index c0f9bc65ff..5b4b0c309d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -392,7 +392,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); - // pTask->pBackend = taskBackendOpen(pMeta->path, (char*)pTask->id.idStr); + // pTask->pBackend = taskDbOpen(pMeta->path, (char*)pTask->id.idStr); return TSDB_CODE_SUCCESS; } From bfd205b3f7f7980bed63297449647f1fba7198e0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 11:31:39 +0800 Subject: [PATCH 039/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 118 +++++++++--------------- 1 file changed, 45 insertions(+), 73 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index d33f018a2c..16b0ae0406 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -32,6 +32,7 @@ typedef struct SBackendFileItem { char* name; int8_t type; int64_t size; + int8_t ref; } SBackendFileItem; typedef struct SBackendFile { char* pCurrent; @@ -74,7 +75,7 @@ struct SStreamSnapHandle { int32_t currFileIdx; char* metaPath; - SArray* pBackendSnapSet; + SArray* pDbSnapSet; int32_t currIdx; }; struct SStreamSnapBlockHdr { @@ -142,9 +143,11 @@ int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { char* buf = taosMemoryCalloc(1, 512); - sprintf(buf, "[current: %s,", pSnapFile->pCurrent); - sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); - sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); + sprintf(buf + strlen(buf), "["); + + if (pSnapFile->pCurrent) sprintf(buf, "current: %s,", pSnapFile->pCurrent); + if (pSnapFile->pMainfest) sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); + if (pSnapFile->pOptions) sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); if (pSnapFile->pSst) { for (int i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { char* name = taosArrayGetP(pSnapFile->pSst, i); @@ -160,7 +163,8 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { } int32_t snapFileCvtMeta(SBackendSnapFile2* pSnapFile) { - SBackendFileItem item; + SBackendFileItem item = {0}; + item.ref = 1; // current item.name = pSnapFile->pCurrent; item.type = ROCKSDB_CURRENT_TYPE; @@ -270,6 +274,13 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) { char* sst = taosArrayGetP(pSnap->pSst, i); taosMemoryFree(sst); } + // unite read/write snap file + for (int i = 0; i < taosArrayGetSize(pSnap->pFileList); i++) { + SBackendFileItem* pItem = taosArrayGet(pSnap->pFileList, i); + if (pItem->ref == 0) { + taosMemoryFree(pItem->name); + } + } taosArrayDestroy(pSnap->pFileList); taosArrayDestroy(pSnap->pSst); taosCloseFile(&pSnap->fd); @@ -285,7 +296,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk return -1; } - SArray* pBdSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); for (int i = 0; i < taosArrayGetSize(pSnapSet); i++) { SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); @@ -293,10 +304,10 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk SBackendSnapFile2 snapFile = {0}; code = streamBackendSnapInitFile(path, pSnap, &snapFile); ASSERT(code == 0); - taosArrayPush(pBdSnapSet, &snapFile); + taosArrayPush(pDbSnapSet, &snapFile); } - pHandle->pBackendSnapSet = pBdSnapSet; + pHandle->pDbSnapSet = pDbSnapSet; pHandle->currIdx = 0; return 0; @@ -308,40 +319,15 @@ _err: } void streamSnapHandleDestroy(SStreamSnapHandle* handle) { - // SBanckendFile* pFile = handle->pBackendFile; - if (handle->pBackendSnapSet) { - for (int i = 0; i < taosArrayGetSize(handle->pBackendSnapSet); i++) { - SBackendSnapFile2* pSnapFile = taosArrayGet(handle->pBackendSnapSet, i); + if (handle->pDbSnapSet) { + for (int i = 0; i < taosArrayGetSize(handle->pDbSnapSet); i++) { + SBackendSnapFile2* pSnapFile = taosArrayGet(handle->pDbSnapSet, i); snapFileDebugInfo(pSnapFile); snapFileDestroy(pSnapFile); } - taosArrayDestroy(handle->pBackendSnapSet); + taosArrayDestroy(handle->pDbSnapSet); } taosMemoryFree(handle->metaPath); - - // if (handle->checkpointId == 0) { - // // del tmp dir - // if (pFile && taosIsDir(pFile->path)) { - // taosRemoveDir(pFile->path); - // } - // } else { - // streamBackendDelInUseChkp(handle->handle, handle->checkpointId); - // } - // if (pFile) { - // taosMemoryFree(pFile->pCheckpointMeta); - // taosMemoryFree(pFile->pCurrent); - // taosMemoryFree(pFile->pMainfest); - // taosMemoryFree(pFile->pOptions); - // taosMemoryFree(pFile->path); - // for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { - // char* sst = taosArrayGetP(pFile->pSst, i); - // taosMemoryFree(sst); - // } - // taosArrayDestroy(pFile->pSst); - // taosMemoryFree(pFile); - // } - // taosArrayDestroy(handle->pFileList); - // taosCloseFile(&handle->fd); return; } @@ -374,17 +360,17 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si int32_t code = 0; SStreamSnapHandle* pHandle = &pReader->handle; int32_t idx = pHandle->currIdx; - SBackendSnapFile2* pSnapFile = taosArrayGet(pHandle->pBackendSnapSet, idx); + SBackendSnapFile2* pSnapFile = taosArrayGet(pHandle->pDbSnapSet, idx); SBackendFileItem* item = NULL; _NEXT: if (pSnapFile->fd == NULL) { if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { - if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pBackendSnapSet)) { + if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pDbSnapSet)) { pHandle->currIdx += 1; - pSnapFile = taosArrayGet(pHandle->pBackendSnapSet, pHandle->currIdx); + pSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); goto _NEXT; } else { *ppData = NULL; @@ -466,27 +452,12 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path SBackendSnapFile2 snapFile = {0}; SStreamSnapHandle* pHandle = &pWriter->handle; - pHandle->pBackendSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); - taosArrayPush(pHandle->pBackendSnapSet, &snapFile); + taosArrayPush(pHandle->pDbSnapSet, &snapFile); pHandle->currIdx = 0; pHandle->metaPath = taosStrdup(path); - // SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); - // pFile->path = taosStrdup(path); - // SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); - - // SBackendFileItem item; - // item.name = taosStrdup((char*)ROCKSDB_CURRENT); - // item.type = ROCKSDB_CURRENT_TYPE; - // taosArrayPush(list, &item); - - // pHandle->pBackendFile = pFile; - - // pHandle->pFileList = list; - // pHandle->currFileIdx = 0; - // pHandle->offset = 0; - *ppWriter = pWriter; return 0; } @@ -530,9 +501,10 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t pBackendFile->offset = 0; pBackendFile->currFileIdx += 1; - SBackendFileItem item; + SBackendFileItem item = {0}; item.name = taosStrdup(pHdr->name); item.type = pHdr->type; + taosArrayPush(pBackendFile->pFileList, &item); SBackendFileItem* pItem = taosArrayGet(pBackendFile->pFileList, pBackendFile->currFileIdx); @@ -557,26 +529,26 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa SStreamSnapHandle* pHandle = &pWriter->handle; SStreamTaskSnap snapInfo = pHdr->snapInfo; - SBackendSnapFile2* pBackendFile = taosArrayGet(pHandle->pBackendSnapSet, pHandle->currIdx); - if (pBackendFile->inited == 0) { - pBackendFile->snapInfo = snapInfo; - pBackendFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); - pBackendFile->currFileIdx = 0; - pBackendFile->offset = 0; + SBackendSnapFile2* pDbSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); + if (pDbSnapFile->inited == 0) { + pDbSnapFile->snapInfo = snapInfo; + pDbSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); + pDbSnapFile->currFileIdx = 0; + pDbSnapFile->offset = 0; - SBackendFileItem item; + SBackendFileItem item = {0}; item.name = taosStrdup((char*)ROCKSDB_CURRENT); item.type = ROCKSDB_CURRENT_TYPE; - taosArrayPush(pBackendFile->pFileList, &item); + taosArrayPush(pDbSnapFile->pFileList, &item); - pBackendFile->inited = 1; - return streamSnapWriteImpl(pWriter, pData, nData, pBackendFile); + pDbSnapFile->inited = 1; + return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); } else { - if (snapInfoEqual(&snapInfo, &pBackendFile->snapInfo)) { - return streamSnapWriteImpl(pWriter, pData, nData, pBackendFile); + if (snapInfoEqual(&snapInfo, &pDbSnapFile->snapInfo)) { + return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); } else { SBackendSnapFile2 snapFile = {0}; - taosArrayPush(pHandle->pBackendSnapSet, &snapFile); + taosArrayPush(pHandle->pDbSnapSet, &snapFile); pHandle->currIdx += 1; return streamSnapWrite(pWriter, pData, nData); @@ -585,8 +557,8 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa return code; } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { - SStreamSnapHandle* handle = &pWriter->handle; - streamSnapHandleDestroy(handle); + if (pWriter == NULL) return 0; + streamSnapHandleDestroy(&pWriter->handle); taosMemoryFree(pWriter); return 0; From f704d4b82eb5fed7e739642d31e1ab29a76574d3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 11:45:18 +0800 Subject: [PATCH 040/151] refactor backend --- source/libs/stream/src/streamSnapshot.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 16b0ae0406..1717a284bc 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -449,14 +449,14 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path if (pWriter == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - SBackendSnapFile2 snapFile = {0}; SStreamSnapHandle* pHandle = &pWriter->handle; - pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); - - taosArrayPush(pHandle->pDbSnapSet, &snapFile); pHandle->currIdx = 0; pHandle->metaPath = taosStrdup(path); + pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + + SBackendSnapFile2 snapFile = {0}; + taosArrayPush(pHandle->pDbSnapSet, &snapFile); *ppWriter = pWriter; return 0; From 861d26e356297994f01d5cc51b873d1d19fa8fad Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 15:24:50 +0800 Subject: [PATCH 041/151] refactor backend --- source/libs/stream/src/streamMeta.c | 14 ++------ source/libs/stream/src/streamSnapshot.c | 44 ++++++++++++++----------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f4c8021ac6..9d76951adb 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -203,7 +203,7 @@ _EXIT: streamBackendCleanup((void*)pBackend); return code; } -int32_t streamMetaMayConvertBackendFormat(SStreamMeta* pMeta) { +int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int8_t compatible = streamMetaCheckBackendCompatible(pMeta); if (compatible == STREAM_STATA_COMPATIBLE) { return 0; @@ -263,7 +263,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - if (streamMetaMayConvertBackendFormat(pMeta) < 0) { + if (streamMetaMayCvtDbFormat(pMeta) < 0) { goto _err; } @@ -759,16 +759,6 @@ static void doClear(void* pKey, void* pVal, TBC* pCur, SArray* pRecycleList) { int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta) { if (pMeta == NULL) return 0; - // void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); - // while (pIter) { - // STaskDbWrapper* taskBackend = *(STaskDbWrapper**)pIter; - // if (taskBackend != NULL) { - // taskDbRemoveRef(taskBackend); - // } - // pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); - // } - // taosHashClear(pMeta->pTaskBackendUnique); - return streamMetaLoadAllTasks(pMeta); } int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 1717a284bc..112589613f 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -134,6 +134,7 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) { TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { char fullname[256] = {0}; + STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname); return taosOpenFile(fullname, opt); } @@ -469,54 +470,53 @@ int32_t snapInfoEqual(SStreamTaskSnap* a, SStreamTaskSnap* b) { return 1; } -int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData, - SBackendSnapFile2* pBackendFile) { +int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nData, SBackendSnapFile2* pSnapFile) { int code = -1; SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData; SStreamSnapHandle* pHandle = &pWriter->handle; SStreamTaskSnap snapInfo = pHdr->snapInfo; - SStreamTaskSnap* pSnapInfo = &pBackendFile->snapInfo; + SStreamTaskSnap* pSnapInfo = &pSnapFile->snapInfo; - SBackendFileItem* pItem = taosArrayGet(pBackendFile->pFileList, pBackendFile->currFileIdx); + SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - if (pBackendFile->fd == 0) { - pBackendFile->fd = streamOpenFile(pHandle->metaPath, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pBackendFile->fd == NULL) { + if (pSnapFile->fd == 0) { + pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pSnapFile->fd == NULL) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pHandle->metaPath, TD_DIRSEP, pHdr->name, tstrerror(code)); } } if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { - int64_t bytes = taosPWriteFile(pBackendFile->fd, pHdr->data, pHdr->size, pBackendFile->offset); + int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); if (bytes != pHdr->size) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; } - pBackendFile->offset += bytes; + pSnapFile->offset += bytes; } else { - taosCloseFile(&pBackendFile->fd); - pBackendFile->offset = 0; - pBackendFile->currFileIdx += 1; + taosCloseFile(&pSnapFile->fd); + pSnapFile->offset = 0; + pSnapFile->currFileIdx += 1; SBackendFileItem item = {0}; item.name = taosStrdup(pHdr->name); item.type = pHdr->type; - taosArrayPush(pBackendFile->pFileList, &item); + taosArrayPush(pSnapFile->pFileList, &item); - SBackendFileItem* pItem = taosArrayGet(pBackendFile->pFileList, pBackendFile->currFileIdx); - pBackendFile->fd = streamOpenFile(pHandle->metaPath, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pBackendFile->fd == NULL) { + SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pSnapFile->fd == NULL) { code = TAOS_SYSTEM_ERROR(terrno); - qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pBackendFile->path, TD_DIRSEP, + qError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP, pHdr->name, tstrerror(code)); } - taosPWriteFile(pBackendFile->fd, pHdr->data, pHdr->size, pBackendFile->offset); - pBackendFile->offset += pHdr->size; + taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); + pSnapFile->offset += pHdr->size; } code = 0; _EXIT: @@ -531,6 +531,11 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa SBackendSnapFile2* pDbSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); if (pDbSnapFile->inited == 0) { + char* path = taosMemoryCalloc(1, strlen(pHandle->metaPath) + 256); + sprintf(path, "%s%s%" PRId64 "_%" PRId64 "%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, snapInfo.streamId, + snapInfo.taskId, TD_DIRSEP, "state", TD_DIRSEP, "checkpoints", TD_DIRSEP, snapInfo.chkpId); + + pDbSnapFile->path = path; pDbSnapFile->snapInfo = snapInfo; pDbSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); pDbSnapFile->currFileIdx = 0; @@ -539,6 +544,7 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa SBackendFileItem item = {0}; item.name = taosStrdup((char*)ROCKSDB_CURRENT); item.type = ROCKSDB_CURRENT_TYPE; + taosArrayPush(pDbSnapFile->pFileList, &item); pDbSnapFile->inited = 1; From a56d831a144a25b540b224d6ee635b2085fd79ee Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 20:16:41 +0800 Subject: [PATCH 042/151] refactor backend --- include/libs/stream/tstream.h | 4 +- source/dnode/vnode/src/tq/tq.c | 5 +- source/libs/stream/inc/streamBackendRocksdb.h | 8 +- source/libs/stream/src/streamBackendRocksdb.c | 84 ++++++++++++++++--- source/libs/stream/src/streamMeta.c | 21 +++-- source/libs/stream/src/streamSnapshot.c | 38 +++++---- source/libs/stream/src/streamState.c | 4 +- source/libs/stream/src/streamTask.c | 2 - 8 files changed, 120 insertions(+), 46 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 260609e697..1d4e87e2bd 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -410,7 +410,7 @@ typedef struct SStreamMeta { int32_t walScanCounter; void* streamBackend; int64_t streamBackendRid; - SHashObj* pTaskBackendUnique; + SHashObj* pTaskDbUnique; TdThreadMutex backendMutex; SMetaHbInfo hbInfo; int32_t closedTask; @@ -727,7 +727,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref); +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t chkpId, int64_t* ref); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3546df7e91..fbd8a3ee05 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -738,7 +738,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, ver); if (code != TSDB_CODE_SUCCESS) return code; - pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr, &pTask->backendRefId); + pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr, pTask->checkpointingId, + &pTask->backendRefId); + + // taskDbUpdateChkpId(pTask->pBackend, pTask->checkpointingId); if (pTask->pBackend == NULL) return -1; streamTaskOpenAllUpstreamInput(pTask); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 0f97bda4c5..2403bca7f6 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -83,12 +83,14 @@ int32_t streamBackendLoadCheckpointInfo(void* pMeta); int32_t streamBackendDoCheckpoint(void* pMeta, int64_t checkpointId); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); -int32_t streamStateConvertDataFormat(char* path, char* key, void* cfInst); +int32_t streamStateCvtDataFormat(char* path, char* key, void* cfInst); -STaskDbWrapper* taskDbOpen(char* path, char* key); +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); void taskDbDestroy(void* pBackend); int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); +void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId); + void* taskDbAddRef(void* pTaskDb); void taskDbRemoveRef(void* pTaskDb); @@ -182,7 +184,7 @@ int32_t streamBackendTriggerChkp(void* pMeta, char* dst); int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); -int32_t taskDbBuildSnap(void* arg, int64_t chkpId); +int32_t taskDbBuildSnap(void* arg); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 78f67d06d5..39c423b7fb 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -450,6 +450,60 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { return 0; } +int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) { + // impl later + int32_t code = 0; + + /*param@1: checkpointId dir + param@2: state + copy pChkpIdDir's file to state dir + opt to set hard link to previous file + */ + + char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128); + sprintf(prefixPath, "%s%s%s", path, TD_DIRSEP, key); + + if (!taosIsDir(prefixPath)) { + code = taosMkDir(prefixPath); + ASSERT(code == 0); + } + + char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256); + sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "defaultPath"); + if (!taosIsDir(defaultPath)) { + taosMulMkDir(defaultPath); + } + + if (chkpId != 0) { + char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); + sprintf(chkpPath, "%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); + if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { + if (taosIsDir(defaultPath)) { + // remove dir if exists + // taosRenameFile(const char *oldName, const char *newName) + taosRemoveDir(defaultPath); + } + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + if (code != 0) { + qError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + qInfo("start to restart stream backend at checkpoint path: %s", chkpPath); + } + + } else { + qError("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, + tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); + taosMkDir(defaultPath); + } + taosMemoryFree(chkpPath); + } + + *dbPath = defaultPath; + *dbPrefixPath = prefixPath; + + return 0; +} void* streamBackendInit(const char* streamPath, int64_t chkpId) { char* backendPath = NULL; @@ -829,13 +883,13 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { return 0; // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); - // void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + // void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); // while (pIter) { // int64_t id = *(int64_t*)pIter; // SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); // if (wrapper == NULL) { - // pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + // pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); // continue; // } @@ -940,7 +994,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return code; } - sprintf(pChkpIdDir, "%s%scheckpoint%" PRId64, pChkpDir, TD_DIRSEP, chkpId); + sprintf(pChkpIdDir, "%s%s%s%" PRId64, pChkpDir, TD_DIRSEP, "checkpoint", chkpId); if (taosIsDir(pChkpIdDir)) { qInfo("stream rm exist checkpoint%s", pChkpIdDir); taosRemoveFile(pChkpIdDir); @@ -950,19 +1004,19 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return 0; } -int32_t taskDbBuildSnap(void* arg, int64_t chkpId) { +int32_t taskDbBuildSnap(void* arg) { SStreamMeta* pMeta = arg; - void* pIter = taosHashIterate(pMeta->pTaskBackendUnique, NULL); + void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); int32_t code = 0; while (pIter) { STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter; taskDbAddRef(pTaskDb); - code = taskDbDoCheckpoint((STaskDbWrapper*)pTaskDb, chkpId); + code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId); taskDbRemoveRef(pTaskDb); - pIter = taosHashIterate(pMeta->pTaskBackendUnique, pIter); + pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); } return 0; } @@ -1070,6 +1124,8 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir); + pTaskDb->chkpId = chkpId; + _EXIT: taosMemoryFree(pChkpDir); taosMemoryFree(pChkpIdDir); @@ -1692,14 +1748,20 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta *stateFullPath = statePath; return 0; } -STaskDbWrapper* taskDbOpen(char* path, char* key) { +void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { + STaskDbWrapper* p = pTaskDb; + taosThreadMutexLock(&p->mutex); + p->chkpId = chkpId; + taosThreadMutexUnlock(&p->mutex); +} +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { char* statePath = NULL; char* err = NULL; char* dbPath = NULL; char** cfNames = NULL; size_t nCf = 0; - if (taskDbBuildFullPath(path, key, &dbPath, &statePath) != 0) { + if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { return NULL; } @@ -1861,12 +1923,12 @@ _EXIT: return code; } -int32_t streamStateConvertDataFormat(char* path, char* key, void* pCfInst) { +int32_t streamStateCvtDataFormat(char* path, char* key, void* pCfInst) { int nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); int32_t code = 0; - STaskDbWrapper* pTaskDb = taskDbOpen(path, key); + STaskDbWrapper* pTaskDb = taskDbOpen(path, key, 0); RocksdbCfInst* pSrcBackend = pCfInst; for (int i = 0; i < nCf; i++) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9d76951adb..016f446fe9 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -182,7 +182,7 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { return ret; } -int32_t streamMetaConvertBackendFormat(SStreamMeta* pMeta) { +int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { int32_t code = 0; int64_t chkpId = streamGetLatestCheckpointId(pMeta); SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId); @@ -190,7 +190,7 @@ int32_t streamMetaConvertBackendFormat(SStreamMeta* pMeta) { void* pIter = taosHashIterate(pBackend->cfInst, NULL); while (pIter) { void* key = taosHashGetKey(pIter, NULL); - code = streamStateConvertDataFormat(pMeta->path, key, *(void**)pIter); + code = streamStateCvtDataFormat(pMeta->path, key, *(void**)pIter); if (code != 0) { qError("failed to cvt data"); goto _EXIT; @@ -210,7 +210,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { } else if (compatible == STREAM_STATA_NEED_CONVERT) { qInfo("stream state need covert backend format"); - return streamMetaConvertBackendFormat(pMeta); + return streamMetaCvtDbFormat(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { qError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " @@ -222,9 +222,9 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { return 0; } -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) { +void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t chkpId, int64_t* ref) { taosThreadMutexLock(&pMeta->backendMutex); - void** ppBackend = taosHashGet(pMeta->pTaskBackendUnique, key, strlen(key)); + void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { taskDbAddRef(*ppBackend); *ref = ((STaskDbWrapper*)*ppBackend)->refId; @@ -232,7 +232,7 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) return *ppBackend; } - void* pBackend = taskDbOpen(pMeta->path, key); + void* pBackend = taskDbOpen(pMeta->path, key, chkpId); if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); return NULL; @@ -240,7 +240,7 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t* ref) *ref = taosAddRef(taskDbWrapperId, pBackend); - taosHashPut(pMeta->pTaskBackendUnique, key, strlen(key), &pBackend, sizeof(void*)); + taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexUnlock(&pMeta->backendMutex); return pBackend; } @@ -301,8 +301,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->hbInfo.tickCounter = 0; pMeta->hbInfo.stopFlag = 0; - pMeta->pTaskBackendUnique = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); // start backend // taosInitRWLatch(&pMeta->chkpDirLock); @@ -411,7 +410,7 @@ void streamMetaClear(SStreamMeta* pMeta) { taosRemoveRef(streamBackendId, pMeta->streamBackendRid); taosHashClear(pMeta->pTasks); - taosHashClear(pMeta->pTaskBackendUnique); + taosHashClear(pMeta->pTaskDbUnique); taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->chkpSaved); @@ -452,7 +451,7 @@ void streamMetaCloseImpl(void* arg) { taosArrayDestroy(pMeta->chkpInUse); taosHashCleanup(pMeta->pTasks); - taosHashCleanup(pMeta->pTaskBackendUnique); + taosHashCleanup(pMeta->pTaskDbUnique); taosMemoryFree(pMeta->path); taosThreadMutexDestroy(&pMeta->backendMutex); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 112589613f..676effe778 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -110,7 +110,7 @@ const char* ROCKSDB_CURRENT = "CURRENT"; const char* ROCKSDB_CHECKPOINT_META = "CHECKPOINT"; static int64_t kBlockSize = 64 * 1024; -int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, int64_t chkpId, void* pMeta); +int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, void* pMeta); void streamSnapHandleDestroy(SStreamSnapHandle* handle); // static void streamBuildFname(char* path, char* file, char* fullname) @@ -139,7 +139,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamBackendGetSnapInfo(void* arg, char* path, int64_t chkpId) { return taskDbBuildSnap(arg, chkpId); } +int32_t streamTaskDbGetSnapInfo(void* arg, char* path) { return taskDbBuildSnap(arg); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { @@ -235,20 +235,23 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { taosCloseDir(&pDir); return 0; } -int32_t streamBackendSnapInitFile(char* path, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { +int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { // SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); int32_t code = -1; - char* snapPath = taosMemoryCalloc(1, strlen(path) + 256); - sprintf(snapPath, "%s%s%" PRId64 "_%" PRId64 "%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, pSnap->streamId, - pSnap->taskId, TD_DIRSEP, "state", TD_DIRSEP, "checkpoints", TD_DIRSEP, pSnap->chkpId); - if (taosIsDir(snapPath)) { + char* path = taosMemoryCalloc(1, strlen(metaPath) + 256); + char idstr[64] = {0}; + sprintf(idstr, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)(pSnap->taskId)); + + sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", metaPath, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, + "checkpoint", pSnap->chkpId); + if (taosIsDir(path)) { goto _ERROR; } pSnapFile->pSst = taosArrayInit(16, sizeof(void*)); pSnapFile->pFileList = taosArrayInit(64, sizeof(SBackendFileItem)); - pSnapFile->path = snapPath; + pSnapFile->path = path; pSnapFile->snapInfo = *pSnap; if ((code = snapFileReadMeta(pSnapFile)) != 0) { goto _ERROR; @@ -262,7 +265,7 @@ int32_t streamBackendSnapInitFile(char* path, SStreamTaskSnap* pSnap, SBackendSn code = 0; _ERROR: - taosMemoryFree(snapPath); + taosMemoryFree(path); return code; } void snapFileDestroy(SBackendSnapFile2* pSnap) { @@ -288,11 +291,11 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) { return; } -int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId, void* pMeta) { +int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { // impl later SArray* pSnapSet = NULL; - int32_t code = streamBackendGetSnapInfo(pMeta, path, chkpId); + int32_t code = streamTaskDbGetSnapInfo(pMeta, path); if (code != 0) { return -1; } @@ -339,7 +342,7 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* pa return TSDB_CODE_OUT_OF_MEMORY; } - if (streamSnapHandleInit(&pReader->handle, (char*)path, chkpId, pMeta) < 0) { + if (streamSnapHandleInit(&pReader->handle, (char*)path, pMeta) < 0) { taosMemoryFree(pReader); return -1; } @@ -531,9 +534,16 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa SBackendSnapFile2* pDbSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); if (pDbSnapFile->inited == 0) { + char idstr[64] = {0}; + sprintf(idstr, "0x%" PRIx64 "-0x%x", snapInfo.streamId, (int32_t)(snapInfo.taskId)); + char* path = taosMemoryCalloc(1, strlen(pHandle->metaPath) + 256); - sprintf(path, "%s%s%" PRId64 "_%" PRId64 "%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, snapInfo.streamId, - snapInfo.taskId, TD_DIRSEP, "state", TD_DIRSEP, "checkpoints", TD_DIRSEP, snapInfo.chkpId); + sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", path, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, + "checkpoint", snapInfo.chkpId); + if (!taosIsDir(path)) { + code = taosMulMkDir(path); + ASSERT(code == 0); + } pDbSnapFile->path = path; pDbSnapFile->snapInfo = snapInfo; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index d4ac1838d0..b8158c774e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -116,7 +116,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz // taosWLockLatch(&pMeta->lock); // taosThreadMutexLock(&pMeta->backendMutex); // void* uniqueId = - // taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + // taosHashGet(pMeta->pTaskDbUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); // if (uniqueId == NULL) { // int code = streamStateOpenBackend(pMeta->streamBackend, pState); // if (code == -1) { @@ -124,7 +124,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz // taosMemoryFree(pState); // return NULL; // } - // taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, + // taosHashPut(pMeta->pTaskDbUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, // &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); // } else { // int64_t id = *(int64_t*)uniqueId; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 5b4b0c309d..8da515a650 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -392,8 +392,6 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); - // pTask->pBackend = taskDbOpen(pMeta->path, (char*)pTask->id.idStr); - return TSDB_CODE_SUCCESS; } From 0ba7e2278a9531b02640e8ac533f2d62c9a399c2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 20:21:11 +0800 Subject: [PATCH 043/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 39c423b7fb..d3e8a4e428 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -476,7 +476,7 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** if (chkpId != 0) { char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); - sprintf(chkpPath, "%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); + sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { if (taosIsDir(defaultPath)) { // remove dir if exists From 8c1a9f7c585f05387d191f98ae1b01b7357cf76f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 20:28:40 +0800 Subject: [PATCH 044/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d3e8a4e428..274b298445 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -469,7 +469,7 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** } char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256); - sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "defaultPath"); + sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "state"); if (!taosIsDir(defaultPath)) { taosMulMkDir(defaultPath); } From d6ff2820060b157b94bf47c1674a6a4cc59db964 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Oct 2023 20:42:36 +0800 Subject: [PATCH 045/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 274b298445..48e9d506d6 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1768,6 +1768,7 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); pTaskDb->idstr = taosStrdup(key); pTaskDb->path = statePath; + taosThreadMutexInit(&pTaskDb->mutex, NULL); taskDbInitChkpOpt(pTaskDb); taskDbInitDBOpt(pTaskDb); From 8392c99d36435173532dd61a9df7ffa2c374b078 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 13 Oct 2023 14:54:00 +0800 Subject: [PATCH 046/151] refactor backend --- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fbd8a3ee05..778a01719f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -748,14 +748,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SStreamTask* pStateTask = pTask; - SStreamTask task = {0}; - if (pTask->info.fillHistory) { - task.id = pTask->streamTaskId; - task.pMeta = pTask->pMeta; - pStateTask = &task; - } + // SStreamTask task = {0}; + // if (pTask->info.fillHistory) { + // task.id = pTask->streamTaskId; + // task.pMeta = pTask->pMeta; + // pStateTask = &task; + // } - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pStateTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId); return -1; @@ -782,14 +782,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { SStreamTask* pSateTask = pTask; - SStreamTask task = {0}; - if (pTask->info.fillHistory) { - task.id = pTask->streamTaskId; - task.pMeta = pTask->pMeta; - pSateTask = &task; - } + // SStreamTask task = {0}; + // if (pTask->info.fillHistory) { + // task.id = pTask->streamTaskId; + // task.pMeta = pTask->pMeta; + // pSateTask = &task; + // } - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId); return -1; From b6005d618351e9301a387aff6d9f836b529f5d51 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 13 Oct 2023 16:07:34 +0800 Subject: [PATCH 047/151] refactor backend --- .../executor/src/streamtimewindowoperator.c | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 1c909cb47d..cf558be1be 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1081,10 +1081,10 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera int32_t mapSize = 0; buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { - SWinKey key = {0}; + SWinKey key = {0}; buf = decodeSWinKey(buf, &key); SRowBuffPos* pPos = NULL; - int32_t resSize = pInfo->aggSup.resultRowSize; + int32_t resSize = pInfo->aggSup.resultRowSize; pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize); tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); } @@ -1388,10 +1388,12 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) { void* pBuf = NULL; int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; - taosMemoryFree(pBuf); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); - pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); + if (code == 0) { + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); + } } SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { @@ -2527,16 +2529,16 @@ void resetWinRange(STimeWindow* winRange) { void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); SResultWindowInfo winInfo = {0}; - int32_t size = 0; - void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); - int32_t num = size / sizeof(SSessionKey); - SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t num = size / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; @@ -2671,7 +2673,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - // for stream + // for stream void* buff = NULL; int32_t len = 0; int32_t res = @@ -2792,7 +2794,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { tSimpleHashCleanup(pInfo->pStUpdated); pInfo->pStUpdated = NULL; - if(pInfo->isHistoryOp) { + if (pInfo->isHistoryOp) { getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); } @@ -2826,8 +2828,9 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pOperator->operatorType = pPhyNode->type; if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, - destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState); } setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo, @@ -3738,8 +3741,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, - destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); pInfo->stateStore = pTaskInfo->storageAPI.stateStore; From 77542cc461f1b78da131b5d8f9f6bf2af190f9df Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 17 Oct 2023 11:23:39 +0800 Subject: [PATCH 048/151] fix stream snap deadlock --- source/libs/stream/inc/streamBackendRocksdb.h | 2 +- source/libs/stream/src/streamMeta.c | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 2403bca7f6..637fbf1017 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -65,8 +65,8 @@ typedef struct { rocksdb_compactionfilterfactory_t* filterFactory; TdThreadMutex mutex; char* idstr; - int64_t refId; char* path; + int64_t refId; int64_t chkpId; SArray* chkpSaved; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 016f446fe9..656b115bcf 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -237,8 +237,9 @@ void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t chkpI taosThreadMutexUnlock(&pMeta->backendMutex); return NULL; } - - *ref = taosAddRef(taskDbWrapperId, pBackend); + int64_t tref = taosAddRef(taskDbWrapperId, pBackend); + *ref = tref; + ((STaskDbWrapper*)pBackend)->refId = tref; taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexUnlock(&pMeta->backendMutex); @@ -824,8 +825,6 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) { taosArrayPush(pMeta->pTaskList, &pTask->id); } else { - tdbFree(pKey); - tdbFree(pVal); taosMemoryFree(pTask); continue; } From 2004c1a34681a67d433ccc9624da031fbf52540f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 17 Oct 2023 18:19:52 +0800 Subject: [PATCH 049/151] fix transfer crash --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/tq/tq.c | 8 ++---- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 3 -- source/libs/stream/inc/streamBackendRocksdb.h | 7 +++-- source/libs/stream/inc/streamInt.h | 1 + source/libs/stream/src/streamBackendRocksdb.c | 9 +++++- source/libs/stream/src/streamMeta.c | 28 +++++++++++++------ source/libs/stream/src/streamSnapshot.c | 27 ++++++++++-------- source/libs/stream/src/streamTask.c | 5 ++++ 9 files changed, 58 insertions(+), 32 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d4e87e2bd..04866cb3c4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -727,7 +727,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t chkpId, int64_t* ref); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 778a01719f..6f1675f528 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -738,11 +738,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, ver); if (code != TSDB_CODE_SUCCESS) return code; - pTask->pBackend = streamMetaGetBackendByTaskKey(pTq->pStreamMeta, (char*)pTask->id.idStr, pTask->checkpointingId, - &pTask->backendRefId); - - // taskDbUpdateChkpId(pTask->pBackend, pTask->checkpointingId); - if (pTask->pBackend == NULL) return -1; + // code = streamTaskSetDb(pTq->pStreamMeta, pTask); + // taskDbUpdateChkpId(pTask->pBackend, pTask->checkpointingId); + // if (pTask->pBackend == NULL) return -1; streamTaskOpenAllUpstreamInput(pTask); diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 7b3f1aac6d..566affa50d 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -242,11 +242,8 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t goto _err; } tDecoderClear(&decoder); - // tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) - taosWLockLatch(&pTq->pStreamMeta->lock); int64_t key[2] = {task.streamId, task.taskId}; - taosWLockLatch(&pTq->pStreamMeta->lock); if (tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn) < 0) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 637fbf1017..c108acce3e 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -66,8 +66,11 @@ typedef struct { TdThreadMutex mutex; char* idstr; char* path; - int64_t refId; + int64_t refId; + void* pTask; + int64_t streamId; + int64_t taskId; int64_t chkpId; SArray* chkpSaved; SArray* chkpInUse; @@ -184,7 +187,7 @@ int32_t streamBackendTriggerChkp(void* pMeta, char* dst); int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); -int32_t taskDbBuildSnap(void* arg); +int32_t taskDbBuildSnap(void* arg, SArray* pSnap); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index a5e7531f7e..801b1d6211 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -44,6 +44,7 @@ typedef struct { int64_t streamId; int64_t taskId; int64_t chkpId; + char* dbPrefixPath; } SStreamTaskSnap; extern SStreamGlobalEnv streamEnv; extern int32_t streamBackendId; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 48e9d506d6..8aa6878b10 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1004,7 +1004,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return 0; } -int32_t taskDbBuildSnap(void* arg) { +int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { SStreamMeta* pMeta = arg; void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); int32_t code = 0; @@ -1017,6 +1017,13 @@ int32_t taskDbBuildSnap(void* arg) { taskDbRemoveRef(pTaskDb); pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); + + SStreamTask* pTask = pTaskDb->pTask; + SStreamTaskSnap snap = {.streamId = pTask->id.streamId, + .taskId = pTask->id.taskId, + .chkpId = pTaskDb->chkpId, + .dbPrefixPath = taosStrdup(pTaskDb->path)}; + taosArrayPush(pSnap, &snap); } return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 656b115bcf..3ba360a4ae 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -222,28 +222,40 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { return 0; } -void* streamMetaGetBackendByTaskKey(SStreamMeta* pMeta, char* key, int64_t chkpId, int64_t* ref) { +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg) { + SStreamTask* pTask = arg; + + char* key = (char*)pTask->id.idStr; + int64_t chkpId = pTask->checkpointingId; + taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); if (ppBackend != NULL && *ppBackend != NULL) { taskDbAddRef(*ppBackend); - *ref = ((STaskDbWrapper*)*ppBackend)->refId; + + STaskDbWrapper* pBackend = *ppBackend; + + pTask->backendRefId = pBackend->refId; + pTask->pBackend = pBackend; taosThreadMutexUnlock(&pMeta->backendMutex); - return *ppBackend; + return 0; } - void* pBackend = taskDbOpen(pMeta->path, key, chkpId); + STaskDbWrapper* pBackend = taskDbOpen(pMeta->path, key, chkpId); if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); - return NULL; + return -1; } + int64_t tref = taosAddRef(taskDbWrapperId, pBackend); - *ref = tref; - ((STaskDbWrapper*)pBackend)->refId = tref; + pTask->backendRefId = tref; + pTask->pBackend = pBackend; + pBackend->refId = tref; + pBackend->pTask = pTask; taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexUnlock(&pMeta->backendMutex); - return pBackend; + return 0; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { int32_t code = -1; diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 676effe778..88e47e127b 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -139,7 +139,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamTaskDbGetSnapInfo(void* arg, char* path) { return taskDbBuildSnap(arg); } +int32_t streamTaskDbGetSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { @@ -236,16 +236,13 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { return 0; } int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { - // SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); int32_t code = -1; - char* path = taosMemoryCalloc(1, strlen(metaPath) + 256); - char idstr[64] = {0}; - sprintf(idstr, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)(pSnap->taskId)); - - sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", metaPath, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, - "checkpoint", pSnap->chkpId); - if (taosIsDir(path)) { + char* path = taosMemoryCalloc(1, strlen(pSnap->dbPrefixPath) + 256); + // char idstr[64] = {0}; + sprintf(path, "%s%s%s%s%s%" PRId64 "", pSnap->dbPrefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", + pSnap->chkpId); + if (!taosIsDir(path)) { goto _ERROR; } @@ -261,7 +258,7 @@ int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBacke } snapFileDebugInfo(pSnapFile); - + path = NULL; code = 0; _ERROR: @@ -294,8 +291,8 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) { int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { // impl later - SArray* pSnapSet = NULL; - int32_t code = streamTaskDbGetSnapInfo(pMeta, path); + SArray* pSnapSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); + int32_t code = streamTaskDbGetSnapInfo(pMeta, path, pSnapSet); if (code != 0) { return -1; } @@ -310,6 +307,11 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta ASSERT(code == 0); taosArrayPush(pDbSnapSet, &snapFile); } + for (int i = 0; i < taosArrayGetSize(pSnapSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); + taosMemoryFree(pSnap->dbPrefixPath); + } + taosArrayDestroy(pSnapSet); pHandle->pDbSnapSet = pDbSnapSet; pHandle->currIdx = 0; @@ -389,6 +391,7 @@ _NEXT: item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); } } + item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8da515a650..7295323ec1 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -356,6 +356,7 @@ void tFreeStreamTask(SStreamTask* pTask) { } if (pTask->pBackend) { taskDbRemoveRef(pTask->pBackend); + pTask->pBackend = NULL; } @@ -390,6 +391,10 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i streamTaskInitTokenBucket(&pTask->tokenBucket, 150, 100); taosThreadMutexInit(&pTask->lock, NULL); + + if (streamTaskSetDb(pMeta, pTask) != 0) { + return -1; + } streamTaskOpenAllUpstreamInput(pTask); return TSDB_CODE_SUCCESS; From d67fdb5e46f75bbda6699338ecaed08c09dec74d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 17 Oct 2023 21:08:31 +0800 Subject: [PATCH 050/151] fix transfer crash --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 +- source/libs/stream/src/streamSnapshot.c | 22 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index b2835f2198..b7c440dfa5 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -139,7 +139,7 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS pWriter->sver = sver; pWriter->ever = ever; - sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "received"); + sprintf(tdir, "%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM); taosMkDir(tdir); SStreamSnapWriter* pSnapWriter = NULL; diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 88e47e127b..70c3031eb2 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -393,8 +393,10 @@ _NEXT: } item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, - item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); + qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 + ", file no.%d, total set:%d, current set idx: %d", + STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx, + (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); @@ -422,9 +424,16 @@ _NEXT: if (pSnapFile->currFileIdx >= taosArrayGetSize(pSnapFile->pFileList)) { // finish - *ppData = NULL; - *size = 0; - return 0; + if (pHandle->currIdx + 1 < taosArrayGetSize(pHandle->pDbSnapSet)) { + // skip to next snap set + pHandle->currIdx += 1; + pSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); + goto _NEXT; + } else { + *ppData = NULL; + *size = 0; + return 0; + } } item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); pSnapFile->fd = streamOpenFile(pSnapFile->path, item->name, TD_FILE_READ); @@ -541,10 +550,11 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa sprintf(idstr, "0x%" PRIx64 "-0x%x", snapInfo.streamId, (int32_t)(snapInfo.taskId)); char* path = taosMemoryCalloc(1, strlen(pHandle->metaPath) + 256); - sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", path, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, + sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", pHandle->metaPath, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", snapInfo.chkpId); if (!taosIsDir(path)) { code = taosMulMkDir(path); + qInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); ASSERT(code == 0); } From e49b9b2276bb7b64b961d85015e70974daeaf2c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 11:34:11 +0800 Subject: [PATCH 051/151] fix transfer crash --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 10 +++---- source/libs/stream/src/streamBackendRocksdb.c | 26 +++++++++++++++++-- source/libs/stream/src/streamSnapshot.c | 11 +++++++- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index b7c440dfa5..885d12a6aa 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -167,6 +167,11 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) return code; } + +int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) { + tqDebug("vgId:%d, vnode %s snapshot write data", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); + return streamSnapWrite(pWriter->pWriterImpl, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); +} int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); int32_t code = streamStateLoadTasks(pWriter); @@ -178,8 +183,3 @@ int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) { return streamMetaReloadAllTasks(pWriter->pTq->pStreamMeta); } - -int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) { - tqDebug("vgId:%d, vnode %s snapshot write data", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); - return streamSnapWrite(pWriter->pWriterImpl, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); -} diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8aa6878b10..d1ddd5c81a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -492,8 +492,30 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** } } else { - qError("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, - tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); + qInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, + tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); + taosMkDir(defaultPath); + } + taosMemoryFree(chkpPath); + } else { + char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); + sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", + (int64_t)-1); + qInfo("no chkp id specified, try to restart from received chkp id -1, dir: %s", chkpPath); + if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { + if (taosIsDir(defaultPath)) { + taosRemoveDir(defaultPath); + } + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + if (code != 0) { + qError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + qInfo("start to restart stream backend at checkpoint path: %s", chkpPath); + } + } else { + qInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, + tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); taosMkDir(defaultPath); } taosMemoryFree(chkpPath); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 70c3031eb2..3993fe09a1 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -400,7 +400,13 @@ _NEXT: uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); - if (nread == -1) { + + qInfo("%s read impl %d, file name: %s", STREAM_STATE_TRANSFER, (int)nread, item->name); + if (nread == 0) { + code = TAOS_SYSTEM_ERROR(errno); + qError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, + item->type, tstrerror(code)); + } else if (nread == -1) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, item->type, tstrerror(code)); @@ -509,6 +515,8 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t code = TAOS_SYSTEM_ERROR(terrno); qError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; + } else { + qInfo("succ to write data %s", pItem->name); } pSnapFile->offset += bytes; } else { @@ -531,6 +539,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t } taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); + qInfo("succ to write data %s", pItem->name); pSnapFile->offset += pHdr->size; } code = 0; From 1b755dd4e79c5a1355f667ee7fba1cdc24671b0d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 14:22:33 +0800 Subject: [PATCH 052/151] fix transfer crash --- source/libs/stream/src/streamSnapshot.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 3993fe09a1..f192474a65 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -136,6 +136,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { char fullname[256] = {0}; STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname); + qInfo("stream open file %s", fullname); return taosOpenFile(fullname, opt); } @@ -399,14 +400,9 @@ _NEXT: (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); - int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); + int64_t nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); - qInfo("%s read impl %d, file name: %s", STREAM_STATE_TRANSFER, (int)nread, item->name); - if (nread == 0) { - code = TAOS_SYSTEM_ERROR(errno); - qError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, - item->type, tstrerror(code)); - } else if (nread == -1) { + if (nread == -1) { code = TAOS_SYSTEM_ERROR(terrno); qError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, item->type, tstrerror(code)); From d6840ae07fda355fa764fa166378bf0de9844b6d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 14:22:52 +0800 Subject: [PATCH 053/151] fix transfer crash --- source/libs/stream/src/streamSnapshot.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index f192474a65..5a6fdc314d 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -136,7 +136,6 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { char fullname[256] = {0}; STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname); - qInfo("stream open file %s", fullname); return taosOpenFile(fullname, opt); } From c6ba2ca20568ec5f943660bcb7c271b091f98645 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 Oct 2023 14:49:12 +0800 Subject: [PATCH 054/151] fix transfer crash --- source/libs/stream/src/streamBackendRocksdb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d1ddd5c81a..6b0275f44e 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -433,6 +433,7 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { } taosMkDir(state); code = copyFiles(chkp, state); + qInfo("copy snap file from %s to %s", chkp, state); if (code != 0) { qError("failed to restart stream backend from %s, reason: %s", chkp, tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { @@ -508,6 +509,7 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** } taosMkDir(defaultPath); code = copyFiles(chkpPath, defaultPath); + qInfo("copy snap file from %s to %s", chkpPath, defaultPath); if (code != 0) { qError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { @@ -1805,6 +1807,7 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { + qInfo("newly create db, need to restart"); // pre create db pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); rocksdb_close(pTaskDb->db); From 67dc52d2d624107e24d97e6657dc8e050124ea2e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 19 Oct 2023 15:19:52 +0800 Subject: [PATCH 055/151] refactor stream checkpoint --- source/dnode/mnode/impl/src/mndStream.c | 225 ++++++++++-------------- 1 file changed, 97 insertions(+), 128 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 049b4e737a..fab3a0f414 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -886,7 +886,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB); - //reuse this function for stream + // reuse this function for stream auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, "", detail); @@ -961,107 +961,102 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in return 0; } -// static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) { -// int64_t timestampMs = taosGetTimestampMs(); -// if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) { -// return -1; -// } +static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) { + int32_t code = -1; + int64_t timestampMs = taosGetTimestampMs(); + if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) { + return -1; + } -// STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, "stream-checkpoint"); -// if (pTrans == NULL) return -1; -// mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); -// if (mndTrancCheckConflict(pMnode, pTrans) != 0) { -// mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, -// checkpointId, -// tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); -// mndTransDrop(pTrans); -// return -1; -// } -// mDebug("start to trigger checkpoint for stream:%s, checkpoint: %" PRId64 "", pStream->name, checkpointId); -// atomic_store_64(&pStream->currentTick, 1); -// taosWLockLatch(&pStream->lock); -// // 1. redo action: broadcast checkpoint source msg for all source vg -// int32_t totLevel = taosArrayGetSize(pStream->tasks); -// for (int32_t i = 0; i < totLevel; i++) { -// SArray *pLevel = taosArrayGetP(pStream->tasks, i); -// SStreamTask *pTask = taosArrayGetP(pLevel, 0); -// if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { -// int32_t sz = taosArrayGetSize(pLevel); -// for (int32_t j = 0; j < sz; j++) { -// SStreamTask *pTask = taosArrayGetP(pLevel, j); -// /*A(pTask->info.nodeId > 0);*/ -// SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); -// if (pVgObj == NULL) { -// taosWUnLockLatch(&pStream->lock); -// mndTransDrop(pTrans); -// return -1; -// } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, "stream-checkpoint"); + if (pTrans == NULL) return -1; -// void *buf; -// int32_t tlen; -// if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, -// pTask->id.taskId) < 0) { -// mndReleaseVgroup(pMnode, pVgObj); -// taosWUnLockLatch(&pStream->lock); -// mndTransDrop(pTrans); -// return -1; -// } + mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, + tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); + goto _ERR; + } -// STransAction action = {0}; -// action.epSet = mndGetVgroupEpset(pMnode, pVgObj); -// action.pCont = buf; -// action.contLen = tlen; -// action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE; + mDebug("start to trigger checkpoint for stream:%s, checkpoint: %" PRId64 "", pStream->name, checkpointId); + taosWLockLatch(&pStream->lock); + pStream->currentTick = 1; + // 1. redo action: broadcast checkpoint source msg for all source vg -// mndReleaseVgroup(pMnode, pVgObj); + int32_t totLevel = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < totLevel; i++) { + SArray *pLevel = taosArrayGetP(pStream->tasks, i); + SStreamTask *pTask = taosArrayGetP(pLevel, 0); + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + int32_t sz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + /*A(pTask->info.nodeId > 0);*/ + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + if (pVgObj == NULL) { + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } -// if (mndTransAppendRedoAction(pTrans, &action) != 0) { -// taosMemoryFree(buf); -// taosWUnLockLatch(&pStream->lock); -// mndReleaseStream(pMnode, pStream); -// mndTransDrop(pTrans); -// return -1; -// } -// } -// } -// } -// // 2. reset tick -// pStream->checkpointFreq = checkpointId; -// pStream->checkpointId = checkpointId; -// pStream->checkpointFreq = taosGetTimestampMs(); -// atomic_store_64(&pStream->currentTick, 0); -// // 3. commit log: stream checkpoint info -// pStream->version = pStream->version + 1; -// taosWUnLockLatch(&pStream->lock); + void *buf; + int32_t tlen; + if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, + pTask->id.taskId) < 0) { + mndReleaseVgroup(pMnode, pVgObj); + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } -// // // code condtion + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE; -// SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); -// if (pCommitRaw == NULL) { -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { -// sdbFreeRaw(pCommitRaw); -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) { -// sdbFreeRaw(pCommitRaw); -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } + mndReleaseVgroup(pMnode, pVgObj); -// if (mndTransPrepare(pMnode, pTrans) != 0) { -// mError("failed to prepare trans rebalance since %s", terrstr()); -// goto _ERR; -// } -// mndTransDrop(pTrans); -// return 0; -// _ERR: -// mndTransDrop(pTrans); -// return -1; -// } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + taosWUnLockLatch(&pStream->lock); + goto _ERR; + } + } + } + } + // 2. reset tick + pStream->checkpointFreq = checkpointId; + pStream->checkpointId = checkpointId; + pStream->checkpointFreq = taosGetTimestampMs(); + pStream->currentTick = 0; + // 3. commit log: stream checkpoint info + pStream->version = pStream->version + 1; + taosWUnLockLatch(&pStream->lock); + + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); + if (pCommitRaw == NULL) { + mError("failed to prepare trans rebalance since %s", terrstr()); + goto _ERR; + } + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbFreeRaw(pCommitRaw); + mError("failed to prepare trans rebalance since %s", terrstr()); + goto _ERR; + } + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) { + sdbFreeRaw(pCommitRaw); + mError("failed to prepare trans rebalance since %s", terrstr()); + goto _ERR; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("failed to prepare trans rebalance since %s", terrstr()); + goto _ERR; + } + code = 0; +_ERR: + mndTransDrop(pTrans); + return code; +} static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream, SMnode *pMnode, int64_t checkpointId) { @@ -1111,8 +1106,8 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream pStream->checkpointId = checkpointId; pStream->checkpointFreq = taosGetTimestampMs(); - atomic_store_64(&pStream->currentTick, 0); - // 3. commit log: stream checkpoint info + pStream->currentTick = 0; + // 3. commit log: stream checkpoint info pStream->version = pStream->version + 1; taosWUnLockLatch(&pStream->lock); @@ -1212,43 +1207,17 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; int64_t checkpointId = pMsg->checkpointId; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, NULL, MND_STREAM_CHECKPOINT_NAME); - if (pTrans == NULL) { - mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return -1; - } - mDebug("start to trigger checkpoint, checkpointId: %" PRId64 "", checkpointId); - - const char *pDb = mndGetStreamDB(pMnode); - mndTransSetDbName(pTrans, pDb, "checkpoint"); - taosMemoryFree((void *)pDb); - - if (mndTransCheckConflict(pMnode, pTrans) != 0) { - mError("failed to trigger checkpoint, checkpointId: %" PRId64 ", reason:%s", checkpointId, - tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); - mndTransDrop(pTrans); - return -1; - } - while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - code = mndAddStreamCheckpointToTrans(pTrans, pStream, pMnode, checkpointId); + code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId); sdbRelease(pSdb, pStream); if (code == -1) { break; } } - if (code == 0) { - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("failed to prepre trans rebalance since %s", terrstr()); - } - } - - mndTransDrop(pTrans); return code; } @@ -1324,7 +1293,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB); - //reuse this function for stream + // reuse this function for stream auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, "", detail); @@ -2018,7 +1987,7 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) { const SEp *pEp = GET_ACTIVE_EP(pPrevEpset); - const SEp* p = GET_ACTIVE_EP(pCurrent); + const SEp *p = GET_ACTIVE_EP(pCurrent); if (pEp->port == p->port && strncmp(pEp->fqdn, p->fqdn, TSDB_FQDN_LEN) == 0) { return false; @@ -2287,7 +2256,7 @@ static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamVnodeRevertIndex *p // todo: this process should be executed by the write queue worker of the mnode int32_t mndProcessStreamHb(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode *pMnode = pReq->info.node; SStreamHbMsg req = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -2312,8 +2281,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { for (int32_t i = 0; i < req.numOfTasks; ++i) { STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); - int64_t k[2] = {p->streamId, p->taskId}; - int32_t *index = taosHashGet(execNodeList.pTaskMap, &k, sizeof(k)); + int64_t k[2] = {p->streamId, p->taskId}; + int32_t *index = taosHashGet(execNodeList.pTaskMap, &k, sizeof(k)); if (index == NULL) { continue; } From 1c3c58062d6e8df7372117be13b466a4f9de267a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 Oct 2023 15:40:47 +0800 Subject: [PATCH 056/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 79976a66c4..104289aed3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1064,6 +1064,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { .dbPrefixPath = taosStrdup(pTaskDb->path)}; taosArrayPush(pSnap, &snap); } + return code; } int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { // if (arg == NULL) return 0; From 9d210ec95716f6964db60bb138e310d01c12ca7c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Oct 2023 14:06:25 +0800 Subject: [PATCH 057/151] refactor stream backend --- source/dnode/mnode/impl/src/mndStream.c | 91 ++++++++++++------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3929e897e6..283d3fde38 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1119,36 +1119,30 @@ static const char *mndGetStreamDB(SMnode *pMnode) { return p; } -static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - SStreamObj *pStream = NULL; - int32_t code = 0; +static int32_t mndCheckNodeStatus(SMnode *pMnode) { + bool ready = true; + // check if the node update happens or not + int64_t ts = taosGetTimestampSec(); - { // check if the node update happens or not - int64_t ts = taosGetTimestampSec(); - - if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) { - if (execNodeList.pNodeEntryList != NULL) { - execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); - } - - execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode); + if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) { + if (execNodeList.pNodeEntryList != NULL) { + execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); } - if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) { - mDebug("stream task node change checking done, no vgroups exist, do nothing"); - execNodeList.ts = ts; - return 0; - } + execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode); + } - for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) { - SNodeEntry *pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i); - if (pNodeEntry->stageUpdated) { - mDebug("stream task not ready due to node update detected, checkpoint not issued"); - return 0; - } + if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) { + mDebug("stream task node change checking done, no vgroups exist, do nothing"); + execNodeList.ts = ts; + return -1; + } + + for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) { + SNodeEntry *pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i); + if (pNodeEntry->stageUpdated) { + mDebug("stream task not ready due to node update detected, checkpoint not issued"); + return -1; } SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode); @@ -1161,34 +1155,39 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { if (nodeUpdated) { mDebug("stream task not ready due to node update, checkpoint not issued"); - return 0; + return -1; } } - { // check if all tasks are in TASK_STATUS__NORMAL status - bool ready = true; + // check if all tasks are in TASK_STATUS__NORMAL status - taosThreadMutexLock(&execNodeList.lock); - for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) { - STaskId *p = taosArrayGet(execNodeList.pTaskList, i); - STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p)); - if (pEntry == NULL) { - continue; - } - - if (pEntry->status != TASK_STATUS__NORMAL) { - mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", - pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status)); - ready = false; - break; - } + taosThreadMutexLock(&execNodeList.lock); + for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) { + STaskId *p = taosArrayGet(execNodeList.pTaskList, i); + STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p)); + if (pEntry == NULL) { + continue; } - taosThreadMutexUnlock(&execNodeList.lock); - if (!ready) { - return 0; + if (pEntry->status != TASK_STATUS__NORMAL) { + mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", + pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status)); + ready = false; + break; } } + taosThreadMutexUnlock(&execNodeList.lock); + return ready == true ? 0 : -1; +} +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SStreamObj *pStream = NULL; + int32_t code = 0; + if ((code = mndCheckNodeStatus(pMnode)) != 0) { + return code; + } SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; int64_t checkpointId = pMsg->checkpointId; From ca1ffd584d2f5cf473754b65bb64af49e9d119d9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Oct 2023 16:45:17 +0800 Subject: [PATCH 058/151] refactor stream backend --- source/dnode/mnode/impl/src/mndStream.c | 50 ++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 283d3fde38..87a85c4086 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2450,32 +2450,34 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return TSDB_CODE_ACTION_IN_PROGRESS; } +int32_t mndFindTransByName(SMnode *pMnode, char *key, int32_t *transId) { + SSdb *pSdb = pMnode->pSdb; + STrans *pTrans = NULL; + void *pIter = NULL; + bool found = false; + while (1) { + pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans); + if (pIter == NULL) { + break; + } + + if (strncmp(pTrans->opername, key, tListLen(pTrans->opername) - 1) == 0) { + *transId = pTrans->id; + found = true; + sdbRelease(pSdb, pTrans); + sdbCancelFetch(pSdb, pIter); + break; + } + + sdbRelease(pSdb, pTrans); + } + return found ? 0 : -1; +} int32_t mndResetFromCheckpoint(SMnode *pMnode) { // find the checkpoint trans id int32_t transId = 0; - - { - SSdb *pSdb = pMnode->pSdb; - STrans *pTrans = NULL; - void *pIter = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans); - if (pIter == NULL) { - break; - } - - if (strncmp(pTrans->opername, MND_STREAM_CHECKPOINT_NAME, tListLen(pTrans->opername) - 1) == 0) { - transId = pTrans->id; - sdbRelease(pSdb, pTrans); - sdbCancelFetch(pSdb, pIter); - break; - } - - sdbRelease(pSdb, pTrans); - } - } - - if (transId == 0) { + int32_t code = mndFindTransByName(pMnode, MND_STREAM_CHECKPOINT_NAME, &transId); + if (code == -1) { mError("failed to find the checkpoint trans, reset not executed"); return TSDB_CODE_SUCCESS; } @@ -2492,7 +2494,6 @@ int32_t mndResetFromCheckpoint(SMnode *pMnode) { if (pIter == NULL) { break; } - mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); int32_t code = createStreamResetStatusTrans(pMnode, pStream); if (code != TSDB_CODE_SUCCESS) { @@ -2500,7 +2501,6 @@ int32_t mndResetFromCheckpoint(SMnode *pMnode) { return code; } } - return 0; } From 986a502019f1073a425e709db0a7d1ec17cf753b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Oct 2023 18:22:58 +0800 Subject: [PATCH 059/151] refactor stream backend --- source/dnode/mnode/impl/src/mndStream.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 87a85c4086..b1bcc401f8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2411,6 +2411,7 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + taosWUnLockLatch(&pStream->lock); return terrno; } @@ -2495,7 +2496,7 @@ int32_t mndResetFromCheckpoint(SMnode *pMnode) { break; } mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); - int32_t code = createStreamResetStatusTrans(pMnode, pStream); + code = createStreamResetStatusTrans(pMnode, pStream); if (code != TSDB_CODE_SUCCESS) { sdbCancelFetch(pSdb, pIter); return code; From d46cf878eb48241eacb77a2c073f26a04007ed2f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 24 Oct 2023 11:53:52 +0800 Subject: [PATCH 060/151] refactor stream backend --- source/dnode/mnode/impl/src/mndStream.c | 13 +++++++----- source/libs/stream/src/streamBackendRocksdb.c | 9 ++++++--- source/libs/stream/src/streamState.c | 20 ++++++++++++++++--- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b1bcc401f8..eb813c3d86 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1120,10 +1120,11 @@ static const char *mndGetStreamDB(SMnode *pMnode) { } static int32_t mndCheckNodeStatus(SMnode *pMnode) { - bool ready = true; + bool ready = false; // check if the node update happens or not int64_t ts = taosGetTimestampSec(); + taosThreadMutexLock(&execNodeList.lock); if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) { if (execNodeList.pNodeEntryList != NULL) { execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); @@ -1135,14 +1136,14 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execNodeList.ts = ts; - return -1; + goto _EXIT; } for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) { SNodeEntry *pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i); if (pNodeEntry->stageUpdated) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); - return -1; + goto _EXIT; } SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode); @@ -1155,13 +1156,12 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { if (nodeUpdated) { mDebug("stream task not ready due to node update, checkpoint not issued"); - return -1; + goto _EXIT; } } // check if all tasks are in TASK_STATUS__NORMAL status - taosThreadMutexLock(&execNodeList.lock); for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) { STaskId *p = taosArrayGet(execNodeList.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p)); @@ -1176,6 +1176,9 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { break; } } + ready = true; +_EXIT: + taosThreadMutexUnlock(&execNodeList.lock); return ready == true ? 0 : -1; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 104289aed3..4add9d2912 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2637,6 +2637,9 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { int code = 0; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + if (value == NULL || vLen == 0) { + stError("streamStateSessionPut_rocksdb val: %p, len: %d", value, vLen); + } STREAM_STATE_PUT_ROCKSDB(pState, "sess", &sKey, value, vLen); return code; } @@ -2685,11 +2688,11 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { if (code != 0) { return NULL; } + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); pCur->number = pState->number; - pCur->db = wrapper->rocksdb; + pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 8c44d3eeb2..218d906504 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -706,9 +706,23 @@ void streamStateFreeVal(void* val) { int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen) { #ifdef USE_ROCKSDB - qDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, - key->groupId); - return streamStateSessionPut_rocksdb(pState, key, value, vLen); + int32_t code = TSDB_CODE_SUCCESS; + SRowBuffPos* pos = (SRowBuffPos*)value; + if (pos->needFree) { + if (isFlushedState(pState->pFileState, key->win.ekey, 0)) { + if (!pos->pRowBuff) { + return code; + } + code = streamStateSessionPut_rocksdb(pState, key, pos->pRowBuff, vLen); + streamStateReleaseBuf(pState, pos, true); + putFreeBuff(pState->pFileState, pos); + stDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 ".code:%d", key->win.skey, + key->win.ekey, key->groupId, code); + } else { + code = putSessionWinResultBuff(pState->pFileState, value); + } + } + return code; #else SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, From b3c54a0f06afe0f9e7e5bb8b6dd043d6c54e4517 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 24 Oct 2023 16:04:43 +0800 Subject: [PATCH 061/151] refactor stream backend --- source/libs/stream/src/streamState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 218d906504..a368707cb1 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -757,7 +757,7 @@ int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { #ifdef USE_ROCKSDB qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, key->groupId); - return streamStateSessionDel_rocksdb(pState, key); + return deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); #else SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->pTdbState->txn); From d43b3b4a32b42757f0b16d62d37e485b725c430e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 24 Oct 2023 16:17:54 +0800 Subject: [PATCH 062/151] refactor stream backend --- source/libs/stream/src/streamState.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a368707cb1..32672d7b22 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -687,8 +687,7 @@ void streamStateResetCur(SStreamStateCur* pCur) { } void streamStateFreeCur(SStreamStateCur* pCur) { - if (!pCur || pCur->buffIndex >= 0) { - taosMemoryFree(pCur); + if (!pCur) { return; } qDebug("streamStateFreeCur"); From db0f6258fe8ba2fdbad27a15861fcd7c69221f5e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 25 Oct 2023 19:38:22 +0800 Subject: [PATCH 063/151] fix stream case error --- include/libs/stream/tstream.h | 4 ++-- source/dnode/vnode/src/sma/smaRollup.c | 10 ++++++---- source/dnode/vnode/src/tq/tq.c | 20 ++++++++----------- source/libs/stream/src/streamMeta.c | 4 ++-- source/libs/stream/src/streamState.c | 27 +++----------------------- source/libs/stream/src/streamTask.c | 11 ++++++++--- 6 files changed, 29 insertions(+), 47 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 713a457d52..dc24106575 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -774,7 +774,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char *key); void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaInitForSnode(SStreamMeta* pMeta); @@ -794,4 +794,4 @@ int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInf } #endif -#endif /* ifndef _STREAM_H_ */ +#endif /* ifndef _STREAM_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8da2fff5a6..fe458fdba3 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -256,9 +256,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat taosMemoryFree(s); } - SStreamTask task = {.id.taskId = 0, .id.streamId = 0}; // TODO: assign value - task.pMeta = pVnode->pTq->pStreamMeta; - pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1); + //SStreamTask task = {.id.taskId = 0, .id.streamId = 0}; // TODO: assign value + SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); + pTask->pMeta = pVnode->pTq->pStreamMeta; + + pStreamState = streamStateOpen(taskInfDir, pTask, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; return TSDB_CODE_FAILED; @@ -1429,4 +1431,4 @@ _exit: smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c18a58716c..d028989865 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -749,14 +749,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SStreamTask* pStateTask = pTask; - // SStreamTask task = {0}; - // if (pTask->info.fillHistory) { - // task.id.streamId = pTask->streamTaskId.streamId; - // task.id.taskId = pTask->streamTaskId.taskId; - // task.pMeta = pTask->pMeta; - // pStateTask = &task; - // } + // if (pTask->info.fillHistory) { + // pTask->id.streamId = pTask->streamTaskId.streamId; + // pTask->id.taskId = pTask->streamTaskId.taskId; + // } + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId); @@ -786,10 +784,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { SStreamTask* pSateTask = pTask; // SStreamTask task = {0}; // if (pTask->info.fillHistory) { - // task.id.streamId = pTask->streamTaskId.streamId; - // task.id.taskId = pTask->streamTaskId.taskId; - // task.pMeta = pTask->pMeta; - // pSateTask = &task; + // pTask->id.streamId = pTask->streamTaskId.streamId; + // pTask->id.taskId = pTask->streamTaskId.taskId; // } pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); @@ -1984,4 +1980,4 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; -} +} \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9ae4c2e043..7c616cba5b 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -226,10 +226,10 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { return 0; } -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg) { +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char *key) { SStreamTask* pTask = arg; - char* key = (char*)pTask->id.idStr; + //char* key = (char*)pTask->id.idStr; int64_t chkpId = pTask->checkpointingId; taosThreadMutexLock(&pMeta->backendMutex); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 32672d7b22..a6809b11dd 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -110,31 +110,10 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz pState->streamId = pStreamTask->id.streamId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); + #ifdef USE_ROCKSDB SStreamMeta* pMeta = pStreamTask->pMeta; - // pState->streamBackendRid = pMeta->streamBackendRid; - // taosWLockLatch(&pMeta->lock); - // taosThreadMutexLock(&pMeta->backendMutex); - // void* uniqueId = - // taosHashGet(pMeta->pTaskDbUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - // if (uniqueId == NULL) { - // int code = streamStateOpenBackend(pMeta->streamBackend, pState); - // if (code == -1) { - // taosThreadMutexUnlock(&pMeta->backendMutex); - // taosMemoryFree(pState); - // return NULL; - // } - // taosHashPut(pMeta->pTaskDbUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, - // &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); - // } else { - // int64_t id = *(int64_t*)uniqueId; - // pState->pTdbState->backendCfWrapperId = id; - // pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); - // // already exist stream task for - // qInfo("already exist stream-state for %s", pState->pTdbState->idstr); - // // taosAcquireRef(streamBackendId, pState->streamBackendRid); - // } - // taosThreadMutexUnlock(&pMeta->backendMutex); pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; @@ -1219,4 +1198,4 @@ char* streamStateIntervalDump(SStreamState* pState) { streamStateFreeCur(pCur); return dumpBuf; } -#endif +#endif \ No newline at end of file diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 15f3ba4a94..6d7a03bc6c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -443,9 +443,14 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i } taosThreadMutexInit(&pTask->lock, &attr); - if (streamTaskSetDb(pMeta, pTask) != 0) { - return -1; - } + // if (pTask->info.fillHistory == 1) { + // // + // } else { + + // } + // if (streamTaskSetDb(pMeta, pTask) != 0) { + // return -1; + // } streamTaskOpenAllUpstreamInput(pTask); return TSDB_CODE_SUCCESS; From 87db93e72ec3de889493ba85138989602829b26e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Oct 2023 11:36:29 +0800 Subject: [PATCH 064/151] fix stream case error --- source/libs/stream/src/streamMeta.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7c616cba5b..a2d3001968 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -229,7 +229,6 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char *key) { SStreamTask* pTask = arg; - //char* key = (char*)pTask->id.idStr; int64_t chkpId = pTask->checkpointingId; taosThreadMutexLock(&pMeta->backendMutex); From 4c03d372ef1886af9f0b68d0900fa2e8a1cdbdb6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Oct 2023 14:19:34 +0800 Subject: [PATCH 065/151] fix stream case error --- source/dnode/vnode/src/tq/tq.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d028989865..b1e5058ea9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -750,10 +750,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SStreamTask* pStateTask = pTask; - // if (pTask->info.fillHistory) { - // pTask->id.streamId = pTask->streamTaskId.streamId; - // pTask->id.taskId = pTask->streamTaskId.taskId; - // } + STaskId taskId = {.streamId = 0, .taskId = 0}; + if (pTask->info.fillHistory) { + + taskId.streamId = pTask->id.streamId; + taskId.taskId = pTask->id.taskId; + + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; + } pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { @@ -762,6 +767,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { } else { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } + if (pTask->info.fillHistory) { + pTask->id.streamId = taskId.streamId; + pTask->id.taskId = taskId.taskId; + } SReadHandle handle = { .checkpointId = pTask->chkInfo.checkpointId, @@ -783,10 +792,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { SStreamTask* pSateTask = pTask; // SStreamTask task = {0}; - // if (pTask->info.fillHistory) { - // pTask->id.streamId = pTask->streamTaskId.streamId; - // pTask->id.taskId = pTask->streamTaskId.taskId; - // } + + STaskId taskId = {.streamId = 0, .taskId = 0}; + if (pTask->info.fillHistory) { + taskId.streamId = pTask->id.streamId; + taskId.taskId = pTask->id.taskId; + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; + } pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); if (pTask->pState == NULL) { @@ -796,6 +809,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } + if (pTask->info.fillHistory) { + pTask->id.streamId = taskId.streamId; + pTask->id.taskId = taskId.taskId; + } + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); SReadHandle handle = { .checkpointId = pTask->chkInfo.checkpointId, From 7e1e68d8f5b679f871999194c03b104d18c687da Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Oct 2023 18:27:09 +0800 Subject: [PATCH 066/151] fix stream case error --- source/libs/stream/src/streamBackendRocksdb.c | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4add9d2912..f9b77b4e9a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2287,7 +2287,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, ttlVLen); \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, ttlVLen, wrapper); \ } \ taosMemoryFree(ttlV); \ } while (0); @@ -2328,7 +2328,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe funcname); \ code = -1; \ } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, wrapper); \ } \ taosMemoryFree(val); \ if (vLen != NULL) *vLen = tlen; \ @@ -2725,7 +2725,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta stDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -2766,7 +2766,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -2804,7 +2804,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -2907,7 +2907,7 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (pCur == NULL) return NULL; @@ -2968,7 +2968,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { return NULL; } @@ -3006,7 +3006,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -3044,7 +3044,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { stDebug("streamStateSessionGetKeyByRange_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return -1; } @@ -3314,7 +3314,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->db = wrapper->db; @@ -3437,4 +3437,4 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 8); x = x | (x >> 16); return x + 1; -} +} \ No newline at end of file From ee6b620597fea11be432af0deb3feb84332db718 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 30 Oct 2023 17:29:32 +0800 Subject: [PATCH 067/151] fix mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 16 +++++++++++++++- source/libs/stream/src/streamTask.c | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f9b77b4e9a..35b7b0c233 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1819,6 +1819,9 @@ _EXIT: void taskDbDestroy(void* pBackend) { STaskDbWrapper* wrapper = pBackend; + qDebug("succ to destroy stream backend:%p", wrapper); + + int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); if (wrapper == NULL) return; @@ -1826,7 +1829,6 @@ void taskDbDestroy(void* pBackend) { rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); rocksdb_flushoptions_set_wait(flushOpt, 1); - int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); char* err = NULL; for (int i = 0; i < nCf; i++) { if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); @@ -1849,7 +1851,19 @@ void taskDbDestroy(void* pBackend) { rocksdb_env_destroy(wrapper->env); rocksdb_cache_destroy(wrapper->cache); + taosMemoryFree(wrapper->pCf); + + for (int i = 0; i < nCf; i++) { + rocksdb_options_t *opt = wrapper->pCfOpts[i]; + rocksdb_comparator_t *compare = wrapper->pCompares[i]; + rocksdb_block_based_table_options_t *tblOpt = wrapper->pCfParams[i].tableOpt; + + rocksdb_options_destroy(opt); + rocksdb_comparator_destroy(compare); + rocksdb_block_based_options_destroy(tblOpt); + + } taosMemoryFree(wrapper->pCompares); taosMemoryFree(wrapper->pCfOpts); taosMemoryFree(wrapper->pCfParams); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 6d7a03bc6c..d4b9b39fa9 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -368,6 +368,8 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->pState) { stDebug("s-task:0x%x start to free task state", taskId); streamStateClose(pTask->pState, status == TASK_STATUS__DROPPING); + taskDbRemoveRef(pTask->pBackend); + } if (pTask->id.idStr != NULL) { From 7fad855247e7fd125116a996dd8c7263e1d4ed51 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 30 Oct 2023 18:16:12 +0800 Subject: [PATCH 068/151] fix mem leak --- source/libs/stream/src/streamState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a6809b11dd..16363e1bdd 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -108,7 +108,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz SStreamTask* pStreamTask = pTask; pState->taskId = pStreamTask->id.taskId; pState->streamId = pStreamTask->id.streamId; - sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId); streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); From bb8252df16f5dc74de8167a9abae946b12acaecc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Oct 2023 20:59:24 +0800 Subject: [PATCH 069/151] fix mem leak --- include/libs/stream/streamState.h | 3 ++- source/dnode/vnode/src/sma/smaRollup.c | 4 ++++ source/libs/stream/src/streamState.c | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index b7f100733b..17b766906c 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,6 +35,7 @@ int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); +int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); @@ -130,4 +131,4 @@ char* streamStateIntervalDump(SStreamState* pState); } #endif -#endif /* ifndef _STREAM_STATE_H_ */ +#endif /* ifndef _STREAM_STATE_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index fe458fdba3..5485ab88e3 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -89,7 +89,11 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } if (isDeepFree && pItem->pStreamState) { + //SStreamTask *pTask = pItem->pStreamState->pTdb + streamStateDelTaskDb(pItem->pStreamState); streamStateClose(pItem->pStreamState, false); + + } if (isDeepFree && pInfo->taskInfo[i]) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 16363e1bdd..e874f42b7a 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -209,6 +209,12 @@ _err: #endif } +int32_t streamStateDelTaskDb(SStreamState* pState) { + SStreamTask* pTask = pState->pTdbState->pOwner; + taskDbRemoveRef(pTask->pBackend); + taosMemoryFree(pTask); + return 0; +} void streamStateClose(SStreamState* pState, bool remove) { SStreamTask* pTask = pState->pTdbState->pOwner; #ifdef USE_ROCKSDB From 6cab686e82a9a46d29828e34f61456bfc0fa59e3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Nov 2023 09:37:54 +0800 Subject: [PATCH 070/151] fix mem leak --- source/dnode/snode/src/snode.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index ad28237bf4..85d25a4b4b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -60,7 +60,6 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer return code; } pTask->pBackend = NULL; - ASSERT(0); streamTaskOpenAllUpstreamInput(pTask); @@ -448,4 +447,4 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { ASSERT(0); } return 0; -} +} \ No newline at end of file From 1e96edcdd763af19e9f281ce303bd4b92f9a04fb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Nov 2023 17:30:46 +0800 Subject: [PATCH 071/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/dnode/mnode/impl/src/mndStream.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b6a15ae368..f8b976d90e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2532,15 +2532,24 @@ int32_t doKillActiveCheckpointTrans(SMnode *pMnode) { } sdbRelease(pSdb, pTrans); - } - return 0; + + if (transId == 0) { + mError("failed to find the checkpoint trans, reset not executed"); + return TSDB_CODE_SUCCESS; + } + + pTrans = mndAcquireTrans(pMnode, transId); + mInfo("kill checkpoint trans:%d", transId); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + return TSDB_CODE_SUCCESS; } int32_t mndResetFromCheckpoint(SMnode* pMnode) { doKillActiveCheckpointTrans(pMnode); - int32_t code = 0; // set all tasks status to be normal, refactor later to be stream level, instead of vnode level. SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; @@ -2553,7 +2562,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) { // todo this transaction should exist be only one mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); - code = createStreamResetStatusTrans(pMnode, pStream); + int32_t code = createStreamResetStatusTrans(pMnode, pStream); if (code != TSDB_CODE_SUCCESS) { sdbCancelFetch(pSdb, pIter); return code; From b9114922705895e6b3c3128d5a5c7d3201ecee1b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Nov 2023 09:46:36 +0800 Subject: [PATCH 072/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 41c9ca2361..a5bb9e59d8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -484,7 +484,7 @@ void streamMetaCloseImpl(void* arg) { taosHashCleanup(pMeta->pTaskDbUnique); taosHashCleanup(pMeta->pUpdateTaskSet); //taosHashCleanup(pMeta->pTaskBackendUnique); - //taosHashCleanup(pMeta->updateInfo.pTasks); + taosHashCleanup(pMeta->updateInfo.pTasks); taosHashCleanup(pMeta->startInfo.pReadyTaskSet); taosMemoryFree(pMeta->pHbInfo); From 7e7a39f077527d5c95bdc016a815fc344fc8417b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Nov 2023 10:15:34 +0800 Subject: [PATCH 073/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamMeta.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a5bb9e59d8..d12661c494 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -65,6 +65,7 @@ static void streamMetaEnvInit() { } void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } + void streamMetaCleanup() { taosCloseRef(streamBackendId); taosCloseRef(streamBackendCfWrapperId); @@ -319,15 +320,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->expandFunc = expandFunc; pMeta->stage = stage; - int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); - pMeta->rid = taosAddRef(streamMetaId, pMeta); - *pRid = pMeta->rid; - metaRefMgtAdd(pMeta->vgId, pRid); - - pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); - pMeta->pHbInfo->tickCounter = 0; - pMeta->pHbInfo->stopFlag = 0; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -347,6 +340,17 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->numOfStreamTasks = 0; stInfo("vgId:%d open stream meta successfully, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, stage); + + pMeta->rid = taosAddRef(streamMetaId, pMeta); + + int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); + memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); + metaRefMgtAdd(pMeta->vgId, pRid); + + pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); + pMeta->pHbInfo->tickCounter = 0; + pMeta->pHbInfo->stopFlag = 0; + return pMeta; _err: From 0a73832fd2ba492ed16610de6480c731eae3d7a8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Nov 2023 20:39:55 +0800 Subject: [PATCH 074/151] add checkpoint delta --- source/libs/stream/src/streamBackendRocksdb.c | 268 ++++++++++++------ 1 file changed, 188 insertions(+), 80 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 7271247ef2..bcfeb77ff9 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -20,6 +20,28 @@ #include "tcommon.h" #include "tref.h" +typedef struct SDbChkp { + int8_t init; + char* pCurrent; + char* pManifest; + SArray* pSST; + int64_t preCkptId; + int64_t curChkpId; + char* path; + + char* buf; + int32_t len; + + // ping-pong buf + SHashObj* pSstTbl[2]; + int8_t idx; + + SArray* pAdd; + SArray* pDel; + int8_t update; + + TdThreadRwlock rwLock; +} SDbChkp; typedef struct { int8_t init; char* pCurrent; @@ -39,6 +61,10 @@ typedef struct { SArray* pAdd; SArray* pDel; int8_t update; + + SHashObj *pDbChkpTbl; + + TdThreadRwlock rwLock; } SBackendManager; typedef struct SCompactFilteFactory { @@ -145,42 +171,23 @@ void destroyFunc(void* arg); int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); -SBackendManager* bkdMgtCreate(char* path) { - SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); - p->curChkpId = 0; - p->preCkptId = 0; - p->pSST = taosArrayInit(64, sizeof(void*)); - p->path = taosStrdup(path); - p->len = strlen(path) + 128; - p->buf = taosMemoryCalloc(1, p->len); - p->idx = 0; - p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); +void dbChkpDestroy(SDbChkp* pChkp) { + taosMemoryFree(pChkp->buf); + taosMemoryFree(pChkp->path); - p->pAdd = taosArrayInit(64, sizeof(void*)); - p->pDel = taosArrayInit(64, sizeof(void*)); - p->update = 0; - return p; + taosArrayDestroyP(pChkp->pSST, taosMemoryFree); + taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); + taosArrayDestroyP(pChkp->pDel, taosMemoryFree); + + taosHashCleanup(pChkp->pSstTbl[0]); + taosHashCleanup(pChkp->pSstTbl[1]); + + taosMemoryFree(pChkp->pCurrent); + taosMemoryFree(pChkp->pManifest); + } -void bkdMgtDestroy(SBackendManager* bm) { - if (bm == NULL) return; - taosMemoryFree(bm->buf); - taosMemoryFree(bm->path); - - taosArrayDestroyP(bm->pSST, taosMemoryFree); - taosArrayDestroyP(bm->pAdd, taosMemoryFree); - taosArrayDestroyP(bm->pDel, taosMemoryFree); - - taosHashCleanup(bm->pSstTbl[0]); - taosHashCleanup(bm->pSstTbl[1]); - - taosMemoryFree(bm->pCurrent); - taosMemoryFree(bm->pManifest); - - taosMemoryFree(bm); -} int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { int32_t code = 0; @@ -204,7 +211,12 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { return code; } -int32_t bkdMgtGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) { + +int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { + taosThreadRwlockWrlock(&p->rwLock); + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; const char* pCurrent = "CURRENT"; int32_t currLen = strlen(pCurrent); @@ -214,81 +226,112 @@ int32_t bkdMgtGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) { const char* pSST = ".sst"; int32_t sstLen = strlen(pSST); - memset(bm->buf, 0, bm->len); - sprintf(bm->buf, "%s%scheckpoint%" PRId64 "", bm->path, TD_DIRSEP, chkpId); + memset(p->buf, 0, p->len); + sprintf(p->buf, "%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, chkpId); - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); - TdDirPtr pDir = taosOpenDir(bm->buf); + TdDirPtr pDir = taosOpenDir(p->buf); TdDirEntryPtr de = NULL; int8_t dummy = 0; while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { - taosMemoryFreeClear(bm->pCurrent); - bm->pCurrent = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); + taosMemoryFreeClear(p->pCurrent); + p->pCurrent = taosStrdup(name); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { - taosMemoryFreeClear(bm->pManifest); - bm->pManifest = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); + taosMemoryFreeClear(p->pManifest); + p->pManifest = taosStrdup(name); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { // char* p = taosStrdup(name); - taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy)); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } } - if (bm->init == 0) { - bm->preCkptId = -1; - bm->curChkpId = chkpId; - bm->init = 1; + if (p->init == 0) { + p->preCkptId = -1; + p->curChkpId = chkpId; + p->init = 1; - void* pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], NULL); + void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { size_t len; char* name = taosHashGetKey(pIter, &len); if (name != NULL && len != 0) { - taosArrayPush(bm->pAdd, &name); + taosArrayPush(p->pAdd, &name); } - pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], pIter); + pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); } - if (taosArrayGetSize(bm->pAdd) > 0) bm->update = 1; + if (taosArrayGetSize(p->pAdd) > 0) p->update = 1; } else { - int32_t code = compareHashTable(bm->pSstTbl[bm->idx], bm->pSstTbl[1 - bm->idx], bm->pAdd, bm->pDel); + int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); if (code != 0) { // dead code - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); - taosHashClear(bm->pSstTbl[1 - bm->idx]); - bm->update = 0; + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + taosHashClear(p->pSstTbl[1 - p->idx]); + p->update = 0; taosCloseDir(&pDir); return code; } - bm->preCkptId = bm->curChkpId; - bm->curChkpId = chkpId; - if (taosArrayGetSize(bm->pAdd) == 0 && taosArrayGetSize(bm->pDel) == 0) { - bm->update = 0; + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; + if (taosArrayGetSize(p->pAdd) == 0 && taosArrayGetSize(p->pDel) == 0) { + p->update = 0; } } - taosHashClear(bm->pSstTbl[bm->idx]); - bm->idx = 1 - bm->idx; + taosHashClear(p->pSstTbl[p->idx]); + p->idx = 1 - p->idx; taosCloseDir(&pDir); + taosThreadRwlockUnlock(&p->rwLock); return 0; + +} +SDbChkp* dbChktCreate(char* path, int64_t initChkpId) { + SDbChkp *p = taosMemoryCalloc(1, sizeof(SDbChkp)); + p->curChkpId = initChkpId; + p->preCkptId = -1; + p->pSST = taosArrayInit(64, sizeof(void*)); + p->path = taosStrdup(path); + p->len = strlen(path) + 128; + p->buf = taosMemoryCalloc(1, p->len); + + p->idx = 0; + p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + + p->pAdd = taosArrayInit(64, sizeof(void*)); + p->pDel = taosArrayInit(64, sizeof(void*)); + p->update = 0; + taosThreadRwlockInit(&p->rwLock, NULL); + + SArray *list = NULL; + int32_t code = dbChkpGetDelta(p, initChkpId, list); + + return p; } -int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) { +int32_t dbChkpInit(SDbChkp* p) { + if (p == NULL) return 0; + return 0; +} +int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { + + taosThreadRwlockRdlock(&p->rwLock); int32_t code = 0; - int32_t len = bm->len + 128; + int32_t len = p->len + 128; char* srcBuf = taosMemoryCalloc(1, len); char* dstBuf = taosMemoryCalloc(1, len); @@ -296,8 +339,8 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) { char* srcDir = taosMemoryCalloc(1, len); char* dstDir = taosMemoryCalloc(1, len); - sprintf(srcDir, "%s%s%s%" PRId64 "", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId); - sprintf(dstDir, "%s%s%s", bm->path, TD_DIRSEP, dname); + sprintf(srcDir, "%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoint", p->curChkpId); + sprintf(dstDir, "%s%s%s", p->path, TD_DIRSEP, dname); if (!taosDirExist(srcDir)) { stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); @@ -314,30 +357,30 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) { // clear current file memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); taosRemoveFile(dstBuf); memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); taosRemoveFile(dstBuf); // add file to $name dir - for (int i = 0; i < taosArrayGetSize(bm->pAdd); i++) { + for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { memset(dstBuf, 0, len); memset(srcBuf, 0, len); - char* filename = taosArrayGetP(bm->pAdd, i); + char* filename = taosArrayGetP(p->pAdd, i); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); taosCopyFile(srcBuf, dstBuf); } // del file in $name - for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) { + for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { memset(dstBuf, 0, len); memset(srcBuf, 0, len); - char* filename = taosArrayGetP(bm->pDel, i); + char* filename = taosArrayGetP(p->pDel, i); sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); taosRemoveFile(dstBuf); } @@ -345,27 +388,92 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) { // copy current file to dst dir memset(srcBuf, 0, len); memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pCurrent); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); taosCopyFile(srcBuf, dstBuf); // copy manifest file to dst dir memset(srcBuf, 0, len); memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pManifest); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); taosCopyFile(srcBuf, dstBuf); // clear delta data buf - taosArrayClearP(bm->pAdd, taosMemoryFree); - taosArrayClearP(bm->pDel, taosMemoryFree); + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); _ERROR: + taosThreadRwlockUnlock(&p->rwLock); taosMemoryFree(srcBuf); taosMemoryFree(dstBuf); taosMemoryFree(srcDir); taosMemoryFree(dstDir); return code; + +} +SBackendManager* bkdMgtCreate(char* path) { + SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); + p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + taosThreadRwlockInit(&p->rwLock, NULL); + return p; +} + +void bkdMgtDestroy(SBackendManager* bm) { + if (bm == NULL) return; + void *pIter = taosHashIterate(bm->pDbChkpTbl, NULL); + while (pIter) { + SDbChkp *pChkp = *(SDbChkp **)(pIter); + dbChkpDestroy(pChkp); + + pIter = taosHashIterate(bm->pDbChkpTbl, pIter); + } + + taosThreadRwlockDestroy(&bm->rwLock); + + taosMemoryFree(bm); +} +int32_t bkdMgtGetDelta(SBackendManager* bm, char *taskId, int64_t chkpId, SArray* list) { + int32_t code = 0; + taosThreadRwlockWrlock(&bm->rwLock); + + SDbChkp *pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpGetDelta(pChkp, chkpId, list); + + taosThreadRwlockUnlock(&bm->rwLock); + return code ; +} + +int32_t bkdMgtAddChkp(SBackendManager *bm, char *task, char *path) { + int32_t code = -1; + + taosThreadRwlockWrlock(&bm->rwLock); + SDbChkp **pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); + if (pp == NULL) { + SDbChkp *p = dbChktCreate(path, 0); + if (p != NULL) { + taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void *)); + code = 0; + } + } else { + stError("task chkp already exists"); + } + + taosThreadRwlockUnlock(&bm->rwLock); + + return code; +} + +int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) { + int32_t code = 0; + taosThreadRwlockRdlock(&bm->rwLock); + + SDbChkp *p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpDumpTo(p, dname); + + taosThreadRwlockUnlock(&bm->rwLock); + return code; + } SCfInit ginitDict[] = { From 069dd8de84c2a1759b7a01e0fea5647ca99aa25f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Nov 2023 09:28:30 +0800 Subject: [PATCH 075/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index bcfeb77ff9..589627c41a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -39,7 +39,7 @@ typedef struct SDbChkp { SArray* pAdd; SArray* pDel; int8_t update; - + TdThreadRwlock rwLock; } SDbChkp; typedef struct { From d82d51cf8f646892a2760da8c2353a611e43dd48 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Nov 2023 10:40:00 +0800 Subject: [PATCH 076/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 589627c41a..664802bd8a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -258,9 +258,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { } } if (p->init == 0) { - p->preCkptId = -1; - p->curChkpId = chkpId; - p->init = 1; void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { @@ -272,6 +269,10 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); } if (taosArrayGetSize(p->pAdd) > 0) p->update = 1; + + p->init = 1; + p->preCkptId = -1; + p->curChkpId = chkpId; } else { int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); if (code != 0) { @@ -284,12 +285,15 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { return code; } - p->preCkptId = p->curChkpId; - p->curChkpId = chkpId; if (taosArrayGetSize(p->pAdd) == 0 && taosArrayGetSize(p->pDel) == 0) { p->update = 0; } + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; } + + taosHashClear(p->pSstTbl[p->idx]); p->idx = 1 - p->idx; From feeead50c5343359244ad63fab5818f805db0c50 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 13:26:38 +0800 Subject: [PATCH 077/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 664802bd8a..797329d52b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1691,7 +1691,7 @@ const char* compactFilteName(void* arg) { return "stream_filte"; } rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { SCompactFilteFactory* state = arg; rocksdb_compactionfilter_t* filter = - rocksdb_compactionfilter_create(NULL, destroyCompactFilte, compactFilte, compactFilteName); + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilte, compactFilteName); return filter; } @@ -1773,7 +1773,7 @@ void taskDbRemoveRef(void* pTaskDb) { } // void taskDbDestroy(STaskDbWrapper* wrapper); -void taskDbInitDBOpt(STaskDbWrapper* pTaskDb) { +void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_env_t* env = rocksdb_create_default_env(); rocksdb_cache_t* cache = rocksdb_cache_create_lru(256); @@ -1895,7 +1895,7 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { taosThreadMutexInit(&pTaskDb->mutex, NULL); taskDbInitChkpOpt(pTaskDb); - taskDbInitDBOpt(pTaskDb); + taskDbInitOpt(pTaskDb); statePath = NULL; cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); From ba52769867133e60984e745cbb8e287717a9ff0a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 17:56:09 +0800 Subject: [PATCH 078/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 797329d52b..efb9bb599c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -93,10 +93,18 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst); void destroyCompactFilteFactory(void* arg); void destroyCompactFilte(void* arg); const char* compactFilteFactoryName(void* arg); +const char* compactFilteFactoryNameSess(void* arg); +const char* compactFilteFactoryNameState(void* arg); +const char* compactFilteFactoryNameFunc(void* arg); +const char* compactFilteFactoryNameFill(void* arg); const char* compactFilteName(void* arg); unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterSess(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; @@ -303,7 +311,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { return 0; } -SDbChkp* dbChktCreate(char* path, int64_t initChkpId) { +SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { SDbChkp *p = taosMemoryCalloc(1, sizeof(SDbChkp)); p->curChkpId = initChkpId; p->preCkptId = -1; @@ -454,7 +462,7 @@ int32_t bkdMgtAddChkp(SBackendManager *bm, char *task, char *path) { taosThreadRwlockWrlock(&bm->rwLock); SDbChkp **pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); if (pp == NULL) { - SDbChkp *p = dbChktCreate(path, 0); + SDbChkp *p = dbChkpCreate(path, 0); if (p != NULL) { taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void *)); code = 0; @@ -497,6 +505,40 @@ SCfInit ginitDict[] = { encodeValueFunc, decodeValueFunc}, }; + +typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); +typedef const char* (*FactoryNameFunc)(void* arg); +typedef void(*DestroyFactoryFunc)(void *arg); + +typedef struct { + void *funcName; + + DestroyFactoryFunc destroy; + CreateFactoryFunc create; + FactoryNameFunc factoryName; +} SCfFilterFactory; + +SCfFilterFactory ginitFilterDict[] = { + {"default", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, + {"state", destroyCompactFilteFactory,compactFilteFactoryCreateFilterState,compactFilteFactoryNameState}, + {"fill", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFill,compactFilteFactoryNameFill}, + {"sess", destroyCompactFilteFactory,compactFilteFactoryCreateFilterSess,compactFilteFactoryNameSess}, + {"func", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFunc,compactFilteFactoryNameFunc}, + {"parname", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, + {"partag", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, +}; + + + +// pTaskDb->filterFactory = rocksdb_compactionfilterfactory_create( +// NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); +// rocksdb_options_set_compaction_filter_factory(pTaskDb->dbOpt, pTaskDb->filterFactory); +void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { + rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,destroyCompactFilteFactory, ginitFilterDict[i].create, ginitFilterDict[i].funcName); + rocksdb_options_set_compaction_filter_factory(opt, filterFactory); + +} + bool isValidCheckpoint(const char* dir) { return true; } int32_t copyFiles(const char* src, const char* dst) { @@ -1680,6 +1722,22 @@ const char* compactFilteFactoryName(void* arg) { SCompactFilteFactory* state = arg; return "stream_compact_filter"; } +const char* compactFilteFactoryNameSess(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_filter_sess"; +} +const char* compactFilteFactoryNameState(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_filter_state"; +} +const char* compactFilteFactoryNameFill(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_filter_fill"; +} +const char* compactFilteFactoryNameFunc(void* arg) { + SCompactFilteFactory* state = arg; + return "stream_compact_filter_func"; +} void destroyCompactFilte(void* arg) { (void)arg; } unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, @@ -1688,12 +1746,69 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c } const char* compactFilteName(void* arg) { return "stream_filte"; } +unsigned char compactFilteSess(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; + //return streamStateValueIsStale((char*)val) ? 1 : 0; +} +const char* compactFilteNameSess(void* arg) { return "stream_filte_sess"; } + +unsigned char compactFilteState(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; + //return streamStateValueIsStale((char*)val) ? 1 : 0; +} +const char* compactFilteNameState(void* arg) { return "stream_filte_state"; } + +unsigned char compactFilteFill(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; + //return streamStateValueIsStale((char*)val) ? 1 : 0; +} +const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } + +unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, + char** newval, size_t* newvlen, unsigned char* value_changed) { + // not impl yet + return 0; + //return streamStateValueIsStale((char*)val) ? 1 : 0; +} +const char* compactFilteNameFunc(void* arg) { return "stream_filte_func"; } + + rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { SCompactFilteFactory* state = arg; rocksdb_compactionfilter_t* filter = rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilte, compactFilteName); return filter; } +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterSess(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteSess, compactFilteNameSess); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteState, compactFilteNameState); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteFill, compactFilteNameFill); + return filter; +} +rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx) { + SCompactFilteFactory* state = arg; + rocksdb_compactionfilter_t* filter = + rocksdb_compactionfilter_create(state, destroyCompactFilte, compactFilteFunc, compactFilteNameFunc); + return filter; +} void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -1820,10 +1935,12 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); - + pTaskDb->pCompares[i] = compare; pTaskDb->pCfOpts[i] = opt; pTaskDb->pCfParams[i].tableOpt = tableOpt; + // set filter factory + dbSetFilterFactory(opt, i, NULL); } return; } From aee1b700a6864d4182efc01d9e185afe2959f6c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 18:08:03 +0800 Subject: [PATCH 079/151] Merge branch 'enh/new3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index efb9bb599c..60fd32f1a5 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -530,9 +530,6 @@ SCfFilterFactory ginitFilterDict[] = { -// pTaskDb->filterFactory = rocksdb_compactionfilterfactory_create( -// NULL, destroyCompactFilteFactory, compactFilteFactoryCreateFilter, compactFilteFactoryName); -// rocksdb_options_set_compaction_filter_factory(pTaskDb->dbOpt, pTaskDb->filterFactory); void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,destroyCompactFilteFactory, ginitFilterDict[i].create, ginitFilterDict[i].funcName); rocksdb_options_set_compaction_filter_factory(opt, filterFactory); @@ -1939,6 +1936,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { pTaskDb->pCompares[i] = compare; pTaskDb->pCfOpts[i] = opt; pTaskDb->pCfParams[i].tableOpt = tableOpt; + // set filter factory dbSetFilterFactory(opt, i, NULL); } From 9ca1151b6768d41c0e6973b5969aaa32e6924697 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 19:47:47 +0800 Subject: [PATCH 080/151] refactor checkpoint --- source/libs/stream/src/streamBackendRocksdb.c | 158 +++++++++--------- 1 file changed, 80 insertions(+), 78 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 60fd32f1a5..e674f2b78c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -20,6 +20,8 @@ #include "tcommon.h" #include "tref.h" +#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); + typedef struct SDbChkp { int8_t init; char* pCurrent; @@ -106,7 +108,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rock rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); -const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; typedef int (*EncodeFunc)(void* key, char* buf); typedef int (*DecodeFunc)(void* key, char* buf); @@ -131,7 +132,18 @@ typedef struct { } SCfInit; -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); +typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); +typedef const char* (*FactoryNameFunc)(void* arg); +typedef void(*DestroyFactoryFunc)(void *arg); + +void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg); +typedef struct { + void *funcName; + DestroyFactoryFunc destroy; + CreateFactoryFunc create; + FactoryNameFunc factoryName; +} SCfFilterFactory; + const char* compareDefaultName(void* name); const char* compareStateName(void* name); const char* compareWinKeyName(void* name); @@ -180,21 +192,36 @@ int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); -void dbChkpDestroy(SDbChkp* pChkp) { - taosMemoryFree(pChkp->buf); - taosMemoryFree(pChkp->path); +static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); - taosArrayDestroyP(pChkp->pSST, taosMemoryFree); - taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); - taosArrayDestroyP(pChkp->pDel, taosMemoryFree); +SCfInit ginitDict[] = { + {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, + destroyFunc, encodeValueFunc, decodeValueFunc}, + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, + compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, +}; - taosHashCleanup(pChkp->pSstTbl[0]); - taosHashCleanup(pChkp->pSstTbl[1]); +SCfFilterFactory ginitFilterDict[] = { + {"default", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, + {"state", destroyCompactFilteFactory,compactFilteFactoryCreateFilterState,compactFilteFactoryNameState}, + {"fill", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFill,compactFilteFactoryNameFill}, + {"sess", destroyCompactFilteFactory,compactFilteFactoryCreateFilterSess,compactFilteFactoryNameSess}, + {"func", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFunc,compactFilteFactoryNameFunc}, + {"parname", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, + {"partag", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, +}; - taosMemoryFree(pChkp->pCurrent); - taosMemoryFree(pChkp->pManifest); - -} +const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { @@ -335,6 +362,22 @@ SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { return p; } +void dbChkpDestroy(SDbChkp* pChkp) { + taosMemoryFree(pChkp->buf); + taosMemoryFree(pChkp->path); + + taosArrayDestroyP(pChkp->pSST, taosMemoryFree); + taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); + taosArrayDestroyP(pChkp->pDel, taosMemoryFree); + + taosHashCleanup(pChkp->pSstTbl[0]); + taosHashCleanup(pChkp->pSstTbl[1]); + + taosMemoryFree(pChkp->pCurrent); + taosMemoryFree(pChkp->pManifest); + +} + int32_t dbChkpInit(SDbChkp* p) { if (p == NULL) return 0; return 0; @@ -488,48 +531,6 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) { } -SCfInit ginitDict[] = { - {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, -}; - - -typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef const char* (*FactoryNameFunc)(void* arg); -typedef void(*DestroyFactoryFunc)(void *arg); - -typedef struct { - void *funcName; - - DestroyFactoryFunc destroy; - CreateFactoryFunc create; - FactoryNameFunc factoryName; -} SCfFilterFactory; - -SCfFilterFactory ginitFilterDict[] = { - {"default", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, - {"state", destroyCompactFilteFactory,compactFilteFactoryCreateFilterState,compactFilteFactoryNameState}, - {"fill", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFill,compactFilteFactoryNameFill}, - {"sess", destroyCompactFilteFactory,compactFilteFactoryCreateFilterSess,compactFilteFactoryNameSess}, - {"func", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFunc,compactFilteFactoryNameFunc}, - {"parname", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, - {"partag", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, -}; - - - void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,destroyCompactFilteFactory, ginitFilterDict[i].create, ginitFilterDict[i].funcName); rocksdb_options_set_compaction_filter_factory(opt, filterFactory); @@ -1309,7 +1310,28 @@ void streamBackendDelCompare(void* backend, void* arg) { } } void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } -static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); +void destroyRocksdbCfInst(RocksdbCfInst* inst) { + int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + if (inst->pHandle) { + for (int i = 0; i < cfLen; i++) { + if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); + } + taosMemoryFree(inst->pHandle); + } + + if (inst->cfOpt) { + for (int i = 0; i < cfLen; i++) { + rocksdb_options_destroy(inst->cfOpt[i]); + rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt); + } + taosMemoryFreeClear(inst->cfOpt); + taosMemoryFreeClear(inst->param); + } + if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt); + if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt); + + taosMemoryFree(inst); +} // |key|-----value------| // |key|ttl|len|userData| @@ -1736,6 +1758,7 @@ const char* compactFilteFactoryNameFunc(void* arg) { return "stream_compact_filter_func"; } + void destroyCompactFilte(void* arg) { (void)arg; } unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1807,28 +1830,7 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocks return filter; } -void destroyRocksdbCfInst(RocksdbCfInst* inst) { - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - if (inst->pHandle) { - for (int i = 0; i < cfLen; i++) { - if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); - } - taosMemoryFree(inst->pHandle); - } - if (inst->cfOpt) { - for (int i = 0; i < cfLen; i++) { - rocksdb_options_destroy(inst->cfOpt[i]); - rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt); - } - taosMemoryFreeClear(inst->cfOpt); - taosMemoryFreeClear(inst->param); - } - if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt); - if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt); - - taosMemoryFree(inst); -} int32_t getCfIdx(const char* cfName) { int idx = -1; From 819be65db0a06d1fe6200bfcf3b2a6aabf1dd883 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 19:59:11 +0800 Subject: [PATCH 081/151] refactor checkpoint --- source/libs/stream/src/streamBackendRocksdb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index e674f2b78c..5e00a39417 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -534,7 +534,6 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) { void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,destroyCompactFilteFactory, ginitFilterDict[i].create, ginitFilterDict[i].funcName); rocksdb_options_set_compaction_filter_factory(opt, filterFactory); - } bool isValidCheckpoint(const char* dir) { return true; } From c9bf4f65071102ab682ea9a7bfb93d1766a62015 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 6 Nov 2023 20:19:11 +0800 Subject: [PATCH 082/151] refactor checkpoint --- source/libs/stream/src/streamBackendRocksdb.c | 93 ++++++++++--------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5e00a39417..c3ab5ed3ee 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -91,6 +91,7 @@ uint32_t nextPow2(uint32_t x); int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); +int32_t getCfIdx(const char* cfName); void destroyCompactFilteFactory(void* arg); void destroyCompactFilte(void* arg); @@ -117,6 +118,10 @@ typedef int (*BackendCmpFunc)(void* state, const char* aBuf, size_t aLen, const typedef void (*DestroyFunc)(void* state); typedef int32_t (*EncodeValueFunc)(void* value, int32_t vlen, int64_t ttl, char** dest); typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char** dest); + +typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); +typedef const char* (*FactoryNameFunc)(void* arg); +typedef void(*DestroyFactoryFunc)(void *arg); typedef struct { const char* key; int32_t len; @@ -130,11 +135,12 @@ typedef struct { EncodeValueFunc enValueFunc; DecodeValueFunc deValueFunc; + CreateFactoryFunc createFilter; + DestroyFactoryFunc destroyFilter; + FactoryNameFunc funcName; + } SCfInit; -typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef const char* (*FactoryNameFunc)(void* arg); -typedef void(*DestroyFactoryFunc)(void *arg); void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg); typedef struct { @@ -193,36 +199,45 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); +static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, + rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); SCfInit ginitDict[] = { {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, -}; + destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, -SCfFilterFactory ginitFilterDict[] = { - {"default", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, - {"state", destroyCompactFilteFactory,compactFilteFactoryCreateFilterState,compactFilteFactoryNameState}, - {"fill", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFill,compactFilteFactoryNameFill}, - {"sess", destroyCompactFilteFactory,compactFilteFactoryCreateFilterSess,compactFilteFactoryNameSess}, - {"func", destroyCompactFilteFactory,compactFilteFactoryCreateFilterFunc,compactFilteFactoryNameFunc}, - {"parname", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, - {"partag", destroyCompactFilteFactory,compactFilteFactoryCreateFilter,compactFilteFactoryName}, + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, + encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, compactFilteFactoryNameState}, + + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory,compactFilteFactoryNameFill}, + + {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, + compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess, destroyCompactFilteFactory,compactFilteFactoryNameSess}, + + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, compactFilteFactoryNameFunc}, + + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, + + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, }; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; +int32_t getCfIdx(const char* cfName) { + int idx = -1; + size_t len = strlen(cfName); + for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { + if (len == ginitDict[i].len && strncmp(cfName, ginitDict[i].key, strlen(cfName)) == 0) { + idx = i; + break; + } + } + return idx; +} int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { int32_t code = 0; @@ -383,7 +398,6 @@ int32_t dbChkpInit(SDbChkp* p) { return 0; } int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { - taosThreadRwlockRdlock(&p->rwLock); int32_t code = 0; int32_t len = p->len + 128; @@ -532,7 +546,7 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) { } void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { - rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,destroyCompactFilteFactory, ginitFilterDict[i].create, ginitFilterDict[i].funcName); + rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName); rocksdb_options_set_compaction_filter_factory(opt, filterFactory); } @@ -1011,7 +1025,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { return 0; } -static int32_t compareCheckpoint(const void* a, const void* b) { +static int32_t chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; return x < y ? -1 : 1; @@ -1056,7 +1070,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { continue; } } - taosArraySort(suffix, compareCheckpoint); + taosArraySort(suffix, chkpIdComp); // free previous chkpSaved taosArrayClear(pMeta->chkpSaved); for (int i = 0; i < taosArrayGetSize(suffix); i++) { @@ -1335,8 +1349,6 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { // |key|-----value------| // |key|ttl|len|userData| -static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, - rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int len = aLen < bLen ? aLen : bLen; @@ -1764,6 +1776,10 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c return streamStateValueIsStale((char*)val) ? 1 : 0; } const char* compactFilteName(void* arg) { return "stream_filte"; } +const char* compactFilteNameSess(void* arg) { return "stream_filte_sess"; } +const char* compactFilteNameState(void* arg) { return "stream_filte_state"; } +const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } +const char* compactFilteNameFunc(void* arg) { return "stream_filte_func"; } unsigned char compactFilteSess(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1771,7 +1787,6 @@ unsigned char compactFilteSess(void* arg, int level, const char* key, size_t kle return 0; //return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteNameSess(void* arg) { return "stream_filte_sess"; } unsigned char compactFilteState(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1779,7 +1794,6 @@ unsigned char compactFilteState(void* arg, int level, const char* key, size_t kl return 0; //return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteNameState(void* arg) { return "stream_filte_state"; } unsigned char compactFilteFill(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1787,7 +1801,6 @@ unsigned char compactFilteFill(void* arg, int level, const char* key, size_t kle return 0; //return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1795,7 +1808,6 @@ unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t kle return 0; //return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteNameFunc(void* arg) { return "stream_filte_func"; } rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { @@ -1831,17 +1843,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocks -int32_t getCfIdx(const char* cfName) { - int idx = -1; - size_t len = strlen(cfName); - for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - if (len == ginitDict[i].len && strncmp(cfName, ginitDict[i].key, strlen(cfName)) == 0) { - idx = i; - break; - } - } - return idx; -} int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { int32_t code = -1; char* err = NULL; From 021859361f11f59309f2cd62f6854a22c9482320 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 09:22:57 +0800 Subject: [PATCH 083/151] refactor checkpoint --- source/libs/stream/src/streamBackendRocksdb.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c3ab5ed3ee..82de4fb933 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -100,7 +100,15 @@ const char* compactFilteFactoryNameSess(void* arg); const char* compactFilteFactoryNameState(void* arg); const char* compactFilteFactoryNameFunc(void* arg); const char* compactFilteFactoryNameFill(void* arg); -const char* compactFilteName(void* arg); + +const char* compactFilteName(void* arg); +const char* compactFilteNameSess(void* arg); +const char* compactFilteNameState(void* arg); +const char* compactFilteNameFill(void* arg); +const char* compactFilteNameFunc(void* arg); + + + unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); From 95cfc5eb4625fc378914b0ab14cc2456e76f2740 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 09:55:55 +0800 Subject: [PATCH 084/151] refactor checkpoint --- source/libs/stream/src/streamBackendRocksdb.c | 272 +++++++++--------- 1 file changed, 132 insertions(+), 140 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 82de4fb933..c6dafc4e41 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -41,7 +41,7 @@ typedef struct SDbChkp { SArray* pAdd; SArray* pDel; int8_t update; - + TdThreadRwlock rwLock; } SDbChkp; typedef struct { @@ -64,7 +64,7 @@ typedef struct { SArray* pDel; int8_t update; - SHashObj *pDbChkpTbl; + SHashObj* pDbChkpTbl; TdThreadRwlock rwLock; } SBackendManager; @@ -90,16 +90,16 @@ uint32_t nextPow2(uint32_t x); int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); -void destroyRocksdbCfInst(RocksdbCfInst* inst); +void destroyRocksdbCfInst(RocksdbCfInst* inst); int32_t getCfIdx(const char* cfName); -void destroyCompactFilteFactory(void* arg); -void destroyCompactFilte(void* arg); -const char* compactFilteFactoryName(void* arg); -const char* compactFilteFactoryNameSess(void* arg); -const char* compactFilteFactoryNameState(void* arg); -const char* compactFilteFactoryNameFunc(void* arg); -const char* compactFilteFactoryNameFill(void* arg); +void destroyCompactFilteFactory(void* arg); +void destroyCompactFilte(void* arg); +const char* compactFilteFactoryName(void* arg); +const char* compactFilteFactoryNameSess(void* arg); +const char* compactFilteFactoryNameState(void* arg); +const char* compactFilteFactoryNameFunc(void* arg); +const char* compactFilteFactoryNameFill(void* arg); const char* compactFilteName(void* arg); const char* compactFilteNameSess(void* arg); @@ -107,8 +107,6 @@ const char* compactFilteNameState(void* arg); const char* compactFilteNameFill(void* arg); const char* compactFilteNameFunc(void* arg); - - unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); @@ -117,7 +115,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rock rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); - typedef int (*EncodeFunc)(void* key, char* buf); typedef int (*DecodeFunc)(void* key, char* buf); typedef int (*ToStringFunc)(void* key, char* buf); @@ -129,7 +126,7 @@ typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); typedef const char* (*FactoryNameFunc)(void* arg); -typedef void(*DestroyFactoryFunc)(void *arg); +typedef void (*DestroyFactoryFunc)(void* arg); typedef struct { const char* key; int32_t len; @@ -143,19 +140,18 @@ typedef struct { EncodeValueFunc enValueFunc; DecodeValueFunc deValueFunc; - CreateFactoryFunc createFilter; + CreateFactoryFunc createFilter; DestroyFactoryFunc destroyFilter; FactoryNameFunc funcName; } SCfInit; - -void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg); +void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg); typedef struct { - void *funcName; + void* funcName; DestroyFactoryFunc destroy; - CreateFactoryFunc create; - FactoryNameFunc factoryName; + CreateFactoryFunc create; + FactoryNameFunc factoryName; } SCfFilterFactory; const char* compareDefaultName(void* name); @@ -205,32 +201,38 @@ void destroyFunc(void* arg); int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); - -static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); +static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); SCfInit ginitDict[] = { {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, + destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, + compactFilteFactoryName}, {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, compactFilteFactoryNameState}, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, + compactFilteFactoryNameState}, {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory,compactFilteFactoryNameFill}, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory, + compactFilteFactoryNameFill}, {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess, destroyCompactFilteFactory,compactFilteFactoryNameSess}, + compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess, + destroyCompactFilteFactory, compactFilteFactoryNameSess}, {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, compactFilteFactoryNameFunc}, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, + compactFilteFactoryNameFunc}, {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc,compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, + compactFilteFactoryName}, {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, + encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, + compactFilteFactoryName}, }; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; @@ -270,7 +272,7 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { return code; } -int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { +int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { taosThreadRwlockWrlock(&p->rwLock); p->preCkptId = p->curChkpId; @@ -316,7 +318,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { } } if (p->init == 0) { - void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { size_t len; @@ -351,7 +352,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { p->curChkpId = chkpId; } - taosHashClear(p->pSstTbl[p->idx]); p->idx = 1 - p->idx; @@ -359,10 +359,9 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray *list) { taosThreadRwlockUnlock(&p->rwLock); return 0; - } SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { - SDbChkp *p = taosMemoryCalloc(1, sizeof(SDbChkp)); + SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp)); p->curChkpId = initChkpId; p->preCkptId = -1; p->pSST = taosArrayInit(64, sizeof(void*)); @@ -379,7 +378,7 @@ SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { p->update = 0; taosThreadRwlockInit(&p->rwLock, NULL); - SArray *list = NULL; + SArray* list = NULL; int32_t code = dbChkpGetDelta(p, initChkpId, list); return p; @@ -398,11 +397,10 @@ void dbChkpDestroy(SDbChkp* pChkp) { taosMemoryFree(pChkp->pCurrent); taosMemoryFree(pChkp->pManifest); - } int32_t dbChkpInit(SDbChkp* p) { - if (p == NULL) return 0; + if (p == NULL) return 0; return 0; } int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { @@ -487,7 +485,6 @@ _ERROR: taosMemoryFree(srcDir); taosMemoryFree(dstDir); return code; - } SBackendManager* bkdMgtCreate(char* path) { SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); @@ -498,9 +495,9 @@ SBackendManager* bkdMgtCreate(char* path) { void bkdMgtDestroy(SBackendManager* bm) { if (bm == NULL) return; - void *pIter = taosHashIterate(bm->pDbChkpTbl, NULL); + void* pIter = taosHashIterate(bm->pDbChkpTbl, NULL); while (pIter) { - SDbChkp *pChkp = *(SDbChkp **)(pIter); + SDbChkp* pChkp = *(SDbChkp**)(pIter); dbChkpDestroy(pChkp); pIter = taosHashIterate(bm->pDbChkpTbl, pIter); @@ -510,52 +507,52 @@ void bkdMgtDestroy(SBackendManager* bm) { taosMemoryFree(bm); } -int32_t bkdMgtGetDelta(SBackendManager* bm, char *taskId, int64_t chkpId, SArray* list) { +int32_t bkdMgtGetDelta(SBackendManager* bm, char* taskId, int64_t chkpId, SArray* list) { int32_t code = 0; taosThreadRwlockWrlock(&bm->rwLock); - SDbChkp *pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + SDbChkp* pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); code = dbChkpGetDelta(pChkp, chkpId, list); taosThreadRwlockUnlock(&bm->rwLock); - return code ; + return code; } -int32_t bkdMgtAddChkp(SBackendManager *bm, char *task, char *path) { +int32_t bkdMgtAddChkp(SBackendManager* bm, char* task, char* path) { int32_t code = -1; taosThreadRwlockWrlock(&bm->rwLock); - SDbChkp **pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); + SDbChkp** pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); if (pp == NULL) { - SDbChkp *p = dbChkpCreate(path, 0); + SDbChkp* p = dbChkpCreate(path, 0); if (p != NULL) { - taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void *)); + taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void*)); code = 0; } } else { stError("task chkp already exists"); } - + taosThreadRwlockUnlock(&bm->rwLock); - - return code; + + return code; } -int32_t bkdMgtDumpTo(SBackendManager* bm, char *taskId, char* dname) { +int32_t bkdMgtDumpTo(SBackendManager* bm, char* taskId, char* dname) { int32_t code = 0; taosThreadRwlockRdlock(&bm->rwLock); - SDbChkp *p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - code = dbChkpDumpTo(p, dname); + SDbChkp* p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpDumpTo(p, dname); taosThreadRwlockUnlock(&bm->rwLock); return code; - } -void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void *arg) { - rocksdb_compactionfilterfactory_t *filterFactory = rocksdb_compactionfilterfactory_create(arg,ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName); - rocksdb_options_set_compaction_filter_factory(opt, filterFactory); +void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg) { + rocksdb_compactionfilterfactory_t* filterFactory = rocksdb_compactionfilterfactory_create( + arg, ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName); + rocksdb_options_set_compaction_filter_factory(opt, filterFactory); } bool isValidCheckpoint(const char* dir) { return true; } @@ -1330,7 +1327,7 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } +void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); if (inst->pHandle) { @@ -1357,7 +1354,6 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { // |key|-----value------| // |key|ttl|len|userData| - int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int len = aLen < bLen ? aLen : bLen; int ret = memcmp(aBuf, bBuf, len); @@ -1777,7 +1773,6 @@ const char* compactFilteFactoryNameFunc(void* arg) { return "stream_compact_filter_func"; } - void destroyCompactFilte(void* arg) { (void)arg; } unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { @@ -1790,34 +1785,33 @@ const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } const char* compactFilteNameFunc(void* arg) { return "stream_filte_func"; } unsigned char compactFilteSess(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, - char** newval, size_t* newvlen, unsigned char* value_changed) { + char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - //return streamStateValueIsStale((char*)val) ? 1 : 0; + // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteState(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, - char** newval, size_t* newvlen, unsigned char* value_changed) { + char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - //return streamStateValueIsStale((char*)val) ? 1 : 0; + // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteFill(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, - char** newval, size_t* newvlen, unsigned char* value_changed) { + char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - //return streamStateValueIsStale((char*)val) ? 1 : 0; + // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, - char** newval, size_t* newvlen, unsigned char* value_changed) { + char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - //return streamStateValueIsStale((char*)val) ? 1 : 0; + // return streamStateValueIsStale((char*)val) ? 1 : 0; } - rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx) { SCompactFilteFactory* state = arg; rocksdb_compactionfilter_t* filter = @@ -1849,8 +1843,6 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocks return filter; } - - int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_t nCf) { int32_t code = -1; char* err = NULL; @@ -1942,7 +1934,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); - + pTaskDb->pCompares[i] = compare; pTaskDb->pCfOpts[i] = opt; pTaskDb->pCfParams[i].tableOpt = tableOpt; @@ -2065,7 +2057,7 @@ _EXIT: void taskDbDestroy(void* pBackend) { STaskDbWrapper* wrapper = pBackend; qDebug("succ to destroy stream backend:%p", wrapper); - + int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); if (wrapper == NULL) return; @@ -2074,7 +2066,7 @@ void taskDbDestroy(void* pBackend) { rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); rocksdb_flushoptions_set_wait(flushOpt, 1); - char* err = NULL; + char* err = NULL; for (int i = 0; i < nCf; i++) { if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); if (err != NULL) { @@ -2096,18 +2088,16 @@ void taskDbDestroy(void* pBackend) { rocksdb_env_destroy(wrapper->env); rocksdb_cache_destroy(wrapper->cache); - taosMemoryFree(wrapper->pCf); for (int i = 0; i < nCf; i++) { - rocksdb_options_t *opt = wrapper->pCfOpts[i]; - rocksdb_comparator_t *compare = wrapper->pCompares[i]; - rocksdb_block_based_table_options_t *tblOpt = wrapper->pCfParams[i].tableOpt; + rocksdb_options_t* opt = wrapper->pCfOpts[i]; + rocksdb_comparator_t* compare = wrapper->pCompares[i]; + rocksdb_block_based_table_options_t* tblOpt = wrapper->pCfParams[i].tableOpt; rocksdb_options_destroy(opt); rocksdb_comparator_destroy(compare); rocksdb_block_based_options_destroy(tblOpt); - } taosMemoryFree(wrapper->pCompares); taosMemoryFree(wrapper->pCfOpts); @@ -2527,7 +2517,39 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe return rocksdb_create_iterator_cf(wrapper->db, *readOpt, ((rocksdb_column_family_handle_t**)wrapper->pCf)[idx]); } -#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ +#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ + rocksdb_t* db = wrapper->db; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ + if (err != NULL) { \ + stError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ + ttlVLen, wrapper); \ + } \ + taosMemoryFree(ttlV); \ + } while (0); + +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ do { \ code = 0; \ char buf[128] = {0}; \ @@ -2543,62 +2565,32 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ rocksdb_t* db = wrapper->db; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ - rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ - if (err != NULL) { \ - stError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ - taosMemoryFree(err); \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ code = -1; \ } else { \ - qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, ttlVLen, wrapper); \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \ + wrapper); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ } \ - taosMemoryFree(ttlV); \ - } while (0); - -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->db; \ - rocksdb_readoptions_t* opts = wrapper->readOpt; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, wrapper); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ } while (0); #define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ @@ -2986,7 +2978,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta stDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -3027,7 +3019,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -3065,7 +3057,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -3168,7 +3160,7 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; if (pCur == NULL) return NULL; @@ -3229,7 +3221,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { return NULL; } @@ -3267,7 +3259,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return NULL; } @@ -3305,7 +3297,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { stDebug("streamStateSessionGetKeyByRange_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { return -1; } @@ -3575,7 +3567,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = createStreamStateCursor(); + SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->db = wrapper->db; From e5449bf161ec320b6f7893b7eb9c05082374e433 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 11:49:56 +0800 Subject: [PATCH 085/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 132 ++++++++---------- 1 file changed, 62 insertions(+), 70 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c6dafc4e41..5edba5e180 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -22,6 +22,7 @@ #define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); +int32_t copyFiles(const char* src, const char* dst); typedef struct SDbChkp { int8_t init; char* pCurrent; @@ -67,7 +68,7 @@ typedef struct { SHashObj* pDbChkpTbl; TdThreadRwlock rwLock; -} SBackendManager; +} SBkdMgt; typedef struct SCompactFilteFactory { void* status; @@ -146,7 +147,6 @@ typedef struct { } SCfInit; -void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg); typedef struct { void* funcName; DestroyFactoryFunc destroy; @@ -486,14 +486,14 @@ _ERROR: taosMemoryFree(dstDir); return code; } -SBackendManager* bkdMgtCreate(char* path) { - SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager)); +SBkdMgt* bkdMgtCreate(char* path) { + SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt)); p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taosThreadRwlockInit(&p->rwLock, NULL); return p; } -void bkdMgtDestroy(SBackendManager* bm) { +void bkdMgtDestroy(SBkdMgt* bm) { if (bm == NULL) return; void* pIter = taosHashIterate(bm->pDbChkpTbl, NULL); while (pIter) { @@ -507,7 +507,7 @@ void bkdMgtDestroy(SBackendManager* bm) { taosMemoryFree(bm); } -int32_t bkdMgtGetDelta(SBackendManager* bm, char* taskId, int64_t chkpId, SArray* list) { +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list) { int32_t code = 0; taosThreadRwlockWrlock(&bm->rwLock); @@ -518,7 +518,7 @@ int32_t bkdMgtGetDelta(SBackendManager* bm, char* taskId, int64_t chkpId, SArray return code; } -int32_t bkdMgtAddChkp(SBackendManager* bm, char* task, char* path) { +int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { int32_t code = -1; taosThreadRwlockWrlock(&bm->rwLock); @@ -538,7 +538,7 @@ int32_t bkdMgtAddChkp(SBackendManager* bm, char* task, char* path) { return code; } -int32_t bkdMgtDumpTo(SBackendManager* bm, char* taskId, char* dname) { +int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { int32_t code = 0; taosThreadRwlockRdlock(&bm->rwLock); @@ -549,53 +549,8 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* taskId, char* dname) { return code; } -void dbSetFilterFactory(rocksdb_options_t* opt, int32_t i, void* arg) { - rocksdb_compactionfilterfactory_t* filterFactory = rocksdb_compactionfilterfactory_create( - arg, ginitDict[i].destroyFilter, ginitDict[i].createFilter, ginitDict[i].funcName); - rocksdb_options_set_compaction_filter_factory(opt, filterFactory); -} - bool isValidCheckpoint(const char* dir) { return true; } -int32_t copyFiles(const char* src, const char* dst) { - int32_t code = 0; - // opt later, just hard link - int32_t sLen = strlen(src); - int32_t dLen = strlen(dst); - char* srcName = taosMemoryCalloc(1, sLen + 64); - char* dstName = taosMemoryCalloc(1, dLen + 64); - - TdDirPtr pDir = taosOpenDir(src); - if (pDir == NULL) { - taosMemoryFree(srcName); - taosMemoryFree(dstName); - return -1; - } - - TdDirEntryPtr de = NULL; - while ((de = taosReadDir(pDir)) != NULL) { - char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - - sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); - sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); - if (!taosDirEntryIsDir(de)) { - code = taosCopyFile(srcName, dstName); - if (code == -1) { - goto _err; - } - } - - memset(srcName, 0, sLen + 64); - memset(dstName, 0, dLen + 64); - } - -_err: - taosMemoryFreeClear(srcName); - taosMemoryFreeClear(dstName); - taosCloseDir(&pDir); - return code >= 0 ? 0 : -1; -} int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { // impl later int32_t code = 0; @@ -1754,23 +1709,23 @@ void destroyCompactFilteFactory(void* arg) { } const char* compactFilteFactoryName(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter"; + return "stream_compact_factory_filter"; } const char* compactFilteFactoryNameSess(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter_sess"; + return "stream_compact_factory_filter_sess"; } const char* compactFilteFactoryNameState(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter_state"; + return "stream_compact_factory_filter_state"; } const char* compactFilteFactoryNameFill(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter_fill"; + return "stream_compact_factory_filter_fill"; } const char* compactFilteFactoryNameFunc(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_filter_func"; + return "stream_compact_factory_filter_func"; } void destroyCompactFilte(void* arg) { (void)arg; } @@ -1788,21 +1743,18 @@ unsigned char compactFilteSess(void* arg, int level, const char* key, size_t kle char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteState(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteFill(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { // not impl yet return 0; - // return streamStateValueIsStale((char*)val) ? 1 : 0; } unsigned char compactFilteFunc(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, @@ -1935,12 +1887,13 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); + rocksdb_compactionfilterfactory_t* filterFactory = + rocksdb_compactionfilterfactory_create(NULL, cfPara->destroyFilter, cfPara->createFilter, cfPara->funcName); + rocksdb_options_set_compaction_filter_factory(opt, filterFactory); + pTaskDb->pCompares[i] = compare; pTaskDb->pCfOpts[i] = opt; pTaskDb->pCfParams[i].tableOpt = tableOpt; - - // set filter factory - dbSetFilterFactory(opt, i, NULL); } return; } @@ -2054,8 +2007,8 @@ _EXIT: return NULL; } -void taskDbDestroy(void* pBackend) { - STaskDbWrapper* wrapper = pBackend; +void taskDbDestroy(void* pDb) { + STaskDbWrapper* wrapper = pDb; qDebug("succ to destroy stream backend:%p", wrapper); int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -2107,7 +2060,7 @@ void taskDbDestroy(void* pBackend) { if (wrapper->db) rocksdb_close(wrapper->db); - taskDbDestroyChkpOpt(pBackend); + taskDbDestroyChkpOpt(wrapper); taosMemoryFree(wrapper->idstr); taosMemoryFree(wrapper->path); @@ -2116,14 +2069,14 @@ void taskDbDestroy(void* pBackend) { return; } -int32_t taskDbOpenCfByKey(STaskDbWrapper* pBackend, const char* key) { +int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { int32_t code = 0; char* err = NULL; int8_t idx = getCfIdx(key); if (idx == -1) return -1; rocksdb_column_family_handle_t* cf = - rocksdb_create_column_family(pBackend->db, pBackend->pCfOpts[idx], ginitDict[idx].key, &err); + rocksdb_create_column_family(pDb->db, pDb->pCfOpts[idx], ginitDict[idx].key, &err); if (err != NULL) { stError("failed to open cf, key:%s, reason: %s", key, err); taosMemoryFree(err); @@ -2131,7 +2084,7 @@ int32_t taskDbOpenCfByKey(STaskDbWrapper* pBackend, const char* key) { return code; } - pBackend->pCf[idx] = cf; + pDb->pCf[idx] = cf; return code; } int32_t copyDataAt(RocksdbCfInst* pSrc, STaskDbWrapper* pDst, int8_t i) { @@ -3690,4 +3643,43 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 8); x = x | (x >> 16); return x + 1; +} +int32_t copyFiles(const char* src, const char* dst) { + int32_t code = 0; + // opt later, just hard link + int32_t sLen = strlen(src); + int32_t dLen = strlen(dst); + char* srcName = taosMemoryCalloc(1, sLen + 64); + char* dstName = taosMemoryCalloc(1, dLen + 64); + + TdDirPtr pDir = taosOpenDir(src); + if (pDir == NULL) { + taosMemoryFree(srcName); + taosMemoryFree(dstName); + return -1; + } + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; + + sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); + sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); + if (!taosDirEntryIsDir(de)) { + code = taosCopyFile(srcName, dstName); + if (code == -1) { + goto _err; + } + } + + memset(srcName, 0, sLen + 64); + memset(dstName, 0, dLen + 64); + } + +_err: + taosMemoryFreeClear(srcName); + taosMemoryFreeClear(dstName); + taosCloseDir(&pDir); + return code >= 0 ? 0 : -1; } \ No newline at end of file From e015134d6da2f3f138cde983051ed51344317941 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 14:46:52 +0800 Subject: [PATCH 086/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5edba5e180..cca1057662 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -20,9 +20,6 @@ #include "tcommon.h" #include "tref.h" -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); - -int32_t copyFiles(const char* src, const char* dst); typedef struct SDbChkp { int8_t init; char* pCurrent; @@ -87,8 +84,6 @@ typedef struct { rocksdb_comparator_t** pCompares; } RocksdbCfInst; -uint32_t nextPow2(uint32_t x); - int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -147,13 +142,6 @@ typedef struct { } SCfInit; -typedef struct { - void* funcName; - DestroyFactoryFunc destroy; - CreateFactoryFunc create; - FactoryNameFunc factoryName; -} SCfFilterFactory; - const char* compareDefaultName(void* name); const char* compareStateName(void* name); const char* compareWinKeyName(void* name); @@ -205,6 +193,10 @@ static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); +#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); +int32_t copyFiles(const char* src, const char* dst); +uint32_t nextPow2(uint32_t x); + SCfInit ginitDict[] = { {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, @@ -1709,7 +1701,7 @@ void destroyCompactFilteFactory(void* arg) { } const char* compactFilteFactoryName(void* arg) { SCompactFilteFactory* state = arg; - return "stream_compact_factory_filter"; + return "stream_compact_factory_filter_default"; } const char* compactFilteFactoryNameSess(void* arg) { SCompactFilteFactory* state = arg; @@ -1733,7 +1725,7 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c char** newval, size_t* newvlen, unsigned char* value_changed) { return streamStateValueIsStale((char*)val) ? 1 : 0; } -const char* compactFilteName(void* arg) { return "stream_filte"; } +const char* compactFilteName(void* arg) { return "stream_filte_default"; } const char* compactFilteNameSess(void* arg) { return "stream_filte_sess"; } const char* compactFilteNameState(void* arg) { return "stream_filte_state"; } const char* compactFilteNameFill(void* arg) { return "stream_filte_fill"; } @@ -1837,7 +1829,6 @@ void taskDbRemoveRef(void* pTaskDb) { STaskDbWrapper* pBackend = pTaskDb; taosReleaseRef(taskDbWrapperId, pBackend->refId); } -// void taskDbDestroy(STaskDbWrapper* wrapper); void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_env_t* env = rocksdb_create_default_env(); @@ -1899,7 +1890,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { } void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { pTaskDb->chkpId = -1; - pTaskDb->chkpCap = 4; + pTaskDb->chkpCap = 2; pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); From 042522ac7af0889712a34ce2917e5a19e4a53ce0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 15:18:24 +0800 Subject: [PATCH 087/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 151 +++++++++--------- 1 file changed, 75 insertions(+), 76 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index cca1057662..b8f363004a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -111,34 +111,34 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterState(void* arg, rock rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFunc(void* arg, rocksdb_compactionfiltercontext_t* ctx); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilterFill(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef int (*EncodeFunc)(void* key, char* buf); -typedef int (*DecodeFunc)(void* key, char* buf); -typedef int (*ToStringFunc)(void* key, char* buf); -typedef const char* (*CompareName)(void* statue); -typedef int (*BackendCmpFunc)(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); -typedef void (*DestroyFunc)(void* state); -typedef int32_t (*EncodeValueFunc)(void* value, int32_t vlen, int64_t ttl, char** dest); -typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char** dest); +typedef int (*__db_key_encode_fn_t)(void* key, char* buf); +typedef int (*__db_key_decode_fn_t)(void* key, char* buf); +typedef int (*__db_key_tostr_fn_t)(void* key, char* buf); +typedef const char* (*__db_key_cmpname_fn_t)(void* statue); +typedef int (*__db_key_cmp_fn_t)(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +typedef void (*__db_destroy_cmp_fn_t)(void* state); +typedef int32_t (*__db_value_encode_fn_t)(void* value, int32_t vlen, int64_t ttl, char** dest); +typedef int32_t (*__db_value_decode_fn_t)(void* value, int32_t vlen, int64_t* ttl, char** dest); -typedef rocksdb_compactionfilter_t* (*CreateFactoryFunc)(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef const char* (*FactoryNameFunc)(void* arg); -typedef void (*DestroyFactoryFunc)(void* arg); +typedef rocksdb_compactionfilter_t* (*__db_factory_create_fn_t)(void* arg, rocksdb_compactionfiltercontext_t* ctx); +typedef const char* (*__db_factory_name_fn_t)(void* arg); +typedef void (*__db_factory_destroy_fn_t)(void* arg); typedef struct { - const char* key; - int32_t len; - int idx; - BackendCmpFunc cmpFunc; - EncodeFunc enFunc; - DecodeFunc deFunc; - ToStringFunc toStrFunc; - CompareName cmpName; - DestroyFunc detroyFunc; - EncodeValueFunc enValueFunc; - DecodeValueFunc deValueFunc; + const char* key; + int32_t len; + int idx; + __db_key_cmp_fn_t cmpKey; + __db_key_encode_fn_t enFunc; + __db_key_decode_fn_t deFunc; + __db_key_tostr_fn_t toStrFunc; + __db_key_cmpname_fn_t cmpName; + __db_destroy_cmp_fn_t destroyCmp; + __db_value_encode_fn_t enValueFunc; + __db_value_decode_fn_t deValueFunc; - CreateFactoryFunc createFilter; - DestroyFactoryFunc destroyFilter; - FactoryNameFunc funcName; + __db_factory_create_fn_t createFilter; + __db_factory_destroy_fn_t destroyFilter; + __db_factory_name_fn_t funcName; } SCfInit; @@ -180,14 +180,15 @@ int parKeyEncode(void* k, char* buf); int parKeyDecode(void* k, char* buf); int parKeyToString(void* k, char* buf); -int stremaValueEncode(void* k, char* buf); -int streamValueDecode(void* k, char* buf); -int32_t streamValueToString(void* k, char* buf); -int32_t streaValueIsStale(void* k, int64_t ts); -void destroyFunc(void* arg); +// int stremaValueEncode(void* k, char* buf); +// int streamValueDecode(void* k, char* buf); -int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); -int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); +int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest); +int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest); +int32_t valueToString(void* k, char* buf); +int32_t valueIsStale(void* k, int64_t ts); + +void destroyCompare(void* arg); static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, @@ -199,32 +200,30 @@ uint32_t nextPow2(uint32_t x); SCfInit ginitDict[] = { {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, + destroyCompare, valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterState, destroyCompactFilteFactory, compactFilteFactoryNameState}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory, + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterFill, destroyCompactFilteFactory, compactFilteFactoryNameFill}, {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterSess, + compareSessionKeyName, destroyCompare, valueEncode, valueDecode, compactFilteFactoryCreateFilterSess, destroyCompactFilteFactory, compactFilteFactoryNameSess}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilterFunc, destroyCompactFilteFactory, compactFilteFactoryNameFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, - compactFilteFactoryName}, + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, - compactFilteFactoryName}, + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyCompare, + valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, }; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; @@ -1583,23 +1582,23 @@ int parKeyToString(void* k, char* buf) { n = sprintf(buf + n, "[groupId:%" PRIi64 "]", *key); return n; } -int stremaValueEncode(void* k, char* buf) { - int len = 0; - SStreamValue* key = k; - len += taosEncodeFixedI64((void**)&buf, key->unixTimestamp); - len += taosEncodeFixedI32((void**)&buf, key->len); - len += taosEncodeBinary((void**)&buf, key->data, key->len); - return len; -} -int streamValueDecode(void* k, char* buf) { - SStreamValue* key = k; - char* p = buf; - p = taosDecodeFixedI64(p, &key->unixTimestamp); - p = taosDecodeFixedI32(p, &key->len); - p = taosDecodeBinary(p, (void**)&key->data, key->len); - return p - buf; -} -int32_t streamValueToString(void* k, char* buf) { +// int stremaValueEncode(void* k, char* buf) { +// int len = 0; +// SStreamValue* key = k; +// len += taosEncodeFixedI64((void**)&buf, key->unixTimestamp); +// len += taosEncodeFixedI32((void**)&buf, key->len); +// len += taosEncodeBinary((void**)&buf, key->data, key->len); +// return len; +// } +// int streamValueDecode(void* k, char* buf) { +// SStreamValue* key = k; +// char* p = buf; +// p = taosDecodeFixedI64(p, &key->unixTimestamp); +// p = taosDecodeFixedI32(p, &key->len); +// p = taosDecodeBinary(p, (void**)&key->data, key->len); +// return p - buf; +// } +int32_t valueToString(void* k, char* buf) { SStreamValue* key = k; int n = 0; n += sprintf(buf + n, "[unixTimestamp:%" PRIi64 ",", key->unixTimestamp); @@ -1609,7 +1608,7 @@ int32_t streamValueToString(void* k, char* buf) { } /*1: stale, 0: no stale*/ -int32_t streaValueIsStale(void* k, int64_t ts) { +int32_t valueIsStale(void* k, int64_t ts) { SStreamValue* key = k; if (key->unixTimestamp < ts) { return 1; @@ -1617,12 +1616,12 @@ int32_t streaValueIsStale(void* k, int64_t ts) { return 0; } -void destroyFunc(void* arg) { +void destroyCompare(void* arg) { (void)arg; return; } -int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { +int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest) { SStreamValue key = {.unixTimestamp = ttl, .len = vlen, .data = (char*)(value)}; int32_t len = 0; if (*dest == NULL) { @@ -1644,7 +1643,7 @@ int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { * ret >= 0 : found valid value * ret < 0 : error or timeout */ -int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { +int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest) { SStreamValue key = {0}; char* p = value; if (streamStateValueIsStale(p)) { @@ -1875,7 +1874,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); rocksdb_compactionfilterfactory_t* filterFactory = @@ -2182,7 +2181,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[idx]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); pCompare[i] = compare; } @@ -2266,7 +2265,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); inst->pCompares[i] = compare; @@ -2346,7 +2345,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->detroyFunc, cf->cmpFunc, cf->cmpName); + rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } @@ -2684,7 +2683,7 @@ int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, cons if (pVLen != NULL) { size_t vlen = 0; const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); - *pVLen = decodeValueFunc((void*)valStr, vlen, NULL, (char**)pVal); + *pVLen = valueDecode((void*)valStr, vlen, NULL, (char**)pVal); } *pKey = pKtmp->key; @@ -3058,7 +3057,7 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* SStateSessionKey* pKTmp = &ktmp; const char* vval = rocksdb_iter_value(pCur->iter, (size_t*)&vLen); char* val = NULL; - int32_t len = decodeValueFunc((void*)vval, vLen, NULL, &val); + int32_t len = valueDecode((void*)vval, vLen, NULL, &val); if (len < 0) { taosMemoryFree(val); return -1; @@ -3152,7 +3151,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, winKeyDecode(&winKey, keyStr); const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); - int32_t len = decodeValueFunc((void*)valStr, vlen, NULL, (char**)pVal); + int32_t len = valueDecode((void*)valStr, vlen, NULL, (char**)pVal); if (len < 0) { return -1; } @@ -3486,7 +3485,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int32_t vlen = 0; const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen); char* val = NULL; - int32_t len = decodeValueFunc((void*)vval, vlen, NULL, NULL); + int32_t len = valueDecode((void*)vval, vlen, NULL, NULL); if (len < 0) { rocksdb_iter_next(pIter); continue; @@ -3542,7 +3541,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { int32_t vlen = 0; const char* val = rocksdb_iter_value(pCur->iter, (size_t*)&vlen); - *len = decodeValueFunc((void*)val, vlen, NULL, &ret); + *len = valueDecode((void*)val, vlen, NULL, &ret); if (*len < 0) { taosMemoryFree(ret); return NULL; From 8f7ad2bd900bd154fc6f38cc48b34d4fb188dbe7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 15:20:58 +0800 Subject: [PATCH 088/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b8f363004a..e579fdc07d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -116,7 +116,7 @@ typedef int (*__db_key_decode_fn_t)(void* key, char* buf); typedef int (*__db_key_tostr_fn_t)(void* key, char* buf); typedef const char* (*__db_key_cmpname_fn_t)(void* statue); typedef int (*__db_key_cmp_fn_t)(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); -typedef void (*__db_destroy_cmp_fn_t)(void* state); +typedef void (*__db_key_cmp_destroy_fn_t)(void* state); typedef int32_t (*__db_value_encode_fn_t)(void* value, int32_t vlen, int64_t ttl, char** dest); typedef int32_t (*__db_value_decode_fn_t)(void* value, int32_t vlen, int64_t* ttl, char** dest); @@ -124,17 +124,17 @@ typedef rocksdb_compactionfilter_t* (*__db_factory_create_fn_t)(void* arg, rocks typedef const char* (*__db_factory_name_fn_t)(void* arg); typedef void (*__db_factory_destroy_fn_t)(void* arg); typedef struct { - const char* key; - int32_t len; - int idx; - __db_key_cmp_fn_t cmpKey; - __db_key_encode_fn_t enFunc; - __db_key_decode_fn_t deFunc; - __db_key_tostr_fn_t toStrFunc; - __db_key_cmpname_fn_t cmpName; - __db_destroy_cmp_fn_t destroyCmp; - __db_value_encode_fn_t enValueFunc; - __db_value_decode_fn_t deValueFunc; + const char* key; + int32_t len; + int idx; + __db_key_cmp_fn_t cmpKey; + __db_key_encode_fn_t enFunc; + __db_key_decode_fn_t deFunc; + __db_key_tostr_fn_t toStrFunc; + __db_key_cmpname_fn_t cmpName; + __db_key_cmp_destroy_fn_t destroyCmp; + __db_value_encode_fn_t enValueFunc; + __db_value_decode_fn_t deValueFunc; __db_factory_create_fn_t createFilter; __db_factory_destroy_fn_t destroyFilter; From 1ee82f5e2c9c81782d822ea64fcce8a98482601e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 15:50:42 +0800 Subject: [PATCH 089/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index e579fdc07d..300251885d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -127,7 +127,7 @@ typedef struct { const char* key; int32_t len; int idx; - __db_key_cmp_fn_t cmpKey; + __db_key_cmp_fn_t cmpFunc; __db_key_encode_fn_t enFunc; __db_key_decode_fn_t deFunc; __db_key_tostr_fn_t toStrFunc; @@ -1874,7 +1874,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); rocksdb_compactionfilterfactory_t* filterFactory = @@ -2181,7 +2181,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[idx]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); pCompare[i] = compare; } @@ -2265,7 +2265,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); inst->pCompares[i] = compare; @@ -2345,7 +2345,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); + rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpFunc, cf->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } From 4f53ffd76f59edae62c211e25aa9e1007e1d4833 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 16:01:57 +0800 Subject: [PATCH 090/151] refactor backend --- source/libs/stream/src/streamBackendRocksdb.c | 630 +++++++++--------- 1 file changed, 306 insertions(+), 324 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 300251885d..074d23cdbf 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -127,7 +127,7 @@ typedef struct { const char* key; int32_t len; int idx; - __db_key_cmp_fn_t cmpFunc; + __db_key_cmp_fn_t cmpKey; __db_key_encode_fn_t enFunc; __db_key_decode_fn_t deFunc; __db_key_tostr_fn_t toStrFunc; @@ -180,9 +180,6 @@ int parKeyEncode(void* k, char* buf); int parKeyDecode(void* k, char* buf); int parKeyToString(void* k, char* buf); -// int stremaValueEncode(void* k, char* buf); -// int streamValueDecode(void* k, char* buf); - int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest); int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest); int32_t valueToString(void* k, char* buf); @@ -240,306 +237,6 @@ int32_t getCfIdx(const char* cfName) { return idx; } -int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { - int32_t code = 0; - size_t len = 0; - void* pIter = taosHashIterate(p2, NULL); - while (pIter) { - char* name = taosHashGetKey(pIter, &len); - if (!taosHashGet(p1, name, len)) { - char* p = taosStrdup(name); - taosArrayPush(diff, &p); - } - pIter = taosHashIterate(p2, pIter); - } - return code; -} -int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { - int32_t code = 0; - - code = compareHashTableImpl(p1, p2, add); - code = compareHashTableImpl(p2, p1, del); - - return code; -} - -int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { - taosThreadRwlockWrlock(&p->rwLock); - - p->preCkptId = p->curChkpId; - p->curChkpId = chkpId; - const char* pCurrent = "CURRENT"; - int32_t currLen = strlen(pCurrent); - - const char* pManifest = "MANIFEST-"; - int32_t maniLen = strlen(pManifest); - - const char* pSST = ".sst"; - int32_t sstLen = strlen(pSST); - - memset(p->buf, 0, p->len); - sprintf(p->buf, "%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, chkpId); - - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); - - TdDirPtr pDir = taosOpenDir(p->buf); - TdDirEntryPtr de = NULL; - int8_t dummy = 0; - while ((de = taosReadDir(pDir)) != NULL) { - char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { - taosMemoryFreeClear(p->pCurrent); - p->pCurrent = taosStrdup(name); - taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - - if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { - taosMemoryFreeClear(p->pManifest); - p->pManifest = taosStrdup(name); - taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { - // char* p = taosStrdup(name); - taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); - continue; - } - } - if (p->init == 0) { - void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); - while (pIter) { - size_t len; - char* name = taosHashGetKey(pIter, &len); - if (name != NULL && len != 0) { - taosArrayPush(p->pAdd, &name); - } - pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); - } - if (taosArrayGetSize(p->pAdd) > 0) p->update = 1; - - p->init = 1; - p->preCkptId = -1; - p->curChkpId = chkpId; - } else { - int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); - if (code != 0) { - // dead code - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); - taosHashClear(p->pSstTbl[1 - p->idx]); - p->update = 0; - taosCloseDir(&pDir); - return code; - } - - if (taosArrayGetSize(p->pAdd) == 0 && taosArrayGetSize(p->pDel) == 0) { - p->update = 0; - } - - p->preCkptId = p->curChkpId; - p->curChkpId = chkpId; - } - - taosHashClear(p->pSstTbl[p->idx]); - p->idx = 1 - p->idx; - - taosCloseDir(&pDir); - taosThreadRwlockUnlock(&p->rwLock); - - return 0; -} -SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { - SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp)); - p->curChkpId = initChkpId; - p->preCkptId = -1; - p->pSST = taosArrayInit(64, sizeof(void*)); - p->path = taosStrdup(path); - p->len = strlen(path) + 128; - p->buf = taosMemoryCalloc(1, p->len); - - p->idx = 0; - p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - - p->pAdd = taosArrayInit(64, sizeof(void*)); - p->pDel = taosArrayInit(64, sizeof(void*)); - p->update = 0; - taosThreadRwlockInit(&p->rwLock, NULL); - - SArray* list = NULL; - int32_t code = dbChkpGetDelta(p, initChkpId, list); - - return p; -} - -void dbChkpDestroy(SDbChkp* pChkp) { - taosMemoryFree(pChkp->buf); - taosMemoryFree(pChkp->path); - - taosArrayDestroyP(pChkp->pSST, taosMemoryFree); - taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); - taosArrayDestroyP(pChkp->pDel, taosMemoryFree); - - taosHashCleanup(pChkp->pSstTbl[0]); - taosHashCleanup(pChkp->pSstTbl[1]); - - taosMemoryFree(pChkp->pCurrent); - taosMemoryFree(pChkp->pManifest); -} - -int32_t dbChkpInit(SDbChkp* p) { - if (p == NULL) return 0; - return 0; -} -int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { - taosThreadRwlockRdlock(&p->rwLock); - int32_t code = 0; - int32_t len = p->len + 128; - - char* srcBuf = taosMemoryCalloc(1, len); - char* dstBuf = taosMemoryCalloc(1, len); - - char* srcDir = taosMemoryCalloc(1, len); - char* dstDir = taosMemoryCalloc(1, len); - - sprintf(srcDir, "%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoint", p->curChkpId); - sprintf(dstDir, "%s%s%s", p->path, TD_DIRSEP, dname); - - if (!taosDirExist(srcDir)) { - stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); - code = -1; - goto _ERROR; - } - - code = taosMkDir(dstDir); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); - goto _ERROR; - } - - // clear current file - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); - taosRemoveFile(dstBuf); - - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); - taosRemoveFile(dstBuf); - - // add file to $name dir - for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { - memset(dstBuf, 0, len); - memset(srcBuf, 0, len); - - char* filename = taosArrayGetP(p->pAdd, i); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - - taosCopyFile(srcBuf, dstBuf); - } - // del file in $name - for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { - memset(dstBuf, 0, len); - memset(srcBuf, 0, len); - - char* filename = taosArrayGetP(p->pDel, i); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - taosRemoveFile(dstBuf); - } - - // copy current file to dst dir - memset(srcBuf, 0, len); - memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); - taosCopyFile(srcBuf, dstBuf); - - // copy manifest file to dst dir - memset(srcBuf, 0, len); - memset(dstBuf, 0, len); - sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); - taosCopyFile(srcBuf, dstBuf); - - // clear delta data buf - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); - -_ERROR: - taosThreadRwlockUnlock(&p->rwLock); - taosMemoryFree(srcBuf); - taosMemoryFree(dstBuf); - taosMemoryFree(srcDir); - taosMemoryFree(dstDir); - return code; -} -SBkdMgt* bkdMgtCreate(char* path) { - SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt)); - p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - taosThreadRwlockInit(&p->rwLock, NULL); - return p; -} - -void bkdMgtDestroy(SBkdMgt* bm) { - if (bm == NULL) return; - void* pIter = taosHashIterate(bm->pDbChkpTbl, NULL); - while (pIter) { - SDbChkp* pChkp = *(SDbChkp**)(pIter); - dbChkpDestroy(pChkp); - - pIter = taosHashIterate(bm->pDbChkpTbl, pIter); - } - - taosThreadRwlockDestroy(&bm->rwLock); - - taosMemoryFree(bm); -} -int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list) { - int32_t code = 0; - taosThreadRwlockWrlock(&bm->rwLock); - - SDbChkp* pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - code = dbChkpGetDelta(pChkp, chkpId, list); - - taosThreadRwlockUnlock(&bm->rwLock); - return code; -} - -int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { - int32_t code = -1; - - taosThreadRwlockWrlock(&bm->rwLock); - SDbChkp** pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); - if (pp == NULL) { - SDbChkp* p = dbChkpCreate(path, 0); - if (p != NULL) { - taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void*)); - code = 0; - } - } else { - stError("task chkp already exists"); - } - - taosThreadRwlockUnlock(&bm->rwLock); - - return code; -} - -int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { - int32_t code = 0; - taosThreadRwlockRdlock(&bm->rwLock); - - SDbChkp* p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - code = dbChkpDumpTo(p, dname); - - taosThreadRwlockUnlock(&bm->rwLock); - return code; -} - bool isValidCheckpoint(const char* dir) { return true; } int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { @@ -1582,22 +1279,6 @@ int parKeyToString(void* k, char* buf) { n = sprintf(buf + n, "[groupId:%" PRIi64 "]", *key); return n; } -// int stremaValueEncode(void* k, char* buf) { -// int len = 0; -// SStreamValue* key = k; -// len += taosEncodeFixedI64((void**)&buf, key->unixTimestamp); -// len += taosEncodeFixedI32((void**)&buf, key->len); -// len += taosEncodeBinary((void**)&buf, key->data, key->len); -// return len; -// } -// int streamValueDecode(void* k, char* buf) { -// SStreamValue* key = k; -// char* p = buf; -// p = taosDecodeFixedI64(p, &key->unixTimestamp); -// p = taosDecodeFixedI32(p, &key->len); -// p = taosDecodeBinary(p, (void**)&key->data, key->len); -// return p - buf; -// } int32_t valueToString(void* k, char* buf) { SStreamValue* key = k; int n = 0; @@ -1874,7 +1555,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); rocksdb_compactionfilterfactory_t* filterFactory = @@ -2181,7 +1862,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[idx]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); pCompare[i] = compare; } @@ -2265,7 +1946,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t SCfInit* cfPara = &ginitDict[i]; rocksdb_comparator_t* compare = - rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_comparator_create(NULL, cfPara->destroyCmp, cfPara->cmpKey, cfPara->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); inst->pCompares[i] = compare; @@ -2345,7 +2026,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpFunc, cf->cmpName); + rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } @@ -3672,4 +3353,305 @@ _err: taosMemoryFreeClear(dstName); taosCloseDir(&pDir); return code >= 0 ? 0 : -1; +} + +int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { + int32_t code = 0; + size_t len = 0; + void* pIter = taosHashIterate(p2, NULL); + while (pIter) { + char* name = taosHashGetKey(pIter, &len); + if (!taosHashGet(p1, name, len)) { + char* p = taosStrdup(name); + taosArrayPush(diff, &p); + } + pIter = taosHashIterate(p2, pIter); + } + return code; +} +int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { + int32_t code = 0; + + code = compareHashTableImpl(p1, p2, add); + code = compareHashTableImpl(p2, p1, del); + + return code; +} + +int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { + taosThreadRwlockWrlock(&p->rwLock); + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; + const char* pCurrent = "CURRENT"; + int32_t currLen = strlen(pCurrent); + + const char* pManifest = "MANIFEST-"; + int32_t maniLen = strlen(pManifest); + + const char* pSST = ".sst"; + int32_t sstLen = strlen(pSST); + + memset(p->buf, 0, p->len); + sprintf(p->buf, "%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, chkpId); + + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + + TdDirPtr pDir = taosOpenDir(p->buf); + TdDirEntryPtr de = NULL; + int8_t dummy = 0; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; + if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { + taosMemoryFreeClear(p->pCurrent); + p->pCurrent = taosStrdup(name); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + + if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { + taosMemoryFreeClear(p->pManifest); + p->pManifest = taosStrdup(name); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { + // char* p = taosStrdup(name); + taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + continue; + } + } + if (p->init == 0) { + void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); + while (pIter) { + size_t len; + char* name = taosHashGetKey(pIter, &len); + if (name != NULL && len != 0) { + taosArrayPush(p->pAdd, &name); + } + pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); + } + if (taosArrayGetSize(p->pAdd) > 0) p->update = 1; + + p->init = 1; + p->preCkptId = -1; + p->curChkpId = chkpId; + } else { + int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); + if (code != 0) { + // dead code + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + taosHashClear(p->pSstTbl[1 - p->idx]); + p->update = 0; + taosCloseDir(&pDir); + return code; + } + + if (taosArrayGetSize(p->pAdd) == 0 && taosArrayGetSize(p->pDel) == 0) { + p->update = 0; + } + + p->preCkptId = p->curChkpId; + p->curChkpId = chkpId; + } + + taosHashClear(p->pSstTbl[p->idx]); + p->idx = 1 - p->idx; + + taosCloseDir(&pDir); + taosThreadRwlockUnlock(&p->rwLock); + + return 0; +} + +SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { + SDbChkp* p = taosMemoryCalloc(1, sizeof(SDbChkp)); + p->curChkpId = initChkpId; + p->preCkptId = -1; + p->pSST = taosArrayInit(64, sizeof(void*)); + p->path = taosStrdup(path); + p->len = strlen(path) + 128; + p->buf = taosMemoryCalloc(1, p->len); + + p->idx = 0; + p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + + p->pAdd = taosArrayInit(64, sizeof(void*)); + p->pDel = taosArrayInit(64, sizeof(void*)); + p->update = 0; + taosThreadRwlockInit(&p->rwLock, NULL); + + SArray* list = NULL; + int32_t code = dbChkpGetDelta(p, initChkpId, list); + + return p; +} + +void dbChkpDestroy(SDbChkp* pChkp) { + taosMemoryFree(pChkp->buf); + taosMemoryFree(pChkp->path); + + taosArrayDestroyP(pChkp->pSST, taosMemoryFree); + taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); + taosArrayDestroyP(pChkp->pDel, taosMemoryFree); + + taosHashCleanup(pChkp->pSstTbl[0]); + taosHashCleanup(pChkp->pSstTbl[1]); + + taosMemoryFree(pChkp->pCurrent); + taosMemoryFree(pChkp->pManifest); +} + +int32_t dbChkpInit(SDbChkp* p) { + if (p == NULL) return 0; + return 0; +} +int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { + taosThreadRwlockRdlock(&p->rwLock); + int32_t code = 0; + int32_t len = p->len + 128; + + char* srcBuf = taosMemoryCalloc(1, len); + char* dstBuf = taosMemoryCalloc(1, len); + + char* srcDir = taosMemoryCalloc(1, len); + char* dstDir = taosMemoryCalloc(1, len); + + sprintf(srcDir, "%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoint", p->curChkpId); + sprintf(dstDir, "%s%s%s", p->path, TD_DIRSEP, dname); + + if (!taosDirExist(srcDir)) { + stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); + code = -1; + goto _ERROR; + } + + code = taosMkDir(dstDir); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); + goto _ERROR; + } + + // clear current file + memset(dstBuf, 0, len); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); + taosRemoveFile(dstBuf); + + memset(dstBuf, 0, len); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); + taosRemoveFile(dstBuf); + + // add file to $name dir + for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { + memset(dstBuf, 0, len); + memset(srcBuf, 0, len); + + char* filename = taosArrayGetP(p->pAdd, i); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); + + taosCopyFile(srcBuf, dstBuf); + } + // del file in $name + for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { + memset(dstBuf, 0, len); + memset(srcBuf, 0, len); + + char* filename = taosArrayGetP(p->pDel, i); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); + taosRemoveFile(dstBuf); + } + + // copy current file to dst dir + memset(srcBuf, 0, len); + memset(dstBuf, 0, len); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); + taosCopyFile(srcBuf, dstBuf); + + // copy manifest file to dst dir + memset(srcBuf, 0, len); + memset(dstBuf, 0, len); + sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); + sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); + taosCopyFile(srcBuf, dstBuf); + + // clear delta data buf + taosArrayClearP(p->pAdd, taosMemoryFree); + taosArrayClearP(p->pDel, taosMemoryFree); + +_ERROR: + taosThreadRwlockUnlock(&p->rwLock); + taosMemoryFree(srcBuf); + taosMemoryFree(dstBuf); + taosMemoryFree(srcDir); + taosMemoryFree(dstDir); + return code; +} +SBkdMgt* bkdMgtCreate(char* path) { + SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt)); + p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + taosThreadRwlockInit(&p->rwLock, NULL); + return p; +} + +void bkdMgtDestroy(SBkdMgt* bm) { + if (bm == NULL) return; + void* pIter = taosHashIterate(bm->pDbChkpTbl, NULL); + while (pIter) { + SDbChkp* pChkp = *(SDbChkp**)(pIter); + dbChkpDestroy(pChkp); + + pIter = taosHashIterate(bm->pDbChkpTbl, pIter); + } + + taosThreadRwlockDestroy(&bm->rwLock); + + taosMemoryFree(bm); +} +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list) { + int32_t code = 0; + taosThreadRwlockWrlock(&bm->rwLock); + + SDbChkp* pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpGetDelta(pChkp, chkpId, list); + + taosThreadRwlockUnlock(&bm->rwLock); + return code; +} + +int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { + int32_t code = -1; + + taosThreadRwlockWrlock(&bm->rwLock); + SDbChkp** pp = taosHashGet(bm->pDbChkpTbl, task, strlen(task)); + if (pp == NULL) { + SDbChkp* p = dbChkpCreate(path, 0); + if (p != NULL) { + taosHashPut(bm->pDbChkpTbl, task, strlen(task), &p, sizeof(void*)); + code = 0; + } + } else { + stError("task chkp already exists"); + } + + taosThreadRwlockUnlock(&bm->rwLock); + + return code; +} + +int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { + int32_t code = 0; + taosThreadRwlockRdlock(&bm->rwLock); + + SDbChkp* p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + code = dbChkpDumpTo(p, dname); + + taosThreadRwlockUnlock(&bm->rwLock); + return code; } \ No newline at end of file From 0b4946eefbc9e232d083d72c096c290dcc8ef7de Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 17:25:04 +0800 Subject: [PATCH 091/151] merge 3.0 --- source/common/src/tglobal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c7d2091383..7339c70487 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -675,10 +675,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddString(pCfg, "snodeIp", tsSnodeIp, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "snodeIp", tsSnodeIp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) From 594da2d6031182299a0d0f6470d0423a8ea9d722 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 10:04:46 +0800 Subject: [PATCH 092/151] merge 3.0 --- source/libs/stream/inc/streamBackendRocksdb.h | 60 ++++++++++++++++++- source/libs/stream/src/streamBackendRocksdb.c | 51 +--------------- 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 122f803ab6..c9cb5ab64c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -66,7 +66,7 @@ typedef struct { TdThreadMutex mutex; char* idstr; char* path; - int64_t refId; + int64_t refId; void* pTask; int64_t streamId; @@ -79,6 +79,53 @@ typedef struct { } STaskDbWrapper; +typedef struct SDbChkp { + int8_t init; + char* pCurrent; + char* pManifest; + SArray* pSST; + int64_t preCkptId; + int64_t curChkpId; + char* path; + + char* buf; + int32_t len; + + // ping-pong buf + SHashObj* pSstTbl[2]; + int8_t idx; + + SArray* pAdd; + SArray* pDel; + int8_t update; + + TdThreadRwlock rwLock; +} SDbChkp; +typedef struct { + int8_t init; + char* pCurrent; + char* pManifest; + SArray* pSST; + int64_t preCkptId; + int64_t curChkpId; + char* path; + + char* buf; + int32_t len; + + // ping-pong buf + SHashObj* pSstTbl[2]; + int8_t idx; + + SArray* pAdd; + SArray* pDel; + int8_t update; + + SHashObj* pDbChkpTbl; + + TdThreadRwlock rwLock; +} SBkdMgt; + void* streamBackendInit(const char* path, int64_t chkpId); void streamBackendCleanup(void* arg); void streamBackendHandleCleanup(void* arg); @@ -194,4 +241,15 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); int32_t taskDbBuildSnap(void* arg, SArray* pSnap); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); + +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); +void taskDbDestroy(void* pDb); + +int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); + +SBkdMgt* bkdMgtCreate(char* path); +int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path); +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list); +int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); +void bkdMgtDestroy(SBkdMgt* bm); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 074d23cdbf..a06ec8c9c6 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -20,53 +20,6 @@ #include "tcommon.h" #include "tref.h" -typedef struct SDbChkp { - int8_t init; - char* pCurrent; - char* pManifest; - SArray* pSST; - int64_t preCkptId; - int64_t curChkpId; - char* path; - - char* buf; - int32_t len; - - // ping-pong buf - SHashObj* pSstTbl[2]; - int8_t idx; - - SArray* pAdd; - SArray* pDel; - int8_t update; - - TdThreadRwlock rwLock; -} SDbChkp; -typedef struct { - int8_t init; - char* pCurrent; - char* pManifest; - SArray* pSST; - int64_t preCkptId; - int64_t curChkpId; - char* path; - - char* buf; - int32_t len; - - // ping-pong buf - SHashObj* pSstTbl[2]; - int8_t idx; - - SArray* pAdd; - SArray* pDel; - int8_t update; - - SHashObj* pDbChkpTbl; - - TdThreadRwlock rwLock; -} SBkdMgt; - typedef struct SCompactFilteFactory { void* status; } SCompactFilteFactory; @@ -223,8 +176,6 @@ SCfInit ginitDict[] = { valueEncode, valueDecode, compactFilteFactoryCreateFilter, destroyCompactFilteFactory, compactFilteFactoryName}, }; -const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; - int32_t getCfIdx(const char* cfName) { int idx = -1; size_t len = strlen(cfName); @@ -868,7 +819,6 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId); taskDbRemoveRef(pTaskDb); - pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); SStreamTask* pTask = pTaskDb->pTask; SStreamTaskSnap snap = {.streamId = pTask->id.streamId, @@ -876,6 +826,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { .chkpId = pTaskDb->chkpId, .dbPrefixPath = taosStrdup(pTaskDb->path)}; taosArrayPush(pSnap, &snap); + pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); } return code; } From 17bd6badec1063bd78fddd7b82c603fec4e93ff6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 8 Nov 2023 10:47:54 +0800 Subject: [PATCH 093/151] refactor(stream): do checkpoint for each task. --- source/dnode/vnode/src/tq/tq.c | 7 +- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamCheckpoint.c | 120 +++++++++++----------- 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1f0f6a3e92..4d5ddec233 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1702,13 +1702,12 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) return TSDB_CODE_SUCCESS; } - // todo handle failure to reset from checkpoint procedure - // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. + // Downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. if (pTask->status.downstreamReady != 1) { pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id pTask->checkpointingId = req.checkpointId; - qError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 + tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 ", set it failure", pTask->id.idStr, req.checkpointId); streamMetaReleaseTask(pMeta, pTask); @@ -1735,6 +1734,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } + + // todo: already in checkpoint status, return error streamProcessCheckpointSourceReq(pTask, &req); taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index feaacb7969..095461bd92 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -112,7 +112,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId); +int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t checkpointId); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 3a52e68e93..bf2c89bea4 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -146,7 +146,6 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo pTask->execInfo.checkpoint += 1; // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task - // already. int32_t code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER); return code; } @@ -169,9 +168,8 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); int64_t checkpointId = pDataBlock->info.version; - - const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + int32_t code = TSDB_CODE_SUCCESS; // set task status if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) { @@ -197,15 +195,15 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc // todo fix race condition: set the status and append checkpoint block int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { - if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || - pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + int8_t type = pTask->outputInfo.type; + if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); - } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); - streamProcessCheckpointReadyMsg(pTask); - streamFreeQitem((SStreamQueueItem*)pBlock); - } + } else { // only one task exists, no need to dispatch downstream info + atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); + streamProcessCheckpointReadyMsg(pTask); + streamFreeQitem((SStreamQueueItem*)pBlock); + } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) > 0); if (pTask->chkInfo.startTs == 0) { @@ -231,11 +229,9 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc id, num); streamFreeQitem((SStreamQueueItem*)pBlock); streamTaskBuildCheckpoint(pTask); - } else { - stDebug( - "s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, dispatch checkpoint msg " - "downstream", - id, num); + } else { // source & agg tasks need to forward the checkpoint msg downwards + stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, continue forwards msg", id, + num); // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure @@ -282,48 +278,47 @@ void streamTaskClearCheckInfo(SStreamTask* pTask) { streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks } -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { +int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t checkpointId) { int32_t vgId = pMeta->vgId; int32_t code = 0; streamMetaWLock(pMeta); - for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) { - STaskId* pId = taosArrayGet(pMeta->pTaskList, i); - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); - if (ppTask == NULL) { - continue; - } + // for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) { + // STaskId* pId = taosArrayGet(pMeta->pTaskList, i); + // SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); + // if (ppTask == NULL) { + // continue; + // } - SStreamTask* p = *ppTask; - if (p->info.fillHistory == 1) { - continue; - } - - ASSERT(p->chkInfo.checkpointId < p->checkpointingId && p->checkpointingId == checkpointId); - - p->chkInfo.checkpointId = p->checkpointingId; - streamTaskClearCheckInfo(p); - - char* str = NULL; - streamTaskGetStatus(p, &str); - - code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); - if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId); - streamMetaWUnLock(pMeta); - return -1; - } else { // save the task - streamMetaSaveTask(pMeta, p); - } - - stDebug( - "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, " - "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, - str); + // SStreamTask* p = *ppTask; + if (p->info.fillHistory == 1) { + // continue; } + ASSERT(p->chkInfo.checkpointId < p->checkpointingId && p->checkpointingId == checkpointId); + + p->chkInfo.checkpointId = p->checkpointingId; + streamTaskClearCheckInfo(p); + + char* str = NULL; + streamTaskGetStatus(p, &str); + + code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + if (code != TSDB_CODE_SUCCESS) { + stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId); + streamMetaWUnLock(pMeta); + return -1; + } else { // save the task + streamMetaSaveTask(pMeta, p); + } + + stDebug( + "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, " + "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", + pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, + str); + code = streamMetaCommit(pMeta); if (code < 0) { stError("vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", pMeta->vgId, @@ -341,24 +336,24 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { // check for all tasks, and do generate the vnode-wide checkpoint data. SStreamMeta* pMeta = pTask->pMeta; - int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); - ASSERT(remain >= 0); +// int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); +// ASSERT(remain >= 0); double el = (taosGetTimestampMs() - pTask->chkInfo.startTs) / 1000.0; - if (remain == 0) { // all tasks are ready +// if (remain == 0) { // all tasks are ready stDebug("s-task:%s all downstreams are ready, ready for do checkpoint", pTask->id.idStr); - streamBackendDoCheckpoint(pMeta, pTask->checkpointingId); - streamSaveAllTaskStatus(pMeta, pTask->checkpointingId); + streamBackendDoCheckpoint(pTask->pBackend, pTask->checkpointingId); + streamSaveAllTaskStatus(pMeta, pTask, pTask->checkpointingId); stInfo( "vgId:%d vnode wide checkpoint completed, save all tasks status, last:%s, level:%d elapsed time:%.2f Sec " "checkpointId:%" PRId64, pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); - } else { - stInfo( - "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " - "not ready:%d/%d", - pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); - } +// } else { +// stInfo( +// "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " +// "not ready:%d/%d", +// pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); +// } // send check point response to upstream task if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -368,6 +363,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { } if (code != TSDB_CODE_SUCCESS) { + // record the failure checkpoint id + pTask->chkInfo.failedId = pTask->checkpointingId; + // todo: let's retry send rsp to upstream/mnode stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, pTask->checkpointingId, tstrerror(code)); From 7b0891981ecfcbc1a16843a6e92a9c0dbabe9474 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 14:14:01 +0800 Subject: [PATCH 094/151] merge 3.0 --- include/libs/stream/tstream.h | 105 ++++++++++++++-------------- source/libs/stream/src/streamMeta.c | 19 +++-- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e4bca14a9e..b41fc75782 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -39,16 +39,16 @@ extern "C" { #define TASK_DOWNSTREAM_NOT_LEADER 0x2 #define TASK_SELF_NEW_STAGE 0x3 -#define TASK_DOWNSTREAM_READY 0x0 -#define TASK_DOWNSTREAM_NOT_READY 0x1 -#define TASK_DOWNSTREAM_NOT_LEADER 0x2 -#define TASK_UPSTREAM_NEW_STAGE 0x3 +#define TASK_DOWNSTREAM_READY 0x0 +#define TASK_DOWNSTREAM_NOT_READY 0x1 +#define TASK_DOWNSTREAM_NOT_LEADER 0x2 +#define TASK_UPSTREAM_NEW_STAGE 0x3 -#define NODE_ROLE_UNINIT 0x1 -#define NODE_ROLE_LEADER 0x2 -#define NODE_ROLE_FOLLOWER 0x3 +#define NODE_ROLE_UNINIT 0x1 +#define NODE_ROLE_LEADER 0x2 +#define NODE_ROLE_FOLLOWER 0x3 -#define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0) +#define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0) #define CLEAR_RELATED_FILLHISTORY_TASK(_t) \ do { \ (_t)->hTaskInfo.id.taskId = 0; \ @@ -378,7 +378,7 @@ typedef struct SHistoryTaskInfo { int32_t tickCount; int32_t retryTimes; int32_t waitInterval; - int64_t haltVer; // offset in wal when halt the stream task + int64_t haltVer; // offset in wal when halt the stream task } SHistoryTaskInfo; typedef struct STaskOutputInfo { @@ -442,7 +442,7 @@ typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; int32_t startAllTasksFlag; - SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing int32_t elapsedTime; } STaskStartInfo; @@ -453,32 +453,32 @@ typedef struct STaskUpdateInfo { // meta typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasksMap; - SArray* pTaskList; // SArray - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - int64_t stage; - int32_t role; - STaskStartInfo startInfo; - SRWLatch lock; - int32_t walScanCounter; - void* streamBackend; - int64_t streamBackendRid; - SHashObj* pTaskDbUnique; - TdThreadMutex backendMutex; - SMetaHbInfo* pHbInfo; + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasksMap; + SArray* pTaskList; // SArray + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + int64_t stage; + int32_t role; + STaskStartInfo startInfo; + SRWLatch lock; + int32_t walScanCounter; + void* streamBackend; + int64_t streamBackendRid; + SHashObj* pTaskDbUnique; + TdThreadMutex backendMutex; + SMetaHbInfo* pHbInfo; STaskUpdateInfo updateInfo; - SHashObj* pUpdateTaskSet; - int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta - int32_t numOfPausedTasks; - int32_t chkptNotReadyTasks; - int64_t rid; + SHashObj* pUpdateTaskSet; + int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta + int32_t numOfPausedTasks; + int32_t chkptNotReadyTasks; + int64_t rid; int64_t chkpId; int32_t chkpCap; @@ -486,6 +486,7 @@ typedef struct SStreamMeta { SArray* chkpInUse; SRWLatch chkpDirLock; + void* qHandle; int32_t pauseTaskNum; } SStreamMeta; @@ -658,8 +659,8 @@ typedef struct STaskStatusEntry { typedef struct SStreamHbMsg { int32_t vgId; int32_t numOfTasks; - SArray* pTaskStatus; // SArray - SArray* pUpdateNodes; // SArray, needs update the epsets in stream tasks for those nodes. + SArray* pTaskStatus; // SArray + SArray* pUpdateNodes; // SArray, needs update the epsets in stream tasks for those nodes. } SStreamHbMsg; int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp); @@ -683,7 +684,7 @@ typedef struct SNodeUpdateInfo { } SNodeUpdateInfo; typedef struct SStreamTaskNodeUpdateMsg { - int32_t transId; // to identify the msg + int32_t transId; // to identify the msg int64_t streamId; int32_t taskId; SArray* pNodeList; // SArray @@ -740,7 +741,7 @@ const char* streamTaskGetStatusStr(ETaskStatus status); void streamTaskResetStatus(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask); -void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); +void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); // recover and fill history void streamTaskCheckDownstream(SStreamTask* pTask); @@ -771,18 +772,18 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer) int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); // common -int32_t streamRestoreParam(SStreamTask* pTask); -void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); -void streamTaskResume(SStreamTask* pTask); -void streamTaskEnablePause(SStreamTask* pTask); -int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); -void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); -void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); -void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); -int32_t streamTaskReleaseState(SStreamTask* pTask); -int32_t streamTaskReloadState(SStreamTask* pTask); -void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); -void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); +int32_t streamRestoreParam(SStreamTask* pTask); +void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); +void streamTaskResume(SStreamTask* pTask); +void streamTaskEnablePause(SStreamTask* pTask); +int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); +void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); +void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); +void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); +int32_t streamTaskReleaseState(SStreamTask* pTask); +int32_t streamTaskReloadState(SStreamTask* pTask); +void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); +void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); @@ -815,7 +816,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaReloadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char *key); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaInitForSnode(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b6c973f0d0..5e5166cc34 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -18,6 +18,7 @@ #include "streamInt.h" #include "tmisce.h" #include "tref.h" +#include "tsched.h" #include "tstream.h" #include "ttimer.h" #include "wal.h" @@ -227,7 +228,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { return 0; } -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char *key) { +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { SStreamTask* pTask = arg; int64_t chkpId = pTask->checkpointingId; @@ -320,8 +321,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->expandFunc = expandFunc; pMeta->stage = stage; - - pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); @@ -350,10 +349,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); pMeta->pHbInfo->tickCounter = 0; pMeta->pHbInfo->stopFlag = 0; - + pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); return pMeta; - _err: +_err: taosMemoryFree(pMeta->path); if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); @@ -483,7 +482,7 @@ void streamMetaCloseImpl(void* arg) { taosHashCleanup(pMeta->pTasksMap); taosHashCleanup(pMeta->pTaskDbUnique); taosHashCleanup(pMeta->pUpdateTaskSet); - //taosHashCleanup(pMeta->pTaskBackendUnique); + // taosHashCleanup(pMeta->pTaskBackendUnique); taosHashCleanup(pMeta->updateInfo.pTasks); taosHashCleanup(pMeta->startInfo.pReadyTaskSet); @@ -1144,7 +1143,9 @@ void metaHbToMnode(void* param, void* tmrId) { } tEncoderClear(&encoder); - SRpcMsg msg = {.info.noResp = 1,}; + SRpcMsg msg = { + .info.noResp = 1, + }; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); pMeta->pHbInfo->hbCount += 1; @@ -1156,7 +1157,7 @@ void metaHbToMnode(void* param, void* tmrId) { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } - _end: +_end: clearHbMsg(&hbMsg, pIdList); taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); @@ -1251,7 +1252,6 @@ void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); taosRUnLockLatch(&pMeta->lock); - } void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); @@ -1261,4 +1261,3 @@ void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosWUnLockLatch(&pMeta->lock); } - From 99dbbe61d7b3b35521823f8b4f2608118bc6c5be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 8 Nov 2023 15:20:47 +0800 Subject: [PATCH 095/151] refactor(stream): do some internal refactor. --- include/libs/stream/tstream.h | 7 +- source/dnode/vnode/src/tq/tq.c | 6 +- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamCheckpoint.c | 98 ++++++++--------------- source/libs/stream/src/streamMeta.c | 9 +-- 5 files changed, 44 insertions(+), 78 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e4bca14a9e..d46c107712 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -294,6 +294,9 @@ typedef struct SCheckpointInfo { int64_t checkpointVer; // latest checkpointId version int64_t nextProcessVer; // current offset in WAL, not serialize it int64_t failedId; // record the latest failed checkpoint id + int64_t checkpointingId; + int32_t downstreamAlignNum; + int32_t checkpointNotReadyTasks; int64_t msgVer; } SCheckpointInfo; @@ -427,9 +430,6 @@ struct SStreamTask { int64_t checkReqId; SArray* checkReqIds; // shuffle int32_t refCnt; - int64_t checkpointingId; - int32_t checkpointAlignCnt; - int32_t checkpointNotReadyTasks; int32_t transferStateAlignCnt; struct SStreamMeta* pMeta; SSHashObj* pNameMap; @@ -477,7 +477,6 @@ typedef struct SStreamMeta { SHashObj* pUpdateTaskSet; int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta int32_t numOfPausedTasks; - int32_t chkptNotReadyTasks; int64_t rid; int64_t chkpId; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4d5ddec233..2d831b9dd1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1705,7 +1705,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // Downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. if (pTask->status.downstreamReady != 1) { pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id - pTask->checkpointingId = req.checkpointId; + pTask->chkInfo.checkpointingId = req.checkpointId; tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 ", set it failure", @@ -1744,10 +1744,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // set the initial value for generating check point // set the mgmt epset info according to the checkout source msg from mnode, todo update mgmt epset if needed - if (pMeta->chkptNotReadyTasks == 0) { - pMeta->chkptNotReadyTasks = pMeta->numOfStreamTasks; - } - total = pMeta->numOfStreamTasks; streamMetaWUnLock(pMeta); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 095461bd92..7e2e1d6553 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -112,7 +112,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t checkpointId); +int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index bf2c89bea4..4f6778a286 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -94,12 +94,12 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea static int32_t streamAlignCheckpoint(SStreamTask* pTask) { int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); - int64_t old = atomic_val_compare_exchange_32(&pTask->checkpointAlignCnt, 0, num); + int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num); if (old == 0) { stDebug("s-task:%s set initial align upstream num:%d", pTask->id.idStr, num); } - return atomic_sub_fetch_32(&pTask->checkpointAlignCnt, 1); + return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1); } static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) { @@ -117,7 +117,7 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint } pBlock->info.type = STREAM_CHECKPOINT; - pBlock->info.version = pTask->checkpointingId; + pBlock->info.version = pTask->chkInfo.checkpointingId; pBlock->info.rows = 1; pBlock->info.childId = pTask->info.selfChildId; @@ -140,8 +140,8 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo // 1. set task status to be prepared for check point, no data are allowed to put into inputQ. streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); - pTask->checkpointingId = pReq->checkpointId; - pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.checkpointingId = pReq->checkpointId; + pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); pTask->chkInfo.startTs = taosGetTimestampMs(); pTask->execInfo.checkpoint += 1; @@ -173,7 +173,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc // set task status if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) { - pTask->checkpointingId = checkpointId; + pTask->chkInfo.checkpointingId = checkpointId; code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code)); @@ -181,17 +181,6 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc } } - { // todo: remove this when the pipeline checkpoint generating is used. - SStreamMeta* pMeta = pTask->pMeta; - streamMetaWLock(pMeta); - - if (pMeta->chkptNotReadyTasks == 0) { - pMeta->chkptNotReadyTasks = pMeta->numOfStreamTasks; - } - - streamMetaWUnLock(pMeta); - } - // todo fix race condition: set the status and append checkpoint block int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { @@ -200,7 +189,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); + atomic_add_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); streamProcessCheckpointReadyMsg(pTask); streamFreeQitem((SStreamQueueItem*)pBlock); } @@ -235,7 +224,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure - pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task // already. And then, dispatch check point msg to all downstream tasks @@ -254,7 +243,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task - int32_t notReady = atomic_sub_fetch_32(&pTask->checkpointNotReadyTasks, 1); + int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); ASSERT(notReady >= 0); if (notReady == 0) { @@ -270,35 +259,27 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { } void streamTaskClearCheckInfo(SStreamTask* pTask) { - pTask->checkpointingId = 0; // clear the checkpoint id + pTask->chkInfo.checkpointingId = 0; // clear the checkpoint id pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time - pTask->checkpointNotReadyTasks = 0; - pTask->checkpointAlignCnt = 0; + pTask->chkInfo.downstreamAlignNum = 0; + pTask->chkInfo.checkpointNotReadyTasks = 0; streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks } -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t checkpointId) { - int32_t vgId = pMeta->vgId; - int32_t code = 0; +int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { + SStreamMeta* pMeta = p->pMeta; + int32_t vgId = pMeta->vgId; + int32_t code = 0; - streamMetaWLock(pMeta); - - // for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) { - // STaskId* pId = taosArrayGet(pMeta->pTaskList, i); - // SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); - // if (ppTask == NULL) { - // continue; - // } - - // SStreamTask* p = *ppTask; if (p->info.fillHistory == 1) { - // continue; + return code; } - ASSERT(p->chkInfo.checkpointId < p->checkpointingId && p->checkpointingId == checkpointId); + streamMetaWLock(pMeta); + ASSERT(p->chkInfo.checkpointId < p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId); - p->chkInfo.checkpointId = p->checkpointingId; + p->chkInfo.checkpointId = p->chkInfo.checkpointingId; streamTaskClearCheckInfo(p); char* str = NULL; @@ -316,8 +297,7 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t chec stDebug( "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, " "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, - str); + vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, str); code = streamMetaCommit(pMeta); if (code < 0) { @@ -332,28 +312,21 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t chec } int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; // check for all tasks, and do generate the vnode-wide checkpoint data. - SStreamMeta* pMeta = pTask->pMeta; -// int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); -// ASSERT(remain >= 0); + int64_t checkpointStartTs = pTask->chkInfo.startTs; - double el = (taosGetTimestampMs() - pTask->chkInfo.startTs) / 1000.0; -// if (remain == 0) { // all tasks are ready - stDebug("s-task:%s all downstreams are ready, ready for do checkpoint", pTask->id.idStr); - streamBackendDoCheckpoint(pTask->pBackend, pTask->checkpointingId); - streamSaveAllTaskStatus(pMeta, pTask, pTask->checkpointingId); - stInfo( - "vgId:%d vnode wide checkpoint completed, save all tasks status, last:%s, level:%d elapsed time:%.2f Sec " - "checkpointId:%" PRId64, - pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); -// } else { -// stInfo( -// "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " -// "not ready:%d/%d", -// pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); -// } + // sink task do not need to save the status, and generated the checkpoint + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + stDebug("s-task:%s level:%d start gen checkpoint", pTask->id.idStr, pTask->info.taskLevel); + streamBackendDoCheckpoint(pTask->pBackend, pTask->chkInfo.checkpointingId); + streamSaveTaskCheckpointInfo(pTask, pTask->chkInfo.checkpointingId); + } + + double el = (taosGetTimestampMs() - checkpointStartTs) / 1000.0; + stInfo("s-task:%s vgId:%d checkpointId:%" PRId64 " save all tasks status, level:%d elapsed time:%.2f Sec ", + pTask->id.idStr, pTask->pMeta->vgId, pTask->chkInfo.checkpointingId, pTask->info.taskLevel, el); // send check point response to upstream task if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -364,17 +337,16 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // record the failure checkpoint id - pTask->chkInfo.failedId = pTask->checkpointingId; + pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; // todo: let's retry send rsp to upstream/mnode stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, - pTask->checkpointingId, tstrerror(code)); + pTask->chkInfo.checkpointingId, tstrerror(code)); } return code; } - //static int64_t kBlockSize = 64 * 1024; //static int sendCheckpointToS3(char* id, SArray* fileList){ // code = s3PutObjectFromFile2(from->fname, object_name); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b6c973f0d0..1a814eaa84 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -230,7 +230,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char *key) { SStreamTask* pTask = arg; - int64_t chkpId = pTask->checkpointingId; + int64_t chkpId = pTask->chkInfo.checkpointingId; taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); @@ -442,7 +442,6 @@ void streamMetaClear(SStreamMeta* pMeta) { taosArrayClear(pMeta->chkpInUse); pMeta->numOfStreamTasks = 0; pMeta->numOfPausedTasks = 0; - pMeta->chkptNotReadyTasks = 0; streamMetaResetStartInfo(&pMeta->startInfo); } @@ -1078,9 +1077,9 @@ void metaHbToMnode(void* param, void* tmrId) { entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } - if ((*pTask)->checkpointingId != 0) { - entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->checkpointingId); - entry.activeCheckpointId = (*pTask)->checkpointingId; + if ((*pTask)->chkInfo.checkpointingId != 0) { + entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); + entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; } if ((*pTask)->exec.pWalReader != NULL) { From f949333e8b73ade4a0e58816bf74130be6204502 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 8 Nov 2023 16:57:55 +0800 Subject: [PATCH 096/151] refactor:do some internal refactor. --- source/libs/stream/src/streamCheckpoint.c | 112 ++++++++++++++-------- 1 file changed, 74 insertions(+), 38 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 4f6778a286..e95c90fcaa 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -270,80 +270,116 @@ void streamTaskClearCheckInfo(SStreamTask* pTask) { int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { SStreamMeta* pMeta = p->pMeta; int32_t vgId = pMeta->vgId; + const char* id = p->id.idStr; int32_t code = 0; if (p->info.fillHistory == 1) { return code; } - streamMetaWLock(pMeta); - ASSERT(p->chkInfo.checkpointId < p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId); + taosThreadMutexLock(&p->lock); + ASSERT(p->chkInfo.checkpointId < p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId); p->chkInfo.checkpointId = p->chkInfo.checkpointingId; + streamTaskClearCheckInfo(p); char* str = NULL; streamTaskGetStatus(p, &str); code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + taosThreadMutexUnlock(&p->lock); + if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId); - streamMetaWUnLock(pMeta); + stDebug("s-task:%s vgId:%d handle event:checkpoint-done failed", id, vgId); return -1; - } else { // save the task - streamMetaSaveTask(pMeta, p); } - stDebug( - "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, " - "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, str); + stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64 + ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: normal, prev:%s", + vgId, id, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, str); - code = streamMetaCommit(pMeta); - if (code < 0) { - stError("vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", pMeta->vgId, - checkpointId, terrstr()); - } else { - stInfo("vgId:%d commit stream meta after do checkpoint, checkpointId:%" PRId64 " DONE", pMeta->vgId, checkpointId); + // save the task if not sink task + if (p->info.taskLevel != TASK_LEVEL__SINK) { + streamMetaWLock(pMeta); + + code = streamMetaSaveTask(pMeta, p); + if (code != TSDB_CODE_SUCCESS) { + streamMetaWUnLock(pMeta); + stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, + vgId, checkpointId, terrstr()); + return code; + } + + code = streamMetaCommit(pMeta); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", + id, vgId, checkpointId, terrstr()); + } + + streamMetaWUnLock(pMeta); } - - streamMetaWUnLock(pMeta); return code; } +void streamTaskSetFailedId(SStreamTask* pTask) { + pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; + pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; +} + int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; - - // check for all tasks, and do generate the vnode-wide checkpoint data. - int64_t checkpointStartTs = pTask->chkInfo.startTs; + int64_t startTs = pTask->chkInfo.startTs; + int64_t ckId = pTask->chkInfo.checkpointingId; // sink task do not need to save the status, and generated the checkpoint if (pTask->info.taskLevel != TASK_LEVEL__SINK) { stDebug("s-task:%s level:%d start gen checkpoint", pTask->id.idStr, pTask->info.taskLevel); - streamBackendDoCheckpoint(pTask->pBackend, pTask->chkInfo.checkpointingId); - streamSaveTaskCheckpointInfo(pTask, pTask->chkInfo.checkpointingId); + code = streamBackendDoCheckpoint(pTask->pBackend, ckId); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, tstrerror(terrno)); + } } - double el = (taosGetTimestampMs() - checkpointStartTs) / 1000.0; - stInfo("s-task:%s vgId:%d checkpointId:%" PRId64 " save all tasks status, level:%d elapsed time:%.2f Sec ", - pTask->id.idStr, pTask->pMeta->vgId, pTask->chkInfo.checkpointingId, pTask->info.taskLevel, el); - // send check point response to upstream task - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - code = streamTaskSendCheckpointSourceRsp(pTask); - } else { - code = streamTaskSendCheckpointReadyMsg(pTask); + if (code == TSDB_CODE_SUCCESS) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + ckId, tstrerror(code)); + } } - if (code != TSDB_CODE_SUCCESS) { - // record the failure checkpoint id - pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; - - // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, - pTask->chkInfo.checkpointingId, tstrerror(code)); + // clear the checkpoint info, and commit the newest checkpoint info if all works are done successfully + if (code == TSDB_CODE_SUCCESS) { + code = streamSaveTaskCheckpointInfo(pTask, ckId); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s commit taskInfo failed, checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, + tstrerror(terrno)); + } } + if (code != TSDB_CODE_SUCCESS) { // clear the checkpoint info if failed + taosThreadMutexLock(&pTask->lock); + streamTaskClearCheckInfo(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + taosThreadMutexUnlock(&pTask->lock); + + streamTaskSetFailedId(pTask); + stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%"PRId64, pTask->id.idStr, ckId); + } + + double el = (taosGetTimestampMs() - startTs) / 1000.0; + stInfo("s-task:%s vgId:%d level:%d, checkpointId:%" PRId64 " ver:%" PRId64 " elapsed time:%.2f Sec, %s ", + pTask->id.idStr, pTask->pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el, + (code == TSDB_CODE_SUCCESS) ? "succ" : "failed"); + return code; } From 76fcc49c6d247de7f9410e7413ea271387585d0f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 17:10:54 +0800 Subject: [PATCH 097/151] fix stream transfer error --- source/libs/stream/inc/streamInt.h | 5 +- source/libs/stream/src/streamCheckpoint.c | 90 +++++++++++------------ source/libs/stream/src/streamMeta.c | 17 +++++ 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 095461bd92..418c4c2848 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -152,7 +152,7 @@ void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); //#define CHECKPOINT_PATH_LEN 128 -//typedef struct SChekpointDataHeader{ +// typedef struct SChekpointDataHeader{ // int64_t size; // char name[CHECKPOINT_PATH_LEN]; // char id[CHECKPOINT_PATH_LEN]; @@ -162,6 +162,9 @@ int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); +typedef int32_t (*__stream_async_exec_fn_t)(void* param); + +int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index bf2c89bea4..3f1be03e75 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "streamInt.h" #include "rsync.h" +#include "streamInt.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -199,11 +199,11 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); - } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); - streamProcessCheckpointReadyMsg(pTask); - streamFreeQitem((SStreamQueueItem*)pBlock); - } + } else { // only one task exists, no need to dispatch downstream info + atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1); + streamProcessCheckpointReadyMsg(pTask); + streamFreeQitem((SStreamQueueItem*)pBlock); + } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) > 0); if (pTask->chkInfo.startTs == 0) { @@ -229,7 +229,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc id, num); streamFreeQitem((SStreamQueueItem*)pBlock); streamTaskBuildCheckpoint(pTask); - } else { // source & agg tasks need to forward the checkpoint msg downwards + } else { // source & agg tasks need to forward the checkpoint msg downwards stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, continue forwards msg", id, num); @@ -331,29 +331,29 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t chec return code; } -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { - int32_t code = 0; - +int32_t streamTaskBuildCheckpointImpl(void* arg) { + int32_t code = 0; + SStreamTask* pTask = arg; // check for all tasks, and do generate the vnode-wide checkpoint data. SStreamMeta* pMeta = pTask->pMeta; -// int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); -// ASSERT(remain >= 0); + // int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); + // ASSERT(remain >= 0); double el = (taosGetTimestampMs() - pTask->chkInfo.startTs) / 1000.0; -// if (remain == 0) { // all tasks are ready - stDebug("s-task:%s all downstreams are ready, ready for do checkpoint", pTask->id.idStr); - streamBackendDoCheckpoint(pTask->pBackend, pTask->checkpointingId); - streamSaveAllTaskStatus(pMeta, pTask, pTask->checkpointingId); - stInfo( - "vgId:%d vnode wide checkpoint completed, save all tasks status, last:%s, level:%d elapsed time:%.2f Sec " - "checkpointId:%" PRId64, - pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); -// } else { -// stInfo( -// "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " -// "not ready:%d/%d", -// pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); -// } + // if (remain == 0) { // all tasks are ready + stDebug("s-task:%s all downstreams are ready, ready for do checkpoint", pTask->id.idStr); + streamBackendDoCheckpoint(pTask->pBackend, pTask->checkpointingId); + streamSaveAllTaskStatus(pMeta, pTask, pTask->checkpointingId); + stInfo( + "vgId:%d vnode wide checkpoint completed, save all tasks status, last:%s, level:%d elapsed time:%.2f Sec " + "checkpointId:%" PRId64, + pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); + // } else { + // stInfo( + // "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec + // " "not ready:%d/%d", pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, + // pMeta->numOfStreamTasks); + // } // send check point response to upstream task if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -373,14 +373,17 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } +int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { + int32_t code = 0; + return streamMetaAsyncExec(pTask->pMeta, streamTaskBuildCheckpointImpl, pTask, NULL); +} - -//static int64_t kBlockSize = 64 * 1024; -//static int sendCheckpointToS3(char* id, SArray* fileList){ +// static int64_t kBlockSize = 64 * 1024; +// static int sendCheckpointToS3(char* id, SArray* fileList){ // code = s3PutObjectFromFile2(from->fname, object_name); // return 0; //} -//static int sendCheckpointToSnode(char* id, SArray* fileList){ +// static int sendCheckpointToSnode(char* id, SArray* fileList){ // if(strlen(id) >= CHECKPOINT_PATH_LEN){ // tqError("uploadCheckpoint id name too long, name:%s", id); // return -1; @@ -466,41 +469,38 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { //} -int uploadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ +int uploadCheckpoint(char* id, char* path) { + if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("uploadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ + if (strlen(tsSnodeIp) != 0) { uploadRsync(id, path); -// }else if(tsS3StreamEnabled){ - + // }else if(tsS3StreamEnabled){ } return 0; } -int downloadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ +int downloadCheckpoint(char* id, char* path) { + if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("downloadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ + if (strlen(tsSnodeIp) != 0) { downloadRsync(id, path); -// }else if(tsS3StreamEnabled){ - + // }else if(tsS3StreamEnabled){ } return 0; } -int deleteCheckpoint(char* id){ - if(id == NULL || strlen(id) == 0){ +int deleteCheckpoint(char* id) { + if (id == NULL || strlen(id) == 0) { stError("deleteCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ + if (strlen(tsSnodeIp) != 0) { deleteRsync(id); -// }else if(tsS3StreamEnabled){ - + // }else if(tsS3StreamEnabled){ } return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 5e5166cc34..4b344478a1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -490,6 +490,7 @@ void streamMetaCloseImpl(void* arg) { taosMemoryFree(pMeta->path); taosThreadMutexDestroy(&pMeta->backendMutex); + taosCleanUpScheduler(pMeta->qHandle); pMeta->role = NODE_ROLE_UNINIT; taosMemoryFree(pMeta); stDebug("end to close stream meta"); @@ -1261,3 +1262,19 @@ void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosWUnLockLatch(&pMeta->lock); } +static void execHelper(struct SSchedMsg* pSchedMsg) { + __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); + if (code != 0 && pSchedMsg->msg != NULL) { + *(int32_t*)pSchedMsg->msg = code; + } +} + +int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = execHelper; + schedMsg.ahandle = fn; + schedMsg.thandle = param; + schedMsg.msg = code; + return taosScheduleTask(pMeta->qHandle, &schedMsg); +} From 2de5c4a97d1c3f98a8eae0045a9d12a463714691 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 17:11:11 +0800 Subject: [PATCH 098/151] add bg thread --- source/libs/stream/src/streamCheckpoint.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 3f1be03e75..d208fabe99 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -331,9 +331,8 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t chec return code; } -int32_t streamTaskBuildCheckpointImpl(void* arg) { - int32_t code = 0; - SStreamTask* pTask = arg; +int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { + int32_t code = 0; // check for all tasks, and do generate the vnode-wide checkpoint data. SStreamMeta* pMeta = pTask->pMeta; // int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); @@ -373,10 +372,6 @@ int32_t streamTaskBuildCheckpointImpl(void* arg) { return code; } -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { - int32_t code = 0; - return streamMetaAsyncExec(pTask->pMeta, streamTaskBuildCheckpointImpl, pTask, NULL); -} // static int64_t kBlockSize = 64 * 1024; // static int sendCheckpointToS3(char* id, SArray* fileList){ From 08e344849804e8978582c36c2e8e6b9eaf26c825 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 19:54:57 +0800 Subject: [PATCH 099/151] add seq --- source/dnode/mnode/impl/inc/mndDef.h | 8 +- source/dnode/mnode/impl/inc/mndStream.h | 5 + source/dnode/mnode/impl/src/mndStream.c | 209 +++++++++++++----------- source/dnode/mnode/sdb/inc/sdb.h | 37 +++-- 4 files changed, 144 insertions(+), 115 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index efa99db74b..1e4bae17f6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -699,6 +699,11 @@ typedef struct { } SStreamObj; +typedef struct SStreamSeq { + char name[24]; + uint64_t seq; + SRWLatch lock; +} SStreamSeq; int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver); void tFreeStreamObj(SStreamObj* pObj); @@ -730,14 +735,13 @@ typedef struct { int8_t type; int32_t numOfCols; SSchema* pSchema; - SRWLatch lock; + SRWLatch lock; } SViewObj; int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj); int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver); void tFreeSViewObj(SViewObj* pObj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 19fd2a3fd4..6813bb5497 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -34,6 +34,11 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +SSdbRaw * mndStreamSeqActionEncode(SStreamObj *pStream); +SSdbRow * mndStreamSeqActionDecode(SSdbRaw *pRaw); +static int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream); +static int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream); +static int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream); // for sma // TODO refactor int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7e0667ddbf..fcdf024a59 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -43,17 +43,17 @@ typedef struct SNodeEntry { } SNodeEntry; typedef struct SStreamExecInfo { - SArray *pNodeEntryList; + SArray * pNodeEntryList; int64_t ts; // snapshot ts int64_t activeCheckpoint; // active check point id - SHashObj *pTaskMap; - SArray *pTaskList; + SHashObj * pTaskMap; + SArray * pTaskList; TdThreadMutex lock; } SStreamExecInfo; typedef struct SVgroupChangeInfo { SHashObj *pDBMap; - SArray *pUpdateNodeList; // SArray + SArray * pUpdateNodeList; // SArray } SVgroupChangeInfo; static int32_t mndNodeCheckSentinel = 0; @@ -78,7 +78,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static SArray *extractNodeListFromStream(SMnode *pMnode); -static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady); +static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); @@ -91,7 +91,7 @@ static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExe static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t doKillActiveCheckpointTrans(SMnode *pMnode); -static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList); +static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); int32_t mndInitStream(SMnode *pMnode) { SSdbTable table = { @@ -103,6 +103,15 @@ int32_t mndInitStream(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndStreamActionUpdate, .deleteFp = (SdbDeleteFp)mndStreamActionDelete, }; + SSdbTable tableSeq = { + .sdbType = SDB_STREAM_SEQ, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStreamSeqActionEncode, + .decodeFp = (SdbDecodeFp)mndStreamSeqActionDecode, + .insertFp = (SdbInsertFp)mndStreamSeqActionInsert, + .updateFp = (SdbUpdateFp)mndStreamSeqActionUpdate, + .deleteFp = (SdbDeleteFp)mndStreamSeqActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); @@ -135,7 +144,13 @@ int32_t mndInitStream(SMnode *pMnode) { execInfo.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); - return sdbSetTable(pMnode->pSdb, table); + if (sdbSetTable(pMnode->pSdb, table) != 0) { + return -1; + } + if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { + return -1; + } + return 0; } void mndCleanupStream(SMnode *pMnode) { @@ -193,9 +208,9 @@ STREAM_ENCODE_OVER: SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRow *pRow = NULL; + SSdbRow * pRow = NULL; SStreamObj *pStream = NULL; - void *buf = NULL; + void * buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) { @@ -272,7 +287,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream } SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName); if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; @@ -300,6 +315,12 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { } } +SSdbRaw * mndStreamSeqActionEncode(SStreamObj *pStream) { return NULL; } +SSdbRow * mndStreamSeqActionDecode(SSdbRaw *pRaw) { return NULL; } +static int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +static int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +static int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream) { return 0; } + static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { int8_t trigger = pStream->conf.trigger; if (trigger == STREAM_TRIGGER_AT_ONCE) { @@ -325,7 +346,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 return TSDB_CODE_SUCCESS; } - SNode *pAst = NULL; + SNode * pAst = NULL; int32_t code = nodesStringToNode(ast, &pAst); SQueryPlan *pPlan = NULL; @@ -350,7 +371,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 } static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) { - SNode *pAst = NULL; + SNode * pAst = NULL; SQueryPlan *pPlan = NULL; mInfo("stream:%s to create", pCreate->name); @@ -589,7 +610,7 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; - SDbObj *pDb = NULL; + SDbObj * pDb = NULL; SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); @@ -715,10 +736,10 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) } static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; int32_t code = -1; - SStreamObj *pStream = NULL; - SDbObj *pDb = NULL; + SStreamObj * pStream = NULL; + SDbObj * pDb = NULL; SCMCreateStreamReq createStreamReq = {0}; SStreamObj streamObj = {0}; @@ -761,7 +782,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { int32_t numOfStream = 0; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -858,12 +879,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - //reuse this function for stream + // reuse this function for stream // TODO if (createStreamReq.sql != NULL) { - auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, - createStreamReq.sql, strlen(createStreamReq.sql)); + auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, createStreamReq.sql, + strlen(createStreamReq.sql)); } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { @@ -879,7 +900,7 @@ _OVER: static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { return 0; } @@ -919,7 +940,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in return -1; } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void * abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); tEncodeStreamCheckpointSourceReq(&encoder, &req); @@ -960,7 +981,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre int32_t totLevel = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < totLevel; i++) { - SArray *pLevel = taosArrayGetP(pStream->tasks, i); + SArray * pLevel = taosArrayGetP(pStream->tasks, i); SStreamTask *pTask = taosArrayGetP(pLevel, 0); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t sz = taosArrayGetSize(pLevel); @@ -973,7 +994,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre goto _ERR; } - void *buf; + void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, pTask->id.taskId) < 0) { @@ -1038,7 +1059,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream int32_t totLevel = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < totLevel; i++) { - SArray *pLevel = taosArrayGetP(pStream->tasks, i); + SArray * pLevel = taosArrayGetP(pStream->tasks, i); SStreamTask *pTask = taosArrayGetP(pLevel, 0); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -1055,7 +1076,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream return -1; } - void *buf; + void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, chkptId, pTask->id.streamId, pTask->id.taskId) < 0) { @@ -1066,7 +1087,8 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream STransAction action = {0}; SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); + initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, + TSDB_CODE_SYN_PROPOSE_NOT_READY); mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1106,9 +1128,9 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream } static const char *mndGetStreamDB(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { @@ -1147,11 +1169,10 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); goto _EXIT; } - bool allReady = true; + bool allReady = true; SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); if (!allReady) { - mWarn("not all vnodes are ready, ignore the checkpoint") - taosArrayDestroy(pNodeSnapshot); + mWarn("not all vnodes are ready, ignore the checkpoint") taosArrayDestroy(pNodeSnapshot); return 0; } @@ -1170,7 +1191,7 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { // check if all tasks are in TASK_STATUS__NORMAL status for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { - STaskId *p = taosArrayGet(execInfo.pTaskList, i); + STaskId * p = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); if (pEntry == NULL) { continue; @@ -1190,9 +1211,9 @@ _EXIT: return ready == true ? 0 : -1; } static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; + void * pIter = NULL; SStreamObj *pStream = NULL; int32_t code = 0; if ((code = mndCheckNodeStatus(pMnode)) != 0) { @@ -1215,7 +1236,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { } static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMDropStreamReq dropReq = {0}; @@ -1343,7 +1364,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { } int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -1351,7 +1372,7 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) } int32_t numOfStreams = 0; - void *pIter = NULL; + void * pIter = NULL; while (1) { SStreamObj *pStream = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -1370,8 +1391,8 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) } static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; @@ -1447,8 +1468,8 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { } static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; @@ -1537,13 +1558,13 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // status char status[20 + VARSTR_HEADER_SIZE] = {0}; - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); if (pe == NULL) { continue; } - const char* pStatus = streamTaskGetStatusStr(pe->status); + const char *pStatus = streamTaskGetStatusStr(pe->status); STR_TO_VARSTR(status, pStatus); // status @@ -1664,7 +1685,7 @@ static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, in } static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMPauseStreamReq pauseReq = {0}; @@ -1780,7 +1801,7 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn } static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMResumeStreamReq pauseReq = {0}; @@ -1865,7 +1886,7 @@ static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChang } static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, - SStreamTaskId* pId, int32_t transId) { + SStreamTaskId *pId, int32_t transId) { SStreamTaskNodeUpdateMsg req = {0}; initNodeUpdateMsg(&req, pInfo, pId, transId); @@ -1888,7 +1909,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha return -1; } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void * abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); tEncodeStreamTaskUpdateMsg(&encoder, &req); @@ -1955,7 +1976,7 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p for (int32_t k = 0; k < numOfTasks; ++k) { SStreamTask *pTask = taosArrayGetP(pLevel, k); - void *pBuf = NULL; + void * pBuf = NULL; int32_t len = 0; streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); @@ -1987,8 +2008,8 @@ static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) // 1. increase the replica does not affect the stream process. // 2. decreasing the replica may affect the stream task execution in the way that there is one or more running stream // tasks on the will be removed replica. -// 3. vgroup redistribution is an combination operation of first increase replica and then decrease replica. So we will -// handle it as mentioned in 1 & 2 items. +// 3. vgroup redistribution is an combination operation of first increase replica and then decrease replica. So we +// will handle it as mentioned in 1 & 2 items. static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList) { SVgroupChangeInfo info = { .pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)), @@ -2030,9 +2051,9 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP return info; } -static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; +static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { + SSdb * pSdb = pMnode->pSdb; + void * pIter = NULL; SVgObj *pVgroup = NULL; *allReady = true; @@ -2079,8 +2100,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange // check all streams that involved this vnode should update the epset info SStreamObj *pStream = NULL; - void *pIter = NULL; - STrans *pTrans = NULL; + void * pIter = NULL; + STrans * pTrans = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2141,9 +2162,9 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange } static SArray *extractNodeListFromStream(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); while (1) { @@ -2190,9 +2211,9 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) { } static void doExtractTasksFromStream(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2240,12 +2261,12 @@ static bool taskNodeExists(SArray *pList, int32_t nodeId) { } int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { - SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); + SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList); - for(int32_t i = 0; i < numOfTask; ++i) { - STaskId* pId = taosArrayGet(execInfo.pTaskList, i); - STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + for (int32_t i = 0; i < numOfTask; ++i) { + STaskId * pId = taosArrayGet(execInfo.pTaskList, i); + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { @@ -2253,18 +2274,18 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { } } - for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) { - STaskId* pId = taosArrayGet(pRemovedTasks, i); + for (int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) { + STaskId *pId = taosArrayGet(pRemovedTasks, i); doRemoveTasks(&execInfo, pId); } mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks), - (int32_t) taosArrayGetSize(execInfo.pTaskList)); + (int32_t)taosArrayGetSize(execInfo.pTaskList)); int32_t size = taosArrayGetSize(pNodeSnapshot); - SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); - for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { - SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i); + SArray *pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { + SNodeEntry *p = taosArrayGet(execInfo.pNodeEntryList, i); for (int32_t j = 0; j < size; ++j) { SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); @@ -2278,7 +2299,7 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList); execInfo.pNodeEntryList = pValidNodeEntryList; - mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList)); + mDebug("remain %d valid node entries", (int32_t)taosArrayGetSize(pValidNodeEntryList)); taosArrayDestroy(pRemovedTasks); return 0; } @@ -2310,7 +2331,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { return 0; } - bool allVnodeReady = true; + bool allVnodeReady = true; SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady); if (!allVnodeReady) { taosArrayDestroy(pNodeSnapshot); @@ -2324,7 +2345,6 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot); if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) { - // kill current active checkpoint transaction, since the transaction is vnode wide. doKillActiveCheckpointTrans(pMnode); code = mndProcessVgroupChange(pMnode, &changeInfo); @@ -2359,7 +2379,7 @@ typedef struct SMStreamNodeCheckMsg { static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { return 0; } @@ -2383,7 +2403,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { SStreamTask *pTask = taosArrayGetP(pLevel, j); STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); + void * p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); if (p == NULL) { STaskStatusEntry entry = {0}; streamTaskStatusInit(&entry, pTask); @@ -2397,7 +2417,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { } } -void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) { +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { int32_t level = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < level; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); @@ -2407,7 +2427,7 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) { SStreamTask *pTask = taosArrayGetP(pLevel, j); STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); + void * p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); if (p != NULL) { taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id)); @@ -2513,9 +2533,9 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { int32_t doKillActiveCheckpointTrans(SMnode *pMnode) { int32_t transId = 0; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; STrans *pTrans = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -2546,13 +2566,13 @@ int32_t doKillActiveCheckpointTrans(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } -int32_t mndResetFromCheckpoint(SMnode* pMnode) { +int32_t mndResetFromCheckpoint(SMnode *pMnode) { doKillActiveCheckpointTrans(pMnode); // set all tasks status to be normal, refactor later to be stream level, instead of vnode level. - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { @@ -2570,15 +2590,15 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) { return 0; } -int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { +int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { int32_t num = taosArrayGetSize(pNodeList); for (int k = 0; k < num; ++k) { - int32_t* pVgId = taosArrayGet(pNodeList, k); + int32_t *pVgId = taosArrayGet(pNodeList, k); int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); for (int i = 0; i < numOfNodes; ++i) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); if (pNodeEntry->nodeId == *pVgId) { mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId); @@ -2591,12 +2611,11 @@ int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { +static void updateStageInfo(STaskStatusEntry *pTaskEntry, int32_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); - for(int32_t j = 0; j < numOfNodes; ++j) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); + for (int32_t j = 0; j < numOfNodes; ++j) { + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); if (pNodeEntry->nodeId == pTaskEntry->nodeId) { - mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, pTaskEntry->stage, stage, pTaskEntry->id.taskId); @@ -2608,7 +2627,7 @@ static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { } int32_t mndProcessStreamHb(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamHbMsg req = {0}; bool checkpointFailed = false; @@ -2661,15 +2680,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { pTaskEntry->status = p->status; if (p->status != TASK_STATUS__READY) { - mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); + mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); } } // current checkpoint is failed, rollback from the checkpoint trans // kill the checkpoint trans and then set all tasks status to be normal if (checkpointFailed && activeCheckpointId != 0) { - bool allReady = true; - SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady); + bool allReady = true; + SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); taosArrayDestroy(p); if (allReady) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index ddde645fae..0a20dcfd09 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -149,7 +149,8 @@ typedef enum { SDB_FUNC = 20, SDB_IDX = 21, SDB_VIEW = 22, - SDB_MAX = 23 + SDB_STREAM_SEQ = 23, + SDB_MAX = 24 } ESdbType; typedef struct SSdbRaw { @@ -169,11 +170,11 @@ typedef struct SSdbRow { } SSdbRow; typedef struct SSdb { - SMnode *pMnode; - SWal *pWal; + SMnode * pMnode; + SWal * pWal; int64_t sync; - char *currDir; - char *tmpDir; + char * currDir; + char * tmpDir; int64_t commitIndex; int64_t commitTerm; int64_t commitConfig; @@ -183,7 +184,7 @@ typedef struct SSdb { int64_t tableVer[SDB_MAX]; int64_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; - SHashObj *hashObjs[SDB_MAX]; + SHashObj * hashObjs[SDB_MAX]; TdThreadRwlock locks[SDB_MAX]; SdbInsertFp insertFps[SDB_MAX]; SdbUpdateFp updateFps[SDB_MAX]; @@ -198,25 +199,25 @@ typedef struct SSdb { typedef struct SSdbIter { TdFilePtr file; int64_t total; - char *name; + char * name; } SSdbIter; typedef struct { - ESdbType sdbType; - EKeyType keyType; - SdbDeployFp deployFp; - SdbEncodeFp encodeFp; - SdbDecodeFp decodeFp; - SdbInsertFp insertFp; - SdbUpdateFp updateFp; - SdbDeleteFp deleteFp; + ESdbType sdbType; + EKeyType keyType; + SdbDeployFp deployFp; + SdbEncodeFp encodeFp; + SdbDecodeFp decodeFp; + SdbInsertFp insertFp; + SdbUpdateFp updateFp; + SdbDeleteFp deleteFp; SdbValidateFp validateFp; } SSdbTable; typedef struct SSdbOpt { const char *path; - SMnode *pMnode; - SWal *pWal; + SMnode * pMnode; + SWal * pWal; int64_t sync; } SSdbOpt; @@ -393,7 +394,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); SSdbRow *sdbAllocRow(int32_t objSize); -void *sdbGetRowObj(SSdbRow *pRow); +void * sdbGetRowObj(SSdbRow *pRow); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config); From 0f98f4daa58201d2bf5344c6a71530d6d19ba48c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 20:29:37 +0800 Subject: [PATCH 100/151] change check out seq --- source/dnode/mnode/impl/src/mndStream.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fcdf024a59..b3fd81f450 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1219,14 +1219,11 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { if ((code = mndCheckNodeStatus(pMnode)) != 0) { return code; } - SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; - int64_t checkpointId = pMsg->checkpointId; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - - code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId); + code = mndProcessStreamCheckpointTrans(pMnode, pStream, pStream->checkpointId + 1); sdbRelease(pSdb, pStream); if (code == -1) { break; From 06f196b71d0e022a1b4217857ee5e1811d096c3d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Nov 2023 09:08:41 +0800 Subject: [PATCH 101/151] change check out seq --- source/dnode/mnode/impl/src/mndStream.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b3fd81f450..42f883aa2d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1020,7 +1020,6 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre } } // 2. reset tick - pStream->checkpointFreq = checkpointId; pStream->checkpointId = checkpointId; pStream->checkpointFreq = taosGetTimestampMs(); pStream->currentTick = 0; From dd2252598ed402b1bda417182e40c9c31e6f72ca Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Nov 2023 17:39:34 +0800 Subject: [PATCH 102/151] upload by rsync --- source/libs/stream/inc/streamBackendRocksdb.h | 3 ++ source/libs/stream/src/streamBackendRocksdb.c | 38 ++++++++++++++++ source/libs/stream/src/streamCheckpoint.c | 43 +++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index c9cb5ab64c..9d84e76a29 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,6 +17,7 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" +//#include "streamInt.h" #include "streamState.h" #include "tcoding.h" #include "tcommon.h" @@ -252,4 +253,6 @@ int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path); int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list); int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); + +int32_t taskDbGenChkpUploadPath(void* arg, int64_t chkpId, int8_t type, char** pathkj); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a06ec8c9c6..55dd938835 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1691,6 +1691,44 @@ void taskDbDestroy(void* pDb) { return; } +int32_t taskDbGenChkpUplaodPath__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) { + int64_t st = taosGetTimestampMs(); + int32_t code = -1; + int64_t refId = pDb->refId; + + if (taosAcquireRef(taskDbWrapperId, refId) == NULL) { + return -1; + } + char* pChkpDir = NULL; + char* pChkpIdDir = NULL; + if (chkpPreBuildDir(pDb->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { + code = -1; + } + + if (taosIsDir(pChkpIdDir) && isValidCheckpoint(pChkpIdDir)) { + code = 0; + *path = pChkpIdDir; + pChkpIdDir = NULL; + } + + taosMemoryFree(pChkpDir); + taosMemoryFree(pChkpIdDir); + taosReleaseRef(taskDbWrapperId, refId); + + return code; +} +int32_t taskDbGenChkpUploadPath(void* arg, int64_t chkpId, int8_t type, char** path) { + STaskDbWrapper* pDb = arg; + UPLOAD_TYPE utype = type; + + if (utype == UPLOAD_RSYNC) { + return taskDbGenChkpUplaodPath__rsync(pDb, chkpId, path); + } else if (utype == UPLOAD_S3) { + return 0; + } + return -1; +} + int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { int32_t code = 0; char* err = NULL; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 34943454ff..51f2a18504 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -15,8 +15,16 @@ #include "cos.h" #include "rsync.h" +#include "streamBackendRocksdb.h" #include "streamInt.h" +typedef struct { + UPLOAD_TYPE type; + char* taskId; + int64_t chkpId; + + SStreamTask* pTask; +} SAsyncUploadArg; int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -328,6 +336,36 @@ void streamTaskSetFailedId(SStreamTask* pTask) { pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; } +int32_t doUploadChkp(void* param) { + SAsyncUploadArg* arg = param; + char* path = NULL; + int32_t code = 0; + if ((code = taskDbGenChkpUploadPath(arg->pTask->pBackend, arg->chkpId, (int8_t)(arg->type), &path)) != 0) { + stError("s-task:%s faile to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); + } + if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) { + stError("s-task:%s faile to upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); + } + + taosMemoryFree(path); + taosMemoryFree(arg->taskId); + taosMemoryFree(arg); + return 0; +} +int32_t streamTaskUploadChkp(SStreamTask* pTask, int64_t chkpId, char* taskId) { + // async upload + UPLOAD_TYPE type = getUploadType(); + if (type == UPLOAD_DISABLE) { + return 0; + } + SAsyncUploadArg* arg = taosMemoryCalloc(1, sizeof(SAsyncUploadArg)); + arg->type = type; + arg->taskId = taosStrdup(taskId); + arg->chkpId = chkpId; + arg->pTask = pTask; + + return streamMetaAsyncExec(pTask->pMeta, doUploadChkp, arg, NULL); +} int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; int64_t startTs = pTask->chkInfo.startTs; @@ -363,6 +401,11 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s commit taskInfo failed, checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, tstrerror(terrno)); + } else { + code = streamTaskUploadChkp(pTask, ckId, (char*)pTask->id.idStr); + if (code != 0) { + stError("s-task:%s failed to upload checkpoint:%" PRId64 " failed", pTask->id.idStr, ckId); + } } } From 5f9b922a042be4ab65e3db2dc9673a117b251997 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Nov 2023 20:15:08 +0800 Subject: [PATCH 103/151] support s3 --- include/libs/stream/tstream.h | 2 + source/libs/stream/inc/streamBackendRocksdb.h | 4 +- source/libs/stream/src/streamBackendRocksdb.c | 42 ++++++++++++++++--- source/libs/stream/src/streamCheckpoint.c | 6 ++- source/libs/stream/src/streamMeta.c | 2 + 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e3c52c78b0..00f01e4266 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -513,6 +513,8 @@ typedef struct SStreamMeta { void* qHandle; int32_t pauseTaskNum; + + void* bkdChkptMgt; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 9d84e76a29..cd620ed5b7 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -250,9 +250,9 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); SBkdMgt* bkdMgtCreate(char* path); int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path); -int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list); +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* name); int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); -int32_t taskDbGenChkpUploadPath(void* arg, int64_t chkpId, int8_t type, char** pathkj); +int32_t taskDbGenChkpUploadPath(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 55dd938835..253c18467e 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1717,14 +1717,30 @@ int32_t taskDbGenChkpUplaodPath__rsync(STaskDbWrapper* pDb, int64_t chkpId, char return code; } -int32_t taskDbGenChkpUploadPath(void* arg, int64_t chkpId, int8_t type, char** path) { + +int32_t taskDbGenChkpUplaodPath__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path) { + SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; + + char* temp = taosMemoryCalloc(1, strlen(pDb->path)); + sprintf(temp, "%s%s%s", pDb->path, TD_DIRSEP, "tmp"); + + if (!taosDirExist(temp)) { + taosMkDir(temp); + } + bkdMgtGetDelta(p, pDb->idstr, chkpId, NULL, temp); + + *path = temp; + + return 0; +} +int32_t taskDbGenChkpUploadPath(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) { STaskDbWrapper* pDb = arg; UPLOAD_TYPE utype = type; if (utype == UPLOAD_RSYNC) { - return taskDbGenChkpUplaodPath__rsync(pDb, chkpId, path); + return taskDbGenChkpUplaodPath__rsync(pDb,chkpId, path); } else if (utype == UPLOAD_S3) { - return 0; + return taskDbGenChkpUplaodPath__s3(pDb,mgt, chkpId, path); } return -1; } @@ -3603,14 +3619,28 @@ void bkdMgtDestroy(SBkdMgt* bm) { taosMemoryFree(bm); } -int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list) { +int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* dname) { int32_t code = 0; - taosThreadRwlockWrlock(&bm->rwLock); + taosThreadRwlockWrlock(&bm->rwLock); SDbChkp* pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - code = dbChkpGetDelta(pChkp, chkpId, list); + + if (pChkp == NULL) { + char* taskPath = taosMemoryCalloc(1, strlen(bm->path) + 64); + sprintf(taskPath, "%s%s%s", bm->path, TD_DIRSEP, taskId); + + SDbChkp* p = dbChkpCreate(taskPath, chkpId); + taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)); + + taosMemoryFree(taskPath); + } taosThreadRwlockUnlock(&bm->rwLock); + + code = dbChkpGetDelta(pChkp, chkpId, list); + + code = dbChkpDumpTo(pChkp, dname); + return code; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 51f2a18504..03696e1122 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -340,8 +340,10 @@ int32_t doUploadChkp(void* param) { SAsyncUploadArg* arg = param; char* path = NULL; int32_t code = 0; - if ((code = taskDbGenChkpUploadPath(arg->pTask->pBackend, arg->chkpId, (int8_t)(arg->type), &path)) != 0) { - stError("s-task:%s faile to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); + + if ((code = taskDbGenChkpUploadPath(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, + (int8_t)(arg->type), &path)) != 0) { + stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); } if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) { stError("s-task:%s faile to upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index c5b98784a6..102023a728 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -356,6 +356,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->pHbInfo->stopFlag = 0; pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); + pMeta->bkdChkptMgt = bkdMgtCreate(tpath); + return pMeta; _err: From 4a99bdc2b01eb88be26fea2963c7ccce98dea73d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Nov 2023 20:27:30 +0800 Subject: [PATCH 104/151] support s3 --- source/libs/stream/src/streamBackendRocksdb.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 253c18467e..4f5076b7ff 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1738,9 +1738,9 @@ int32_t taskDbGenChkpUploadPath(void* arg, void* mgt, int64_t chkpId, int8_t typ UPLOAD_TYPE utype = type; if (utype == UPLOAD_RSYNC) { - return taskDbGenChkpUplaodPath__rsync(pDb,chkpId, path); + return taskDbGenChkpUplaodPath__rsync(pDb, chkpId, path); } else if (utype == UPLOAD_S3) { - return taskDbGenChkpUplaodPath__s3(pDb,mgt, chkpId, path); + return taskDbGenChkpUplaodPath__s3(pDb, mgt, chkpId, path); } return -1; } @@ -3633,14 +3633,15 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)); taosMemoryFree(taskPath); + pChkp = p; + + code = dbChkpDumpTo(pChkp, dname); + taosThreadRwlockUnlock(&bm->rwLock); + return code; } - taosThreadRwlockUnlock(&bm->rwLock); - code = dbChkpGetDelta(pChkp, chkpId, list); - - code = dbChkpDumpTo(pChkp, dname); - + taosThreadRwlockUnlock(&bm->rwLock); return code; } From 0669037de0f4b805939c5ad0cae04d06412ff51c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Nov 2023 20:37:08 +0800 Subject: [PATCH 105/151] fix mem leak --- source/libs/stream/src/streamMeta.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 102023a728..41f0d8dcfe 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -499,6 +499,8 @@ void streamMetaCloseImpl(void* arg) { taosThreadMutexDestroy(&pMeta->backendMutex); taosCleanUpScheduler(pMeta->qHandle); + bkdMgtDestroy(pMeta->bkdChkptMgt); + pMeta->role = NODE_ROLE_UNINIT; taosMemoryFree(pMeta); stDebug("end to close stream meta"); From 54fe42d3725a03d2009f9291d9378d4cff380ab9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Nov 2023 09:24:40 +0800 Subject: [PATCH 106/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4f5076b7ff..1a29a82c11 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1724,7 +1724,8 @@ int32_t taskDbGenChkpUplaodPath__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 char* temp = taosMemoryCalloc(1, strlen(pDb->path)); sprintf(temp, "%s%s%s", pDb->path, TD_DIRSEP, "tmp"); - if (!taosDirExist(temp)) { + if (taosDirExist(temp)) { + taosRemoveDir(temp); taosMkDir(temp); } bkdMgtGetDelta(p, pDb->idstr, chkpId, NULL, temp); From ad119ea4c0a3690e2b260e17d3e41ddbb800e119 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Nov 2023 12:07:33 +0800 Subject: [PATCH 107/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 1a29a82c11..519019892f 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1691,7 +1691,7 @@ void taskDbDestroy(void* pDb) { return; } -int32_t taskDbGenChkpUplaodPath__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) { +int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) { int64_t st = taosGetTimestampMs(); int32_t code = -1; int64_t refId = pDb->refId; @@ -1718,7 +1718,7 @@ int32_t taskDbGenChkpUplaodPath__rsync(STaskDbWrapper* pDb, int64_t chkpId, char return code; } -int32_t taskDbGenChkpUplaodPath__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path) { +int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path) { SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; char* temp = taosMemoryCalloc(1, strlen(pDb->path)); @@ -1727,6 +1727,8 @@ int32_t taskDbGenChkpUplaodPath__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 if (taosDirExist(temp)) { taosRemoveDir(temp); taosMkDir(temp); + } else { + taosMkDir(temp); } bkdMgtGetDelta(p, pDb->idstr, chkpId, NULL, temp); @@ -1739,9 +1741,9 @@ int32_t taskDbGenChkpUploadPath(void* arg, void* mgt, int64_t chkpId, int8_t typ UPLOAD_TYPE utype = type; if (utype == UPLOAD_RSYNC) { - return taskDbGenChkpUplaodPath__rsync(pDb, chkpId, path); + return taskDbGenChkpUploadData__rsync(pDb, chkpId, path); } else if (utype == UPLOAD_S3) { - return taskDbGenChkpUplaodPath__s3(pDb, mgt, chkpId, path); + return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path); } return -1; } @@ -3399,7 +3401,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { int32_t sstLen = strlen(pSST); memset(p->buf, 0, p->len); - sprintf(p->buf, "%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, chkpId); + sprintf(p->buf, "%s%s%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId); taosArrayClearP(p->pAdd, taosMemoryFree); taosArrayClearP(p->pDel, taosMemoryFree); @@ -3527,8 +3529,8 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { char* srcDir = taosMemoryCalloc(1, len); char* dstDir = taosMemoryCalloc(1, len); - sprintf(srcDir, "%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoint", p->curChkpId); - sprintf(dstDir, "%s%s%s", p->path, TD_DIRSEP, dname); + sprintf(srcDir, "%s%s%s%s%s%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", p->curChkpId); + sprintf(dstDir, "%s", dname); if (!taosDirExist(srcDir)) { stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); @@ -3602,6 +3604,7 @@ _ERROR: SBkdMgt* bkdMgtCreate(char* path) { SBkdMgt* p = taosMemoryCalloc(1, sizeof(SBkdMgt)); p->pDbChkpTbl = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + p->path = taosStrdup(path); taosThreadRwlockInit(&p->rwLock, NULL); return p; } @@ -3624,7 +3627,8 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, int32_t code = 0; taosThreadRwlockWrlock(&bm->rwLock); - SDbChkp* pChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + SDbChkp** ppChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); + SDbChkp* pChkp = NULL; if (pChkp == NULL) { char* taskPath = taosMemoryCalloc(1, strlen(bm->path) + 64); @@ -3639,9 +3643,12 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, code = dbChkpDumpTo(pChkp, dname); taosThreadRwlockUnlock(&bm->rwLock); return code; + } else { + pChkp = *ppChkp; } code = dbChkpGetDelta(pChkp, chkpId, list); + code = dbChkpDumpTo(pChkp, dname); taosThreadRwlockUnlock(&bm->rwLock); return code; } From a3fdb6dd59a98d63b3761e09b01c3d615302a6a9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Nov 2023 16:06:08 +0800 Subject: [PATCH 108/151] Merge branch '3.0' into enh/refactorBackend --- source/common/src/cos.c | 89 ++++++++++--------- source/libs/stream/inc/streamBackendRocksdb.h | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 55 +++++++++--- source/libs/stream/src/streamCheckpoint.c | 4 +- 4 files changed, 90 insertions(+), 60 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 0b6b0db885..c0b44a5df0 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -70,7 +70,7 @@ static void s3PrintError(const char *func, S3Status status, char error_details[] if (status < S3StatusErrorAccessDenied) { uError("%s: %s", __func__, S3_get_status_name(status)); } else { - uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s: %s, %s, %d", __func__, S3_get_status_name(status), error_details, status); } } @@ -85,7 +85,7 @@ typedef struct { char err_msg[128]; S3Status status; uint64_t content_length; - char *buf; + char * buf; int64_t buf_pos; } TS3SizeCBD; @@ -271,7 +271,7 @@ typedef struct MultipartPartData { S3Status status; put_object_callback_data put_object_data; int seq; - UploadManager *manager; + UploadManager * manager; } MultipartPartData; static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { @@ -316,7 +316,7 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; - const char *etag = properties->eTag; + const char * etag = properties->eTag; data->manager->etags[seq - 1] = strdup(etag); data->manager->next_etags_pos = seq; return S3StatusOK; @@ -449,10 +449,10 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { int32_t code = 0; const char *key = object; // const char *uploadId = 0; - const char *filename = 0; + const char * filename = 0; uint64_t contentLength = 0; - const char *cacheControl = 0, *contentType = 0, *md5 = 0; - const char *contentDispositionFilename = 0, *contentEncoding = 0; + const char * cacheControl = 0, *contentType = 0, *md5 = 0; + const char * contentDispositionFilename = 0, *contentEncoding = 0; int64_t expires = -1; S3CannedAcl cannedAcl = S3CannedAclPrivate; int metaPropertiesCount = 0; @@ -466,6 +466,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { data.gb = 0; data.noStatus = noStatus; + uError("ERROR: %s stat file %s: ", __func__, file); if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { uError("ERROR: %s Failed to stat file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); @@ -637,7 +638,7 @@ typedef struct list_bucket_callback_data { char nextMarker[1024]; int keyCount; int allDetails; - SArray *objectArray; + SArray * objectArray; } list_bucket_callback_data; static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, @@ -682,11 +683,11 @@ static void s3FreeObjectKey(void *pItem) { static SArray *getListByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listBucketCallback}; - const char *marker = 0, *delimiter = 0; + const char * marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; data.objectArray = taosArrayInit(32, sizeof(void *)); @@ -725,7 +726,7 @@ static SArray *getListByPrefix(const char *prefix) { void s3DeleteObjects(const char *object_name[], int nobject) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; for (int i = 0; i < nobject; ++i) { @@ -776,7 +777,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &getObjectDataCallback}; @@ -814,7 +815,7 @@ int32_t s3GetObjectToFile(const char *object_name, char *fileName) { const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, - 0, awsRegionG}; + 0, awsRegionG}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &getObjectCallback}; @@ -845,7 +846,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { if (objectArray == NULL) return -1; for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { - char *object = taosArrayGetP(objectArray, i); + char * object = taosArrayGetP(objectArray, i); const char *tmp = strchr(object, '/'); tmp = (tmp == NULL) ? object : tmp + 1; char fileName[PATH_MAX] = {0}; @@ -934,12 +935,12 @@ static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) { int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket, object, file; - cos_table_t *resp_headers; + cos_table_t * resp_headers; // int traffic_limit = 0; cos_pool_create(&p, NULL); @@ -970,14 +971,14 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) { int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) { int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; - cos_request_options_t *options = NULL; + cos_status_t * s = NULL; + cos_request_options_t * options = NULL; cos_string_t bucket, object, file; - cos_table_t *resp_headers; + cos_table_t * resp_headers; int traffic_limit = 0; - cos_table_t *headers = NULL; + cos_table_t * headers = NULL; cos_resumable_clt_params_t *clt_params = NULL; cos_pool_create(&p, NULL); @@ -1010,11 +1011,11 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) { } void s3DeleteObjectsByPrefix(const char *prefix_str) { - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; cos_request_options_t *options = NULL; int is_cname = 0; cos_string_t bucket; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_string_t prefix; cos_pool_create(&p, NULL); @@ -1029,10 +1030,10 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) { } void s3DeleteObjects(const char *object_name[], int nobject) { - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; cos_string_t bucket; - cos_table_t *resp_headers = NULL; + cos_table_t * resp_headers = NULL; cos_request_options_t *options = NULL; cos_list_t object_list; cos_list_t deleted_object_list; @@ -1066,14 +1067,14 @@ void s3DeleteObjects(const char *object_name[], int nobject) { bool s3Exists(const char *object_name) { bool ret = false; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; - cos_request_options_t *options = NULL; + cos_status_t * s = NULL; + cos_request_options_t * options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers; - cos_table_t *headers = NULL; + cos_table_t * resp_headers; + cos_table_t * headers = NULL; cos_object_exist_status_e object_exist; cos_pool_create(&p, NULL); @@ -1100,15 +1101,15 @@ bool s3Exists(const char *object_name) { bool s3Get(const char *object_name, const char *path) { bool ret = false; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; cos_string_t file; - cos_table_t *resp_headers = NULL; - cos_table_t *headers = NULL; + cos_table_t * resp_headers = NULL; + cos_table_t * headers = NULL; int traffic_limit = 0; //创建内存池 @@ -1144,15 +1145,15 @@ bool s3Get(const char *object_name, const char *path) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { (void)check; int32_t code = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers; - cos_table_t *headers = NULL; - cos_buf_t *content = NULL; + cos_table_t * resp_headers; + cos_table_t * headers = NULL; + cos_buf_t * content = NULL; // cos_string_t file; // int traffic_limit = 0; char range_buf[64]; @@ -1246,7 +1247,7 @@ void s3EvictCache(const char *path, long object_size) { terrno = TAOS_SYSTEM_ERROR(errno); vError("failed to open %s since %s", dir_name, terrstr()); } - SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile)); + SArray * evict_files = taosArrayInit(16, sizeof(SEvictFile)); tdbDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { char *name = taosGetDirEntryName(pDirEntry); @@ -1288,13 +1289,13 @@ void s3EvictCache(const char *path, long object_size) { long s3Size(const char *object_name) { long size = 0; - cos_pool_t *p = NULL; + cos_pool_t * p = NULL; int is_cname = 0; - cos_status_t *s = NULL; + cos_status_t * s = NULL; cos_request_options_t *options = NULL; cos_string_t bucket; cos_string_t object; - cos_table_t *resp_headers = NULL; + cos_table_t * resp_headers = NULL; //创建内存池 cos_pool_create(&p, NULL); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index cd620ed5b7..bce51a9d2f 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -254,5 +254,5 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); -int32_t taskDbGenChkpUploadPath(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj); +int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj); #endif \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 519019892f..604f5f7c2a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1736,7 +1736,7 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 return 0; } -int32_t taskDbGenChkpUploadPath(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) { +int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) { STaskDbWrapper* pDb = arg; UPLOAD_TYPE utype = type; @@ -3363,13 +3363,27 @@ _err: return code >= 0 ? 0 : -1; } +int32_t isBkdDataMeta(char* name) { + const char* pCurrent = "CURRENT"; + int32_t currLen = strlen(pCurrent); + + const char* pManifest = "MANIFEST-"; + int32_t maniLen = strlen(pManifest); + + if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { + return 1; + } else if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { + return 1; + } + return 0; +} int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { int32_t code = 0; size_t len = 0; void* pIter = taosHashIterate(p2, NULL); while (pIter) { char* name = taosHashGetKey(pIter, &len); - if (!taosHashGet(p1, name, len)) { + if (!isBkdDataMeta(name) && !taosHashGet(p1, name, len)) { char* p = taosStrdup(name); taosArrayPush(diff, &p); } @@ -3431,13 +3445,22 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { continue; } } + + void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); + while (pIter) { + char *name = taosHashGetKey(pIter, NULL); + stError("curr file list: %s", name); + pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); + } + if (p->init == 0) { void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { size_t len; char* name = taosHashGetKey(pIter, &len); - if (name != NULL && len != 0) { - taosArrayPush(p->pAdd, &name); + if (name != NULL && !isBkdDataMeta(name)) { + char* fname = taosStrdup(name); + taosArrayPush(p->pAdd, &fname); } pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); } @@ -3538,12 +3561,12 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { goto _ERROR; } - code = taosMkDir(dstDir); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); - goto _ERROR; - } + // code = taosMkDir(dstDir); + // if (code != 0) { + // terrno = TAOS_SYSTEM_ERROR(errno); + // stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); + // goto _ERROR; + // } // clear current file memset(dstBuf, 0, len); @@ -3563,7 +3586,9 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - taosCopyFile(srcBuf, dstBuf); + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + } } // del file in $name for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { @@ -3580,14 +3605,18 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { memset(dstBuf, 0, len); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); - taosCopyFile(srcBuf, dstBuf); + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + } // copy manifest file to dst dir memset(srcBuf, 0, len); memset(dstBuf, 0, len); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); - taosCopyFile(srcBuf, dstBuf); + if (taosCopyFile(srcBuf, dstBuf) < 0) { + stError("failed to copy file from %s to %s", srcBuf, dstBuf); + } // clear delta data buf taosArrayClearP(p->pAdd, taosMemoryFree); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 03696e1122..854cdc12b1 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -341,12 +341,12 @@ int32_t doUploadChkp(void* param) { char* path = NULL; int32_t code = 0; - if ((code = taskDbGenChkpUploadPath(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, + if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, (int8_t)(arg->type), &path)) != 0) { stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); } if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) { - stError("s-task:%s faile to upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); + stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId); } taosMemoryFree(path); From 1e8d46c405a06614103e4e2f6f746a8a2872a158 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Nov 2023 18:37:25 +0800 Subject: [PATCH 109/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 37 +++++++++++-------- source/libs/stream/src/streamCheckpoint.c | 4 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 604f5f7c2a..5c834a6e95 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1722,7 +1722,7 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; char* temp = taosMemoryCalloc(1, strlen(pDb->path)); - sprintf(temp, "%s%s%s", pDb->path, TD_DIRSEP, "tmp"); + sprintf(temp, "%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "tmp", chkpId); if (taosDirExist(temp)) { taosRemoveDir(temp); @@ -3363,16 +3363,16 @@ _err: return code >= 0 ? 0 : -1; } -int32_t isBkdDataMeta(char* name) { +int32_t isBkdDataMeta(char* name, int32_t len) { const char* pCurrent = "CURRENT"; int32_t currLen = strlen(pCurrent); const char* pManifest = "MANIFEST-"; int32_t maniLen = strlen(pManifest); - if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { + if (len >= maniLen && strncmp(name, pManifest, maniLen) == 0) { return 1; - } else if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { + } else if (len == currLen && strcmp(name, pCurrent) == 0) { return 1; } return 0; @@ -3383,9 +3383,10 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { void* pIter = taosHashIterate(p2, NULL); while (pIter) { char* name = taosHashGetKey(pIter, &len); - if (!isBkdDataMeta(name) && !taosHashGet(p1, name, len)) { - char* p = taosStrdup(name); - taosArrayPush(diff, &p); + if (!isBkdDataMeta(name, len) && !taosHashGet(p1, name, len)) { + char* fname = taosMemoryCalloc(1, len + 1); + strncpy(fname, name, len); + taosArrayPush(diff, &fname); } pIter = taosHashIterate(p2, pIter); } @@ -3429,18 +3430,17 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) { taosMemoryFreeClear(p->pCurrent); p->pCurrent = taosStrdup(name); - taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) { taosMemoryFreeClear(p->pManifest); p->pManifest = taosStrdup(name); - taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); + // taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) { - // char* p = taosStrdup(name); taosHashPut(p->pSstTbl[1 - p->idx], name, strlen(name), &dummy, sizeof(dummy)); continue; } @@ -3448,18 +3448,23 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { - char *name = taosHashGetKey(pIter, NULL); - stError("curr file list: %s", name); + size_t len = 0; + char* name = taosHashGetKey(pIter, &len); + + char* buf = taosMemoryCalloc(1, len + 1); + strncpy(buf, name, len); + stError("curr file list: %s", buf); pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); } if (p->init == 0) { void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { - size_t len; + size_t len = 0; char* name = taosHashGetKey(pIter, &len); - if (name != NULL && !isBkdDataMeta(name)) { - char* fname = taosStrdup(name); + if (name != NULL && !isBkdDataMeta(name, len)) { + char* fname = taosMemoryCalloc(1, len + 1); + strncpy(fname, name, len); taosArrayPush(p->pAdd, &fname); } pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); @@ -3579,8 +3584,8 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { // add file to $name dir for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { - memset(dstBuf, 0, len); memset(srcBuf, 0, len); + memset(dstBuf, 0, len); char* filename = taosArrayGetP(p->pAdd, i); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 854cdc12b1..f99713a431 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -348,7 +348,7 @@ int32_t doUploadChkp(void* param) { if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) { stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId); } - + taosRemoveDir(path); taosMemoryFree(path); taosMemoryFree(arg->taskId); taosMemoryFree(arg); @@ -435,6 +435,7 @@ static int uploadCheckpointToS3(char* id, char* path) { if (pDir == NULL) return -1; TdDirEntryPtr de = NULL; + s3Init(); while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || taosDirEntryIsDir(de)) continue; @@ -454,6 +455,7 @@ static int uploadCheckpointToS3(char* id, char* path) { return -1; } stDebug("[s3] upload checkpoint:%s", filename); + break; } taosCloseDir(&pDir); From cf3e01e9426b2e29415aafa75b61522c914f6aa9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Nov 2023 10:34:44 +0800 Subject: [PATCH 110/151] Merge branch '3.0' into enh/refactorBackend --- source/common/src/tglobal.c | 4 ++-- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ab6ff53fbc..58e5e4e79e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -247,7 +247,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; -int32_t tsStreamCheckpointInterval = 60; +int32_t tsStreamCheckpointInterval = 30; float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 30; int32_t tsTtlUnit = 86400; @@ -720,7 +720,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 20, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5c834a6e95..72a7c718bd 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1521,7 +1521,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { } void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { pTaskDb->chkpId = -1; - pTaskDb->chkpCap = 2; + pTaskDb->chkpCap = 4; pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); From 1786f7077d3ed8bb7bf5c0e3fc2cbfb8783668bc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Nov 2023 14:35:23 +0800 Subject: [PATCH 111/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 2 ++ source/libs/stream/src/streamMeta.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b65e26be3b..3740d8fe56 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -3654,6 +3654,8 @@ void bkdMgtDestroy(SBkdMgt* bm) { } taosThreadRwlockDestroy(&bm->rwLock); + taosMemoryFree(bm->path); + taosHashCleanup(bm->pDbChkpTbl); taosMemoryFree(bm); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d144d5a58e..8fb10ddc01 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -500,6 +500,8 @@ void streamMetaCloseImpl(void* arg) { taosThreadMutexDestroy(&pMeta->backendMutex); taosCleanUpScheduler(pMeta->qHandle); + taosMemoryFree(pMeta->qHandle); + bkdMgtDestroy(pMeta->bkdChkptMgt); pMeta->role = NODE_ROLE_UNINIT; From 4104840ceaa909015131ef1ed7e224e77e42c931 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Nov 2023 18:07:38 +0800 Subject: [PATCH 112/151] Merge branch '3.0' into enh/refactorBackend --- source/common/src/cos.c | 2 +- source/common/src/tglobal.c | 4 +- source/libs/stream/inc/streamBackendRocksdb.h | 1 + source/libs/stream/src/streamBackendRocksdb.c | 57 ++++++++++++------- source/libs/stream/src/streamCheckpoint.c | 2 +- 5 files changed, 43 insertions(+), 23 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 509bd31512..09b2b8e4db 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -467,7 +467,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // data.infileFD = NULL; // data.noStatus = noStatus; - uError("ERROR: %s stat file %s: ", __func__, file); + // uError("ERROR: %s stat file %s: ", __func__, file); if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { uError("ERROR: %s Failed to stat file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a1566c23a5..8fe93ae201 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -247,7 +247,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; -int32_t tsStreamCheckpointInterval = 30; +int32_t tsStreamCheckpointInterval = 10; float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 15; int32_t tsTtlUnit = 86400; @@ -721,7 +721,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 20, 1200, CFG_SCOPE_SERVER, + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 1, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index a101b6a09d..92ec1899db 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -77,6 +77,7 @@ typedef struct { SArray* chkpInUse; int32_t chkpCap; TdThreadRwlock chkpDirLock; + int64_t dataWritten; } STaskDbWrapper; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 3740d8fe56..0d18b6a900 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -891,6 +891,7 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { } code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir); + pTaskDb->dataWritten = 0; pTaskDb->chkpId = chkpId; @@ -2161,7 +2162,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ + wrapper->dataWritten += 1; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ @@ -2237,7 +2239,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ + wrapper->dataWritten += 1; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ @@ -2277,6 +2280,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { stDebug("streamStateClear_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -3253,6 +3257,7 @@ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rock int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -3285,7 +3290,9 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb char* ttlV = tmpBuf; int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); - STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; + rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); @@ -3303,6 +3310,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { char* err = NULL; STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; + wrapper->dataWritten += 1; rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { stError("streamState failed to write batch, err:%s", err); @@ -3401,6 +3409,26 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { return code; } +void hashTableToDebug(SHashObj* pTbl) { + size_t sz = taosHashGetSize(pTbl); + int32_t total = 0; + char* buf = taosMemoryCalloc(1, sz * 16); + void* pIter = taosHashIterate(pTbl, NULL); + while (pIter) { + size_t len = 0; + char* name = taosHashGetKey(pIter, &len); + char* tname = taosMemoryCalloc(1, len + 1); + memcpy(tname, name, len); + total += sprintf(buf + total, "%s,", tname); + + pIter = taosHashIterate(pTbl, pIter); + taosMemoryFree(tname); + } + buf[total - 1] = 0; + + stTrace("curr file list:[%s]", buf); + taosMemoryFree(buf); +} int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { taosThreadRwlockWrlock(&p->rwLock); @@ -3420,6 +3448,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { taosArrayClearP(p->pAdd, taosMemoryFree); taosArrayClearP(p->pDel, taosMemoryFree); + taosHashClear(p->pSstTbl[1 - p->idx]); TdDirPtr pDir = taosOpenDir(p->buf); TdDirEntryPtr de = NULL; @@ -3445,17 +3474,13 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { continue; } } + taosCloseDir(&pDir); - void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); - while (pIter) { - size_t len = 0; - char* name = taosHashGetKey(pIter, &len); + stTrace("chkp get file list: 1-1"); + hashTableToDebug(p->pSstTbl[1 - p->idx]); - char* buf = taosMemoryCalloc(1, len + 1); - strncpy(buf, name, len); - stError("curr file list: %s", buf); - pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter); - } + stTrace("chkp get file list: 1-2"); + hashTableToDebug(p->pSstTbl[p->idx]); if (p->init == 0) { void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); @@ -3482,7 +3507,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { taosArrayClearP(p->pDel, taosMemoryFree); taosHashClear(p->pSstTbl[1 - p->idx]); p->update = 0; - taosCloseDir(&pDir); return code; } @@ -3493,11 +3517,8 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { p->preCkptId = p->curChkpId; p->curChkpId = chkpId; } - - taosHashClear(p->pSstTbl[p->idx]); p->idx = 1 - p->idx; - taosCloseDir(&pDir); taosThreadRwlockUnlock(&p->rwLock); return 0; @@ -3664,7 +3685,7 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, taosThreadRwlockWrlock(&bm->rwLock); SDbChkp** ppChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - SDbChkp* pChkp = NULL; + SDbChkp* pChkp = ppChkp != NULL ? *ppChkp : NULL; if (pChkp == NULL) { char* taskPath = taosMemoryCalloc(1, strlen(bm->path) + 64); @@ -3679,8 +3700,6 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, code = dbChkpDumpTo(pChkp, dname); taosThreadRwlockUnlock(&bm->rwLock); return code; - } else { - pChkp = *ppChkp; } code = dbChkpGetDelta(pChkp, chkpId, list); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 946656c7d0..f7ac9e61bc 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -457,7 +457,7 @@ static int uploadCheckpointToS3(char* id, char* path) { return -1; } stDebug("[s3] upload checkpoint:%s", filename); - break; + // break; } taosCloseDir(&pDir); From 2867a2febb43768d923b99d86f620119385b1231 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Nov 2023 18:07:56 +0800 Subject: [PATCH 113/151] Merge branch '3.0' into enh/refactorBackend --- cmake/cmake.options | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cmake.options b/cmake/cmake.options index d34c34dd89..e12de3e4e6 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -151,6 +151,7 @@ IF(${BUILD_S3}) IF(${BUILD_WITH_S3}) +add_definitions(-DUSE_S3) option(BUILD_WITH_COS "If build with cos" OFF) ELSE () From 68b6cc1d9d421891aa6cab6693f4632ab7fbc13f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Nov 2023 19:07:58 +0800 Subject: [PATCH 114/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 0d18b6a900..5c59621ca9 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -3409,25 +3409,58 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { return code; } -void hashTableToDebug(SHashObj* pTbl) { +void hashTableToDebug(SHashObj* pTbl, char** buf) { size_t sz = taosHashGetSize(pTbl); int32_t total = 0; - char* buf = taosMemoryCalloc(1, sz * 16); + char* p = taosMemoryCalloc(1, sz * 16); void* pIter = taosHashIterate(pTbl, NULL); while (pIter) { size_t len = 0; char* name = taosHashGetKey(pIter, &len); char* tname = taosMemoryCalloc(1, len + 1); memcpy(tname, name, len); - total += sprintf(buf + total, "%s,", tname); + total += sprintf(p + total, "%s,", tname); pIter = taosHashIterate(pTbl, pIter); taosMemoryFree(tname); } - buf[total - 1] = 0; + p[total - 1] = 0; + *buf = p; +} +void strArrayDebugInfo(SArray* pArr, char** buf) { + int32_t sz = taosArrayGetSize(pArr); + if (sz <= 0) return; - stTrace("curr file list:[%s]", buf); - taosMemoryFree(buf); + char* p = (char*)taosMemoryCalloc(1, 64 + sz * 64); + int32_t total = 0; + + for (int i = 0; i < sz; i++) { + char* name = taosArrayGetP(pArr, i); + total += sprintf(p + total, "%s,", name); + } + p[total - 1] = 0; + + *buf = p; +} +void dbChkpDebugInfo(SDbChkp* pDb) { + // stTrace("chkp get file list: curr"); + char* p[4] = {NULL}; + + hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]); + stTrace("chkp previous file: [%s]", p[0]); + + hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]); + stTrace("chkp curr file: [%s]", p[1]); + + strArrayDebugInfo(pDb->pAdd, &p[2]); + stTrace("chkp newly addded file: [%s]", p[2]); + + strArrayDebugInfo(pDb->pDel, &p[3]); + stTrace("chkp newly deleted file: [%s]", p[3]); + + for (int i = 0; i < 4; i++) { + taosMemoryFree(p[i]); + } } int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { taosThreadRwlockWrlock(&p->rwLock); @@ -3476,12 +3509,6 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { } taosCloseDir(&pDir); - stTrace("chkp get file list: 1-1"); - hashTableToDebug(p->pSstTbl[1 - p->idx]); - - stTrace("chkp get file list: 1-2"); - hashTableToDebug(p->pSstTbl[p->idx]); - if (p->init == 0) { void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL); while (pIter) { @@ -3517,6 +3544,9 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { p->preCkptId = p->curChkpId; p->curChkpId = chkpId; } + + dbChkpDebugInfo(p); + p->idx = 1 - p->idx; taosThreadRwlockUnlock(&p->rwLock); From 4476e46a5fc3663e0562b528e2602ffaa9a50973 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Nov 2023 17:43:13 +0800 Subject: [PATCH 115/151] rm dup data --- include/common/cos.h | 1 + source/libs/stream/inc/streamBackendRocksdb.h | 4 +- source/libs/stream/inc/streamInt.h | 1 + source/libs/stream/src/streamBackendRocksdb.c | 84 ++++++++++--------- source/libs/stream/src/streamCheckpoint.c | 83 +++++++++++++++++- 5 files changed, 129 insertions(+), 44 deletions(-) diff --git a/include/common/cos.h b/include/common/cos.h index 21b645f604..c6b159c1da 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -43,6 +43,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, int32_t s3GetObjectsByPrefix(const char *prefix, const char *path); void s3EvictCache(const char *path, long object_size); long s3Size(const char *object_name); +int32_t s3GetObjectToFile(const char *object_name, char *fileName); #ifdef __cplusplus } diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 92ec1899db..47404f311f 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -77,7 +77,7 @@ typedef struct { SArray* chkpInUse; int32_t chkpCap; TdThreadRwlock chkpDirLock; - int64_t dataWritten; + int64_t dataWritten; } STaskDbWrapper; @@ -255,5 +255,5 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); -int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj); +int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index bbe7bcf65c..c299d0cfe1 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -163,6 +163,7 @@ int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); int deleteCheckpointFile(char* id, char* name); +int downloadCheckpointByName(char* id, char* fname, char* dstName); int32_t onNormalTaskReady(SStreamTask* pTask); int32_t onScanhistoryTaskReady(SStreamTask* pTask); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5c59621ca9..b156bce5dd 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1719,7 +1719,8 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char return code; } -int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path) { +int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list) { + int32_t code = 0; SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; char* temp = taosMemoryCalloc(1, strlen(pDb->path)); @@ -1731,20 +1732,20 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 } else { taosMkDir(temp); } - bkdMgtGetDelta(p, pDb->idstr, chkpId, NULL, temp); + code = bkdMgtGetDelta(p, pDb->idstr, chkpId, list, temp); *path = temp; - return 0; + return code; } -int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) { +int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list) { STaskDbWrapper* pDb = arg; UPLOAD_TYPE utype = type; if (utype == UPLOAD_RSYNC) { return taskDbGenChkpUploadData__rsync(pDb, chkpId, path); } else if (utype == UPLOAD_S3) { - return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path); + return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list); } return -1; } @@ -3559,7 +3560,7 @@ SDbChkp* dbChkpCreate(char* path, int64_t initChkpId) { p->curChkpId = initChkpId; p->preCkptId = -1; p->pSST = taosArrayInit(64, sizeof(void*)); - p->path = taosStrdup(path); + p->path = path; p->len = strlen(path) + 128; p->buf = taosMemoryCalloc(1, p->len); @@ -3597,9 +3598,9 @@ int32_t dbChkpInit(SDbChkp* p) { if (p == NULL) return 0; return 0; } -int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { +int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { taosThreadRwlockRdlock(&p->rwLock); - int32_t code = 0; + int32_t code = -1; int32_t len = p->len + 128; char* srcBuf = taosMemoryCalloc(1, len); @@ -3613,26 +3614,9 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { if (!taosDirExist(srcDir)) { stError("failed to dump srcDir %s, reason: not exist such dir", srcDir); - code = -1; goto _ERROR; } - // code = taosMkDir(dstDir); - // if (code != 0) { - // terrno = TAOS_SYSTEM_ERROR(errno); - // stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr()); - // goto _ERROR; - // } - - // clear current file - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); - taosRemoveFile(dstBuf); - - memset(dstBuf, 0, len); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); - taosRemoveFile(dstBuf); - // add file to $name dir for (int i = 0; i < taosArrayGetSize(p->pAdd); i++) { memset(srcBuf, 0, len); @@ -3644,39 +3628,59 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) { if (taosCopyFile(srcBuf, dstBuf) < 0) { stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; } } // del file in $name for (int i = 0; i < taosArrayGetSize(p->pDel); i++) { - memset(dstBuf, 0, len); - memset(srcBuf, 0, len); - char* filename = taosArrayGetP(p->pDel, i); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename); - taosRemoveFile(dstBuf); + char* p = taosStrdup(filename); + taosArrayPush(list, &p); } // copy current file to dst dir memset(srcBuf, 0, len); memset(dstBuf, 0, len); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent); + sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pCurrent, p->curChkpId); if (taosCopyFile(srcBuf, dstBuf) < 0) { stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; } // copy manifest file to dst dir memset(srcBuf, 0, len); memset(dstBuf, 0, len); sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest); - sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest); + sprintf(dstBuf, "%s%s%s_%" PRId64 "", dstDir, TD_DIRSEP, p->pManifest, p->curChkpId); if (taosCopyFile(srcBuf, dstBuf) < 0) { stError("failed to copy file from %s to %s", srcBuf, dstBuf); + goto _ERROR; } + static char* chkpMeta = "META"; + memset(dstBuf, 0, len); + sprintf(dstDir, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta); + + TdFilePtr pFile = taosOpenFile(dstDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + stError("chkp failed to create meta file: %s", dstDir); + goto _ERROR; + } + char content[128] = {0}; + snprintf(content, sizeof(content), "%s_%" PRId64 "\n%s_%" PRId64 "", p->pCurrent, p->curChkpId, p->pManifest, + p->curChkpId); + if (taosWriteFile(pFile, content, strlen(content)) <= 0) { + stError("chkp failed to write meta file: %s", dstDir); + taosCloseFile(&pFile); + goto _ERROR; + } + taosCloseFile(&pFile); + // clear delta data buf taosArrayClearP(p->pAdd, taosMemoryFree); taosArrayClearP(p->pDel, taosMemoryFree); + code = 0; _ERROR: taosThreadRwlockUnlock(&p->rwLock); @@ -3718,22 +3722,22 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, SDbChkp* pChkp = ppChkp != NULL ? *ppChkp : NULL; if (pChkp == NULL) { - char* taskPath = taosMemoryCalloc(1, strlen(bm->path) + 64); - sprintf(taskPath, "%s%s%s", bm->path, TD_DIRSEP, taskId); + char* path = taosMemoryCalloc(1, strlen(bm->path) + 64); + sprintf(path, "%s%s%s", bm->path, TD_DIRSEP, taskId); - SDbChkp* p = dbChkpCreate(taskPath, chkpId); + SDbChkp* p = dbChkpCreate(path, chkpId); taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)); - taosMemoryFree(taskPath); pChkp = p; - code = dbChkpDumpTo(pChkp, dname); + code = dbChkpDumpTo(pChkp, dname, list); taosThreadRwlockUnlock(&bm->rwLock); return code; } - code = dbChkpGetDelta(pChkp, chkpId, list); - code = dbChkpDumpTo(pChkp, dname); + code = dbChkpGetDelta(pChkp, chkpId, NULL); + code = dbChkpDumpTo(pChkp, dname, list); + taosThreadRwlockUnlock(&bm->rwLock); return code; } @@ -3763,7 +3767,7 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { taosThreadRwlockRdlock(&bm->rwLock); SDbChkp* p = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); - code = dbChkpDumpTo(p, dname); + code = dbChkpDumpTo(p, dname, NULL); taosThreadRwlockUnlock(&bm->rwLock); return code; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index f7ac9e61bc..b61256371b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -338,18 +338,72 @@ void streamTaskSetFailedId(SStreamTask* pTask) { pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; } +int32_t getChkpMeta(char* id, char* path, SArray* list) { + char* file = taosMemoryCalloc(1, strlen(path) + 32); + sprintf(file, "%s%s%s", path, TD_DIRSEP, "META_TMP"); + int32_t code = downloadCheckpointByName(id, "META", file); + if (code != 0) { + stDebug("chkp failed to download meta file:%s", file); + taosMemoryFree(file); + return code; + } + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + char buf[128] = {0}; + if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { + stError("chkp failed to read meta file:%s", file); + code = -1; + } else { + int32_t len = strlen(buf); + for (int i = 0; i < len; i++) { + if (buf[i] == '\n') { + char* item = taosMemoryCalloc(1, i + 1); + memcpy(item, buf, i); + taosArrayPush(list, &item); + + item = taosMemoryCalloc(1, len - i); + memcpy(item, buf + i + 1, len - i - 1); + taosArrayPush(list, &item); + } + } + } + taosCloseFile(&pFile); + taosRemoveFile(file); + taosMemoryFree(file); + return code; +} int32_t doUploadChkp(void* param) { SAsyncUploadArg* arg = param; char* path = NULL; int32_t code = 0; + SArray* list = taosArrayInit(4, sizeof(void*)); if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, - (int8_t)(arg->type), &path)) != 0) { + (int8_t)(arg->type), &path, list)) != 0) { stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); } - if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) { + + code = getChkpMeta(arg->taskId, path, list); + if (code != 0) { + code = 0; + } + + if (code == 0 && (code = uploadCheckpoint(arg->taskId, path)) != 0) { stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId); } + + if (code == 0) { + for (int i = 0; i < taosArrayGetSize(list); i++) { + char* p = taosArrayGetP(list, i); + code = deleteCheckpointFile(arg->taskId, p); + stDebug("try to del file: %s", p); + if (code != 0) { + break; + } + } + } + + taosArrayDestroyP(list, taosMemoryFree); + taosRemoveDir(path); taosMemoryFree(path); taosMemoryFree(arg->taskId); @@ -464,6 +518,17 @@ static int uploadCheckpointToS3(char* id, char* path) { return 0; } +static int downloadCheckpointByNameS3(char* id, char* fname, char* dstName) { + int code = 0; + char* buf = taosMemoryCalloc(1, strlen(id) + strlen(dstName) + 4); + sprintf(buf, "%s/%s", id, fname); + if (s3GetObjectToFile(buf, dstName) != 0) { + code = -1; + } + taosMemoryFree(buf); + return code; +} + UPLOAD_TYPE getUploadType() { if (strlen(tsSnodeAddress) != 0) { return UPLOAD_RSYNC; @@ -487,6 +552,20 @@ int uploadCheckpoint(char* id, char* path) { return 0; } +// fileName: CURRENT +int downloadCheckpointByName(char* id, char* fname, char* dstName) { + if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) { + stError("uploadCheckpointByName parameters invalid"); + return -1; + } + if (strlen(tsSnodeAddress) != 0) { + return 0; + } else if (tsS3StreamEnabled) { + return downloadCheckpointByNameS3(id, fname, dstName); + } + return 0; +} + int downloadCheckpoint(char* id, char* path) { if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("downloadCheckpoint parameters invalid"); From aa2c09ef5b88c346fb0c387611181acc321ca91f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Nov 2023 18:09:00 +0800 Subject: [PATCH 116/151] rm dup data --- source/libs/stream/src/streamCheckpoint.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b61256371b..74c391f386 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -377,6 +377,7 @@ int32_t doUploadChkp(void* param) { int32_t code = 0; SArray* list = taosArrayInit(4, sizeof(void*)); + if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, (int8_t)(arg->type), &path, list)) != 0) { stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); From 20e1bd9cff0f66de278ede485b39a92298998859 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Nov 2023 20:46:56 +0800 Subject: [PATCH 117/151] fix invalid read/write --- source/libs/stream/src/streamBackendRocksdb.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b156bce5dd..376fb7be85 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1723,7 +1723,7 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 int32_t code = 0; SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; - char* temp = taosMemoryCalloc(1, strlen(pDb->path)); + char* temp = taosMemoryCalloc(1, strlen(pDb->path) + 32); sprintf(temp, "%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "tmp", chkpId); if (taosDirExist(temp)) { @@ -3413,7 +3413,7 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { void hashTableToDebug(SHashObj* pTbl, char** buf) { size_t sz = taosHashGetSize(pTbl); int32_t total = 0; - char* p = taosMemoryCalloc(1, sz * 16); + char* p = taosMemoryCalloc(1, sz * 16 + 4); void* pIter = taosHashIterate(pTbl, NULL); while (pIter) { size_t len = 0; @@ -3425,7 +3425,9 @@ void hashTableToDebug(SHashObj* pTbl, char** buf) { pIter = taosHashIterate(pTbl, pIter); taosMemoryFree(tname); } - p[total - 1] = 0; + if (total > 0) { + p[total - 1] = 0; + } *buf = p; } void strArrayDebugInfo(SArray* pArr, char** buf) { From b74105727d3e1caf76aabb72fba78801ffea7f16 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Nov 2023 20:58:35 +0800 Subject: [PATCH 118/151] fix mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 376fb7be85..4e71eff431 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -899,6 +899,7 @@ _EXIT: taosMemoryFree(pChkpDir); taosMemoryFree(pChkpIdDir); taosReleaseRef(taskDbWrapperId, refId); + taosMemoryFree(ppCf); return code; } int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); } @@ -3594,6 +3595,7 @@ void dbChkpDestroy(SDbChkp* pChkp) { taosMemoryFree(pChkp->pCurrent); taosMemoryFree(pChkp->pManifest); + taosMemoryFree(pChkp); } int32_t dbChkpInit(SDbChkp* p) { From 77261068e0a41c204107685030eb6279e1765122 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 11:06:55 +0800 Subject: [PATCH 119/151] fix stream backend convert --- include/libs/stream/tstream.h | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 337b760cf4..4979a576a7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -59,7 +59,7 @@ typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; typedef struct SStreamTaskSM SStreamTaskSM; -#define SSTREAM_TASK_VER 3 +#define SSTREAM_TASK_VER 2 #define SSTREAM_TASK_INCOMPATIBLE_VER 1 #define SSTREAM_TASK_NEED_CONVERT_VER 2 diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4e71eff431..f9bd5476e7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1759,7 +1759,7 @@ int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { rocksdb_column_family_handle_t* cf = rocksdb_create_column_family(pDb->db, pDb->pCfOpts[idx], ginitDict[idx].key, &err); - if (err != NULL) { + if (idx != 0 && err != NULL) { stError("failed to open cf, key:%s, reason: %s", key, err); taosMemoryFree(err); code = -1; From cee33ffb95a860a5e21fabe3c922b72ced545ca2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 11:22:18 +0800 Subject: [PATCH 120/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f9bd5476e7..c3196708ee 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1755,11 +1755,14 @@ int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { int32_t code = 0; char* err = NULL; int8_t idx = getCfIdx(key); + if (idx == -1) return -1; + if (pDb->pCf[idx] != NULL) return code; + rocksdb_column_family_handle_t* cf = rocksdb_create_column_family(pDb->db, pDb->pCfOpts[idx], ginitDict[idx].key, &err); - if (idx != 0 && err != NULL) { + if (err != NULL) { stError("failed to open cf, key:%s, reason: %s", key, err); taosMemoryFree(err); code = -1; From f5ef2e727fd0a0aff10d957bb3b3d0d91321649e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 14:41:04 +0800 Subject: [PATCH 121/151] fix stream backend convert --- source/libs/stream/inc/streamBackendRocksdb.h | 1 + source/libs/stream/src/streamBackendRocksdb.c | 10 ++++++++++ source/libs/stream/src/streamMeta.c | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 47404f311f..e6554e2fdf 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -128,6 +128,7 @@ typedef struct { TdThreadRwlock rwLock; } SBkdMgt; +bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId); void* streamBackendInit(const char* path, int64_t chkpId, int32_t vgId); void streamBackendCleanup(void* arg); void streamBackendHandleCleanup(void* arg); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c3196708ee..d0e343ae78 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -308,6 +308,16 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** return 0; } +bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) { + bool exist = true; + char* state = taosMemoryCalloc(1, strlen(path) + 32); + sprintf(state, "%s%s%s", path, TD_DIRSEP, "state"); + if (!taosDirExist(state)) { + exist = false; + } + taosMemoryFree(state); + return exist; +} void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { char* backendPath = NULL; int32_t code = rebuildDirFromCheckpoint(streamPath, chkpId, &backendPath); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 6b1ef6a7a0..afc7512cf8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -188,8 +188,13 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { } int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { - int32_t code = 0; - int64_t chkpId = streamMetaGetLatestCheckpointId(pMeta); + int32_t code = 0; + int64_t chkpId = streamMetaGetLatestCheckpointId(pMeta); + + bool exist = streamBackendDataIsExist(pMeta->path, chkpId, pMeta->vgId); + if (exist == false) { + return code; + } SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId, pMeta->vgId); void* pIter = taosHashIterate(pBackend->cfInst, NULL); @@ -206,6 +211,14 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { _EXIT: streamBackendCleanup((void*)pBackend); + + if (code == 0) { + char* state = taosMemoryCalloc(1, strlen(pMeta->path) + 32); + sprintf(state, "%s%s%s", pMeta->path, TD_DIRSEP, "state"); + taosRemoveDir(state); + taosMemoryFree(state); + } + return code; } int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { From 8023f2bebdd20fd4ebdff7b030dcf5cddfc55240 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 15:30:01 +0800 Subject: [PATCH 122/151] fix stream backend convert --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index f394cb980d..b7e659057b 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1245,7 +1245,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - code = mndProcessStreamCheckpointTrans(pMnode, pStream, pStream->checkpointId + 1); + code = mndProcessStreamCheckpointTrans(pMnode, pStream, pMsg->checkpointId); sdbRelease(pSdb, pStream); if (code == -1) { break; From 7905f22aae7c787aa9538d657eea32c485e27022 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 17:35:17 +0800 Subject: [PATCH 123/151] fix stream backend convert --- source/dnode/mnode/impl/src/mndStream.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b7e659057b..89e9c9f075 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -202,7 +202,8 @@ STREAM_ENCODE_OVER: return NULL; } - mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream); + mTrace("stream:%s, encode to raw:%p, row:%p, checkpoint:%" PRId64 "", pStream->name, pRaw, pStream, + pStream->checkpointId); return pRaw; } @@ -255,7 +256,8 @@ STREAM_DECODE_OVER: return NULL; } - mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream); + mTrace("stream:%s, decode from raw:%p, row:%p, checkpoint:%" PRId64 "", pStream->name, pRaw, pStream, + pStream->checkpointId); return pRow; } @@ -908,8 +910,11 @@ int64_t mndStreamGenChkpId(SMnode *pMnode) { if (pIter == NULL) break; maxChkpId = TMAX(maxChkpId, pStream->checkpointId); + mError("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId); sdbRelease(pSdb, pStream); } + + mError("generated checkpoint %" PRId64 "", maxChkpId + 1); return maxChkpId + 1; } From 52d7a26c102ca699f5d9b8b6bd3c4e8f9e171ab3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 16 Nov 2023 21:22:47 +0800 Subject: [PATCH 124/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d0e343ae78..72f8167f49 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -230,6 +230,94 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { return 0; } +int32_t readMetaData(char* path, SArray* list) { + char buf[128] = {0}; + char* metaPath = taosMemoryCalloc(1, strlen(path)); + sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META"); + + TdFilePtr pFile = taosOpenFile(path, TD_FILE_READ); + + if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { + taosMemoryFree(metaPath); + taosCloseFile(&pFile); + return -1; + } + int32_t len = strlen(buf); + for (int i = 0; i < len; i++) { + if (buf[i] == '\n') { + char* item = taosMemoryCalloc(1, i + 1); + memcpy(item, buf, i); + taosArrayPush(list, &item); + + item = taosMemoryCalloc(1, len - i); + memcpy(item, buf + i + 1, len - i - 1); + taosArrayPush(list, &item); + } + } + + taosCloseFile(&pFile); + taosMemoryFree(metaPath); + return 0; +} +int32_t validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { + int32_t complete = 1; + int32_t len = strlen(path) + 32; + char* src = taosMemoryCalloc(1, len); + char* dst = taosMemoryCalloc(1, len); + + for (int i = 0; i < taosArrayGetSize(list); i++) { + char* p = taosArrayGetP(list, i); + sprintf(src, "%s%s%s", path, TD_DIRSEP, p); + if (taosStatFile(src, NULL, NULL, NULL) != 0) { + complete = 0; + break; + } + + char temp[64] = {0}; + for (int j = 0; j < strlen(p); j++) { + if (p[j] == '_') { + memcpy(temp, p, j); + } + if (taosStr2int64(p + j + 1) != chkpId) { + complete = 0; + break; + } + } + + sprintf(dst, "%s%s%s", path, TD_DIRSEP, temp); + taosRenameFile(src, dst); + + memset(src, 0, len); + memset(dst, 0, len); + } + + taosMemoryFree(src); + taosMemoryFree(dst); + return complete == 1 ? 0 : -1; +} + +int32_t rebuildFromRemote(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = downloadCheckpoint(key, chkpPath); + if (code != 0) { + return -1; + } + + SArray* list = taosArrayInit(2, sizeof(void*)); + code = readMetaData(chkpPath, list); + if (code == 0) { + code = validAndCvtMeta(chkpPath, list, chkpId); + } + taosArrayDestroyP(list, taosMemoryFree); + + if (code == 0) { + if (taosIsDir(defaultPath)) { + taosRemoveDir(defaultPath); + } + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + } + return code; +} int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) { // impl later int32_t code = 0; @@ -271,11 +359,14 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); } + } else if (!taosIsDir(chkpPath)) { + code = rebuildFromRemote(key, chkpPath, chkpId, defaultPath); } else { stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); taosMkDir(defaultPath); } + taosMemoryFree(chkpPath); } else { char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); @@ -309,7 +400,7 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** } bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) { - bool exist = true; + bool exist = true; char* state = taosMemoryCalloc(1, strlen(path) + 32); sprintf(state, "%s%s%s", path, TD_DIRSEP, "state"); if (!taosDirExist(state)) { From b6994413d0f55d7e5580b54a5d7c183150645d2d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 10:13:39 +0800 Subject: [PATCH 125/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 91 +++++++++---------- 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 72f8167f49..c80b8163a4 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -296,10 +296,10 @@ int32_t validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { return complete == 1 ? 0 : -1; } -int32_t rebuildFromRemote(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { +int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { int32_t code = downloadCheckpoint(key, chkpPath); if (code != 0) { - return -1; + return code; } SArray* list = taosArrayInit(2, sizeof(void*)); @@ -318,16 +318,35 @@ int32_t rebuildFromRemote(char* key, char* chkpPath, int64_t chkpId, char* defau } return code; } + +int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = 0; + if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { + if (taosIsDir(defaultPath)) { + taosRemoveDir(defaultPath); + } + taosMkDir(defaultPath); + code = copyFiles(chkpPath, defaultPath); + if (code != 0) { + stError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); + } + + return code; + } + return -1; +} + +int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + int32_t code = 0; + return code; +} + int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) { // impl later int32_t code = 0; - /*param@1: checkpointId dir - param@2: state - copy pChkpIdDir's file to state dir - opt to set hard link to previous file - */ - char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128); sprintf(prefixPath, "%s%s%s", path, TD_DIRSEP, key); @@ -342,61 +361,33 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** taosMulMkDir(defaultPath); } + char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); if (chkpId != 0) { - char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); - if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { - if (taosIsDir(defaultPath)) { - // remove dir if exists - // taosRenameFile(const char *oldName, const char *newName) - taosRemoveDir(defaultPath); - } - taosMkDir(defaultPath); - code = copyFiles(chkpPath, defaultPath); - if (code != 0) { - stError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); - } - - } else if (!taosIsDir(chkpPath)) { - code = rebuildFromRemote(key, chkpPath, chkpId, defaultPath); - } else { - stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, - tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); - taosMkDir(defaultPath); + code = rebuildFromLocalChkp(key, chkpPath, chkpId, defaultPath); + if (code != 0) { + code = rebuildFromRemoteChkp(key, chkpPath, chkpId, defaultPath); } - taosMemoryFree(chkpPath); + if (code != 0) { + stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, + tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); + code = taosMkDir(defaultPath); + } } else { - char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", (int64_t)-1); - stInfo("no chkp id specified, try to restart from received chkp id -1, dir: %s", chkpPath); - if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { - if (taosIsDir(defaultPath)) { - taosRemoveDir(defaultPath); - } - taosMkDir(defaultPath); - code = copyFiles(chkpPath, defaultPath); - stInfo("copy snap file from %s to %s", chkpPath, defaultPath); - if (code != 0) { - stError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); - } - } else { - stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, - tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); - taosMkDir(defaultPath); + + code = rebuildFromLocalChkp(key, chkpPath, -1, defaultPath); + if (code != 0) { + code = taosMkDir(defaultPath); } - taosMemoryFree(chkpPath); } *dbPath = defaultPath; *dbPrefixPath = prefixPath; - return 0; + return code; } bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) { From e455c4799d98406f2b0120c7703bb443d262ed1d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 10:33:43 +0800 Subject: [PATCH 126/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c80b8163a4..22f5b93416 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -230,13 +230,13 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { return 0; } -int32_t readMetaData(char* path, SArray* list) { - char buf[128] = {0}; +int32_t remoteChkp_readMetaData(char* path, SArray* list) { char* metaPath = taosMemoryCalloc(1, strlen(path)); sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META"); TdFilePtr pFile = taosOpenFile(path, TD_FILE_READ); + char buf[128] = {0}; if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { taosMemoryFree(metaPath); taosCloseFile(&pFile); @@ -259,7 +259,7 @@ int32_t readMetaData(char* path, SArray* list) { taosMemoryFree(metaPath); return 0; } -int32_t validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { +int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { int32_t complete = 1; int32_t len = strlen(path) + 32; char* src = taosMemoryCalloc(1, len); @@ -283,6 +283,7 @@ int32_t validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { break; } } + if (complete == 0) break; sprintf(dst, "%s%s%s", path, TD_DIRSEP, temp); taosRenameFile(src, dst); @@ -293,6 +294,7 @@ int32_t validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { taosMemoryFree(src); taosMemoryFree(dst); + return complete == 1 ? 0 : -1; } @@ -303,9 +305,9 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d } SArray* list = taosArrayInit(2, sizeof(void*)); - code = readMetaData(chkpPath, list); + code = remoteChkp_readMetaData(chkpPath, list); if (code == 0) { - code = validAndCvtMeta(chkpPath, list, chkpId); + code = remoteChkp_validAndCvtMeta(chkpPath, list, chkpId); } taosArrayDestroyP(list, taosMemoryFree); From c7c1bdef6e1b86fcb8d2333abce86cfb81b83655 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 11:44:21 +0800 Subject: [PATCH 127/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 22f5b93416..60738b96bf 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -259,38 +259,53 @@ int32_t remoteChkp_readMetaData(char* path, SArray* list) { taosMemoryFree(metaPath); return 0; } +int32_t remoteChkp_validMetaFile(char* name, char* prename, int64_t chkpId) { + int8_t valid = 0; + for (int i = 0; i < strlen(name); i++) { + if (name[i] == '_') { + memcpy(prename, name, i); + if (taosStr2int64(name + i + 1) != chkpId) { + break; + } else { + valid = 1; + } + } + } + return valid; +} int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { int32_t complete = 1; int32_t len = strlen(path) + 32; char* src = taosMemoryCalloc(1, len); char* dst = taosMemoryCalloc(1, len); + int8_t count = 0; for (int i = 0; i < taosArrayGetSize(list); i++) { char* p = taosArrayGetP(list, i); sprintf(src, "%s%s%s", path, TD_DIRSEP, p); + + // check file exist if (taosStatFile(src, NULL, NULL, NULL) != 0) { complete = 0; break; } + // check file name char temp[64] = {0}; - for (int j = 0; j < strlen(p); j++) { - if (p[j] == '_') { - memcpy(temp, p, j); - } - if (taosStr2int64(p + j + 1) != chkpId) { - complete = 0; - break; - } + if (remoteChkp_validMetaFile(p, temp, chkpId)) { + count++; } - if (complete == 0) break; + // rename file sprintf(dst, "%s%s%s", path, TD_DIRSEP, temp); taosRenameFile(src, dst); memset(src, 0, len); memset(dst, 0, len); } + if (count != taosArrayGetSize(list)) { + complete = 0; + } taosMemoryFree(src); taosMemoryFree(dst); From 1dbdb32dd7cb048a4ae399dc4e50d10beecf9f91 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 11:56:22 +0800 Subject: [PATCH 128/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 60738b96bf..a01fabd3a1 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -319,6 +319,11 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d return code; } + int32_t len = strlen(defaultPath) + 32; + char* tmp = taosMemoryCalloc(1, len); + sprintf(tmp, "%s%s%s", defaultPath, TD_DIRSEP, "_tmp"); + if (taosIsDir(tmp)) taosRemoveDir(tmp); + SArray* list = taosArrayInit(2, sizeof(void*)); code = remoteChkp_readMetaData(chkpPath, list); if (code == 0) { @@ -328,19 +333,31 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d if (code == 0) { if (taosIsDir(defaultPath)) { - taosRemoveDir(defaultPath); + taosRenameFile(defaultPath, tmp); } taosMkDir(defaultPath); code = copyFiles(chkpPath, defaultPath); } + + if (code != 0) { + if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); + if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } return code; } int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { - int32_t code = 0; + int32_t code = -1; + int32_t len = strlen(defaultPath) + 32; + char* tmp = taosMemoryCalloc(1, len); + sprintf(tmp, "%s%s%s", defaultPath, TD_DIRSEP, "_tmp"); + if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { + if (taosIsDir(tmp)) { + taosRemoveDir(tmp); + } if (taosIsDir(defaultPath)) { - taosRemoveDir(defaultPath); + taosRenameFile(defaultPath, tmp); } taosMkDir(defaultPath); code = copyFiles(chkpPath, defaultPath); @@ -349,10 +366,14 @@ int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* de } else { stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); } - - return code; } - return -1; + if (code != 0) { + if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); + if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } + + taosMemoryFree(tmp); + return code; } int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { From ccc74ef558cae8e91b27bf0bbee0bc6ad4557c4b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 15:39:03 +0800 Subject: [PATCH 129/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a01fabd3a1..c91bb2a393 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -321,8 +321,9 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d int32_t len = strlen(defaultPath) + 32; char* tmp = taosMemoryCalloc(1, len); - sprintf(tmp, "%s%s%s", defaultPath, TD_DIRSEP, "_tmp"); + sprintf(tmp, "%s%s", defaultPath, "_tmp"); if (taosIsDir(tmp)) taosRemoveDir(tmp); + if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp); SArray* list = taosArrayInit(2, sizeof(void*)); code = remoteChkp_readMetaData(chkpPath, list); @@ -332,9 +333,6 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d taosArrayDestroyP(list, taosMemoryFree); if (code == 0) { - if (taosIsDir(defaultPath)) { - taosRenameFile(defaultPath, tmp); - } taosMkDir(defaultPath); code = copyFiles(chkpPath, defaultPath); } @@ -342,7 +340,11 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d if (code != 0) { if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } else { + taosRemoveDir(tmp); } + + taosMemoryFree(tmp); return code; } @@ -350,15 +352,15 @@ int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* de int32_t code = -1; int32_t len = strlen(defaultPath) + 32; char* tmp = taosMemoryCalloc(1, len); - sprintf(tmp, "%s%s%s", defaultPath, TD_DIRSEP, "_tmp"); + sprintf(tmp, "%s%s", defaultPath, "_tmp"); + + if (taosIsDir(tmp)) taosRemoveDir(tmp); + if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp); if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { if (taosIsDir(tmp)) { taosRemoveDir(tmp); } - if (taosIsDir(defaultPath)) { - taosRenameFile(defaultPath, tmp); - } taosMkDir(defaultPath); code = copyFiles(chkpPath, defaultPath); if (code != 0) { @@ -370,6 +372,8 @@ int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* de if (code != 0) { if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath); if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath); + } else { + taosRemoveDir(tmp); } taosMemoryFree(tmp); From 286413abbb5c1bf84d127114cb3b0aebf034a243 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 16:14:51 +0800 Subject: [PATCH 130/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 1 + source/libs/stream/src/streamMeta.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c91bb2a393..d2aab7459b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -425,6 +425,7 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** code = taosMkDir(defaultPath); } } + taosMemoryFree(chkpPath); *dbPath = defaultPath; *dbPrefixPath = prefixPath; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index afc7512cf8..138759c179 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -243,7 +243,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { SStreamTask* pTask = arg; - int64_t chkpId = pTask->chkInfo.checkpointingId; + int64_t chkpId = pTask->chkInfo.checkpointId; taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); From e7830bfdfdce50134a6b6e6e82c37351775d1e2f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Nov 2023 16:50:45 +0800 Subject: [PATCH 131/151] fix stream backend convert --- source/libs/stream/src/streamBackendRocksdb.c | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d2aab7459b..4c86beb44d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -39,8 +39,9 @@ typedef struct { int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); -void destroyRocksdbCfInst(RocksdbCfInst* inst); -int32_t getCfIdx(const char* cfName); +void destroyRocksdbCfInst(RocksdbCfInst* inst); +int32_t getCfIdx(const char* cfName); +STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath); void destroyCompactFilteFactory(void* arg); void destroyCompactFilte(void* arg); @@ -188,7 +189,14 @@ int32_t getCfIdx(const char* cfName) { return idx; } -bool isValidCheckpoint(const char* dir) { return true; } +bool isValidCheckpoint(const char* dir) { + STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); + if (pDb == NULL) { + return true; + } + taskDbDestroy(pDb); + return true; +} int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { // impl later @@ -1707,25 +1715,19 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { p->chkpId = chkpId; taosThreadMutexUnlock(&p->mutex); } -STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { - char* statePath = NULL; + +STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { char* err = NULL; - char* dbPath = NULL; char** cfNames = NULL; size_t nCf = 0; - if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { - return NULL; - } - STaskDbWrapper* pTaskDb = taosMemoryCalloc(1, sizeof(STaskDbWrapper)); - pTaskDb->idstr = taosStrdup(key); - pTaskDb->path = statePath; + pTaskDb->idstr = key ? taosStrdup(key) : NULL; + pTaskDb->path = statePath ? taosStrdup(statePath) : NULL; taosThreadMutexInit(&pTaskDb->mutex, NULL); taskDbInitChkpOpt(pTaskDb); taskDbInitOpt(pTaskDb); - statePath = NULL; cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { @@ -1752,19 +1754,27 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { } qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); - taosMemoryFree(dbPath); - - return pTaskDb; _EXIT: - taskDbDestroy(pTaskDb); + taskDbDestroy(pTaskDb); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); - if (dbPath) taosMemoryFree(dbPath); - if (statePath) taosMemoryFree(statePath); return NULL; } +STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { + char* statePath = NULL; + char* dbPath = NULL; + + if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { + return NULL; + } + // taosMemoryFree(statePath); + + STaskDbWrapper* pTaskDb = taskDbOpenImpl(key, statePath, dbPath); + taosMemoryFree(dbPath); + return pTaskDb; +} void taskDbDestroy(void* pDb) { STaskDbWrapper* wrapper = pDb; From 734bf0731713c11acddfffe16b3876f9e06173bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 20 Nov 2023 10:48:49 +0800 Subject: [PATCH 132/151] Merge branch '3.0' into enh/refactorBackend --- source/dnode/mnode/impl/src/mndStream.c | 1 - source/libs/stream/src/streamBackendRocksdb.c | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 99fda7e81c..6a096b900f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1250,7 +1250,6 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { if (taosArrayGetSize(execInfo.pNodeList) == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = ts; - // goto _EXIT; } for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { STaskId * p = taosArrayGet(execInfo.pTaskList, i); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b9041198ac..a1f632d8c2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1740,9 +1740,10 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { rocksdb_list_column_families_destroy(cfNames, nCf); } taosMemoryFree(err); + err = NULL; cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); - ASSERT(err != NULL); + ASSERT(err == NULL); } if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { @@ -1751,10 +1752,11 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { if (cfNames != NULL) { rocksdb_list_column_families_destroy(cfNames, nCf); + cfNames = NULL; } qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); - + return pTaskDb; _EXIT: taskDbDestroy(pTaskDb); @@ -1769,10 +1771,10 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { return NULL; } - // taosMemoryFree(statePath); STaskDbWrapper* pTaskDb = taskDbOpenImpl(key, statePath, dbPath); taosMemoryFree(dbPath); + taosMemoryFree(statePath); return pTaskDb; } From 0d724244dc6284d2d35f93ee49bc2001a7eb3451 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 20 Nov 2023 16:55:14 +0800 Subject: [PATCH 133/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 48 ++++++++++++++----- source/libs/stream/src/streamCheckpoint.c | 24 +++++----- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a1f632d8c2..b643e7186c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -321,7 +321,25 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { return complete == 1 ? 0 : -1; } -int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { +int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + // impl later + int32_t code = 0; + if (taosIsDir(chkpPath)) { + taosRemoveDir(chkpPath); + } + if (taosIsDir(defaultPath)) { + taosRemoveDir(defaultPath); + } + + code = downloadCheckpoint(key, chkpPath); + if (code != 0) { + return code; + } + code = copyFiles(chkpPath, defaultPath); + + return code; +} +int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { int32_t code = downloadCheckpoint(key, chkpPath); if (code != 0) { return code; @@ -355,6 +373,15 @@ int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* d taosMemoryFree(tmp); return code; } +int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + UPLOAD_TYPE type = getUploadType(); + if (type == UPLOAD_S3) { + return rebuildFromRemoteChkp_s3(key, chkpPath, chkpId, defaultPath); + } else if (type == UPLOAD_RSYNC) { + return rebuildFromRemoteChkp_rsync(key, chkpPath, chkpId, defaultPath); + } + return -1; +} int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { int32_t code = -1; @@ -944,7 +971,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI sprintf(pChkpIdDir, "%s%s%s%" PRId64, pChkpDir, TD_DIRSEP, "checkpoint", chkpId); if (taosIsDir(pChkpIdDir)) { stInfo("stream rm exist checkpoint%s", pChkpIdDir); - taosRemoveFile(pChkpIdDir); + taosRemoveDir(pChkpIdDir); } *chkpDir = pChkpDir; *chkpIdDir = pChkpIdDir; @@ -1848,22 +1875,17 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char if (taosAcquireRef(taskDbWrapperId, refId) == NULL) { return -1; } - char* pChkpDir = NULL; - char* pChkpIdDir = NULL; - if (chkpPreBuildDir(pDb->path, chkpId, &pChkpDir, &pChkpIdDir) != 0) { - code = -1; - } - if (taosIsDir(pChkpIdDir) && isValidCheckpoint(pChkpIdDir)) { + char* buf = taosMemoryCalloc(1, strlen(pDb->path) + 128); + sprintf(buf, "%s%s%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); + if (taosIsDir(buf)) { code = 0; - *path = pChkpIdDir; - pChkpIdDir = NULL; + *path = buf; + } else { + taosMemoryFree(buf); } - taosMemoryFree(pChkpDir); - taosMemoryFree(pChkpIdDir); taosReleaseRef(taskDbWrapperId, refId); - return code; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 74c391f386..a96fbca50b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -375,17 +375,16 @@ int32_t doUploadChkp(void* param) { SAsyncUploadArg* arg = param; char* path = NULL; int32_t code = 0; - SArray* list = taosArrayInit(4, sizeof(void*)); + SArray* toDelFiles = taosArrayInit(4, sizeof(void*)); - if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId, - (int8_t)(arg->type), &path, list)) != 0) { + (int8_t)(arg->type), &path, toDelFiles)) != 0) { stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId); } - - code = getChkpMeta(arg->taskId, path, list); - if (code != 0) { - code = 0; + if (arg->type == UPLOAD_S3) { + if (code == 0 && (code = getChkpMeta(arg->taskId, path, toDelFiles)) != 0) { + stError("s-task:%s failed to get checkpoint:%" PRId64 " meta", arg->pTask->id.idStr, arg->chkpId); + } } if (code == 0 && (code = uploadCheckpoint(arg->taskId, path)) != 0) { @@ -393,23 +392,24 @@ int32_t doUploadChkp(void* param) { } if (code == 0) { - for (int i = 0; i < taosArrayGetSize(list); i++) { - char* p = taosArrayGetP(list, i); + for (int i = 0; i < taosArrayGetSize(toDelFiles); i++) { + char* p = taosArrayGetP(toDelFiles, i); code = deleteCheckpointFile(arg->taskId, p); - stDebug("try to del file: %s", p); + stDebug("s-task:%s try to del file: %s", arg->pTask->id.idStr, p); if (code != 0) { break; } } } - taosArrayDestroyP(list, taosMemoryFree); + taosArrayDestroyP(toDelFiles, taosMemoryFree); taosRemoveDir(path); taosMemoryFree(path); + taosMemoryFree(arg->taskId); taosMemoryFree(arg); - return 0; + return code; } int32_t streamTaskUploadChkp(SStreamTask* pTask, int64_t chkpId, char* taskId) { // async upload From bb71bd43869dda25802f60fd0fdd6d18766ed7d4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 20 Nov 2023 21:03:09 +0800 Subject: [PATCH 134/151] fix mem leak --- source/libs/executor/src/streamtimewindowoperator.c | 1 + source/libs/transport/src/transSvr.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 0af7413c75..6e1cfd40f0 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3482,6 +3482,7 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); + taosMemoryFree(pBuf); } static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index bf73c253bc..017969b4e5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -1499,6 +1499,7 @@ int transSendResponse(const STransMsg* msg) { } SExHandle* exh = msg->info.handle; if (exh == NULL) { + rpcFreeCont(msg->pCont); return 0; } int64_t refId = msg->info.refId; From edaa33d28dbf30e230a58604d76c5ff08b14ac64 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 21 Nov 2023 10:18:38 +0800 Subject: [PATCH 135/151] fix mem leak --- source/libs/executor/src/streamtimewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 6e1cfd40f0..893848f010 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3482,7 +3482,7 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); - taosMemoryFree(pBuf); + taosMemoryFree(buf); } static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) { From 947775155b54e5f64903979d4df27b4509be0769 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 21 Nov 2023 20:51:11 +0800 Subject: [PATCH 136/151] fix transfer crash --- source/dnode/mnode/impl/src/mndStream.c | 6 ++++-- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 1 - source/libs/stream/src/streamCheckpoint.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index a60b916401..a00ede7270 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -284,6 +284,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream pOldStream->status = pNewStream->status; pOldStream->updateTime = pNewStream->updateTime; + pOldStream->checkpointId = pNewStream->checkpointId; + pOldStream->checkpointFreq = pNewStream->checkpointFreq; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -1518,7 +1520,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char dstStr[20] = {0}; STR_TO_VARSTR(dstStr, sinkQuota) pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*) dstStr, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); char scanHistoryIdle[20 + VARSTR_HEADER_SIZE] = {0}; strcpy(scanHistoryIdle, "100a"); @@ -1526,7 +1528,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB memset(dstStr, 0, tListLen(dstStr)); STR_TO_VARSTR(dstStr, scanHistoryIdle) pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*) dstStr, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dstStr, false); numOfRows++; sdbRelease(pSdb, pStream); diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 791f5ecfed..2ab710176d 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -106,7 +106,6 @@ int32_t streamStateSnapRead(SStreamStateReader* pReader, uint8_t** ppData) { memcpy(pHdr->data, rowData, len); taosMemoryFree(rowData); tqDebug("vgId:%d, vnode stream-state snapshot read data success", TD_VID(pReader->pTq->pVnode)); - taosMemoryFree(rowData); return code; _err: diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 37630c25ea..7e66885620 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -288,7 +288,7 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { taosThreadMutexLock(&p->lock); - ASSERT(p->chkInfo.checkpointId < p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId && + ASSERT(p->chkInfo.checkpointId <= p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId && p->chkInfo.checkpointVer <= p->chkInfo.processedVer); p->chkInfo.checkpointId = p->chkInfo.checkpointingId; p->chkInfo.checkpointVer = p->chkInfo.processedVer; From a2a68ce256e420a99098c054b2f4e2c1362f85b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Nov 2023 18:28:38 +0800 Subject: [PATCH 137/151] fix(stream): fix memory leak. --- source/dnode/mnode/impl/src/mndStream.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 8ccda66212..e8a097c44e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -168,6 +168,8 @@ void mndCleanupStream(SMnode *pMnode) { taosArrayDestroy(execInfo.pTaskList); taosHashCleanup(execInfo.pTaskMap); taosThreadMutexDestroy(&execInfo.lock); + taosHashCleanup(execInfo.transMgmt.pDBTrans); + taosHashCleanup(execInfo.transMgmt.pWaitingList); mDebug("mnd stream exec info cleanup"); } From 50f996bbf7c17b6f52b7714c68e7cd9e93808cbb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Nov 2023 21:07:58 +0800 Subject: [PATCH 138/151] Merge branch '3.0' into enh/refactorBackend --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6ca47c8616..f07ef71003 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -190,6 +190,7 @@ int32_t getCfIdx(const char* cfName) { } bool isValidCheckpoint(const char* dir) { + return true; STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); if (pDb == NULL) { return true; From 23ae62d268a01993a476f0b8910a53a8a1c369bc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Nov 2023 11:10:39 +0800 Subject: [PATCH 139/151] Merge branch '3.0' into enh/refactorBackend --- source/dnode/mnode/impl/src/mndStream.c | 73 +++++++++---------- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c92dca099b..602b3035b9 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -27,9 +27,9 @@ #include "tmisce.h" #include "tname.h" -#define MND_STREAM_VER_NUMBER 4 -#define MND_STREAM_RESERVE_SIZE 64 -#define MND_STREAM_MAX_NUM 60 +#define MND_STREAM_VER_NUMBER 4 +#define MND_STREAM_RESERVE_SIZE 64 +#define MND_STREAM_MAX_NUM 60 #define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" #define MND_STREAM_PAUSE_NAME "stream-pause" @@ -77,7 +77,7 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); -static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char* pMsg); +static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg); static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans); static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t retryCode); @@ -85,9 +85,9 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); -static int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDbName, size_t len); +static int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); -static void freeCheckpointCandEntry(void*); +static void freeCheckpointCandEntry(void *); static SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); @@ -168,9 +168,8 @@ void mndCleanupStream(SMnode *pMnode) { taosArrayDestroy(execInfo.pTaskList); taosHashCleanup(execInfo.pTaskMap); taosHashCleanup(execInfo.transMgmt.pDBTrans); - taosThreadMutexDestroy(&execInfo.lock); - taosHashCleanup(execInfo.transMgmt.pDBTrans); taosHashCleanup(execInfo.transMgmt.pWaitingList); + taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); } @@ -333,11 +332,11 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { } } -SSdbRaw * mndStreamSeqActionEncode(SStreamObj *pStream) { return NULL; } -SSdbRow * mndStreamSeqActionDecode(SSdbRaw *pRaw) { return NULL; } -int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream) { return 0; } -int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream) { return 0; } -int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream) { return 0; } +SSdbRaw *mndStreamSeqActionEncode(SStreamObj *pStream) { return NULL; } +SSdbRow *mndStreamSeqActionDecode(SSdbRaw *pRaw) { return NULL; } +int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream) { return 0; } +int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream) { return 0; } static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { int8_t trigger = pStream->conf.trigger; @@ -761,7 +760,7 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { // check for number of existed tasks int32_t numOfStream = 0; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while ((pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { if (pStream->sourceDbUid == pStreamObj->sourceDbUid) { @@ -1087,7 +1086,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre } STransAction act = {0}; - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); mndReleaseVgroup(pMnode, pVgObj); initTransAction(&act, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); @@ -1265,7 +1264,7 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { } for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { - STaskId *p = taosArrayGet(execInfo.pTaskList, i); + STaskId * p = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); if (pEntry == NULL) { continue; @@ -1280,13 +1279,13 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { } taosThreadMutexUnlock(&execInfo.lock); - return ready? 0:-1; + return ready ? 0 : -1; } static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; + void * pIter = NULL; SStreamObj *pStream = NULL; int32_t code = 0; @@ -1308,7 +1307,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - void *pIter = NULL; + void * pIter = NULL; int32_t code = 0; taosThreadMutexLock(&execInfo.lock); @@ -1327,7 +1326,7 @@ static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { SCheckpointCandEntry *pEntry = pIter; SStreamObj *ps = mndAcquireStream(pMnode, pEntry->pName); - mDebug("start to launch checkpoint for stream:%s %"PRIx64" in candidate list", pEntry->pName, pEntry->streamId); + mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId); code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId); mndReleaseStream(pMnode, ps); @@ -2497,13 +2496,13 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { } // kill all trans in the dst DB -static void killAllCheckpointTrans(SMnode* pMnode, SVgroupChangeInfo* pChangeInfo) { - void* pIter = NULL; - while((pIter = taosHashIterate(pChangeInfo->pDBMap, pIter)) != NULL) { - char* pDb = (char*) pIter; +static void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { + void *pIter = NULL; + while ((pIter = taosHashIterate(pChangeInfo->pDBMap, pIter)) != NULL) { + char *pDb = (char *)pIter; size_t len = 0; - void* pKey = taosHashGetKey(pDb, &len); + void * pKey = taosHashGetKey(pDb, &len); killActiveCheckpointTrans(pMnode, pKey, len); } } @@ -2652,7 +2651,7 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); } -STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char* pMsg) { +STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, name); if (pTrans == NULL) { mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); @@ -2660,7 +2659,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const return NULL; } - mDebug("s-task:0x%"PRIx64" start to build trans %s", pStream->uid, pMsg); + mDebug("s-task:0x%" PRIx64 " start to build trans %s", pStream->uid, pMsg); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); if (mndTransCheckConflict(pMnode, pTrans) != 0) { @@ -2740,9 +2739,9 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return TSDB_CODE_ACTION_IN_PROGRESS; } -int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t len) { +int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { // data in the hash table will be removed automatically, no need to remove it here. - SStreamTransInfo* pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); + SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); if (pTransInfo == NULL) { return TSDB_CODE_SUCCESS; } @@ -2753,7 +2752,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le return TSDB_CODE_SUCCESS; } - STrans* pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); + STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDBName); mndKillTrans(pMnode, pTrans); @@ -2764,7 +2763,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le } int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { - STrans* pTrans = mndAcquireTrans(pMnode, transId); + STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d to reset task status", transId); mndKillTrans(pMnode, pTrans); @@ -2783,8 +2782,8 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { bool conflict = mndStreamTransConflictOtherTrans(pMnode, pStream->sourceDb, pStream->targetDb); if (conflict) { - mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", - pStream->name, pStream->sourceDb, pStream->targetDb); + mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, + pStream->sourceDb, pStream->targetDb); continue; } @@ -2928,7 +2927,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } -void freeCheckpointCandEntry(void* param) { - SCheckpointCandEntry* pEntry = param; +void freeCheckpointCandEntry(void *param) { + SCheckpointCandEntry *pEntry = param; taosMemoryFreeClear(pEntry->pName); } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f07ef71003..dc61bd25f2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -190,7 +190,7 @@ int32_t getCfIdx(const char* cfName) { } bool isValidCheckpoint(const char* dir) { - return true; + // return true; STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); if (pDb == NULL) { return true; From 6416a6153a32043699a6970f7159a57e632dff1e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Nov 2023 16:29:09 +0800 Subject: [PATCH 140/151] Merge branch '3.0' into enh/refactorBackend --- source/dnode/mnode/impl/src/mndStream.c | 4 +- source/libs/stream/inc/streamBackendRocksdb.h | 9 ++-- source/libs/stream/src/streamBackendRocksdb.c | 53 +++++++++++-------- source/libs/stream/src/streamMeta.c | 6 +-- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 602b3035b9..37c3683edb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -942,11 +942,11 @@ int64_t mndStreamGenChkpId(SMnode *pMnode) { if (pIter == NULL) break; maxChkpId = TMAX(maxChkpId, pStream->checkpointId); - mError("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId); + mDebug("stream %p checkpoint %" PRId64 "", pStream, pStream->checkpointId); sdbRelease(pSdb, pStream); } - mError("generated checkpoint %" PRId64 "", maxChkpId + 1); + mDebug("generated checkpoint %" PRId64 "", maxChkpId + 1); return maxChkpId + 1; } diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index a73c289440..bed0f79f02 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -139,7 +139,8 @@ void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateCvtDataFormat(char* path, char* key, void* cfInst); STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); -void taskDbDestroy(void* pBackend); +void taskDbDestroy(void* pBackend, bool flush); +void taskDbDestroy2(void* pBackend); int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId); @@ -217,7 +218,7 @@ int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** p int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key); int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); void* streamDefaultIterCreate_rocksdb(SStreamState* pState); -bool streamDefaultIterValid_rocksdb(void* iter); +bool streamDefaultIterValid_rocksdb(void* iter); void streamDefaultIterSeek_rocksdb(void* iter, const char* key); void streamDefaultIterNext_rocksdb(void* iter); char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len); @@ -245,8 +246,8 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); -STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); -void taskDbDestroy(void* pDb); +// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); +// void taskDbDestroy(void* pDb, bool flush); int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index dc61bd25f2..ccf166c390 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -190,12 +190,12 @@ int32_t getCfIdx(const char* cfName) { } bool isValidCheckpoint(const char* dir) { - // return true; + return true; STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); if (pDb == NULL) { - return true; + return false; } - taskDbDestroy(pDb); + taskDbDestroy(pDb, false); return true; } @@ -1788,7 +1788,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { return pTaskDb; _EXIT: - taskDbDestroy(pTaskDb); + taskDbDestroy(pTaskDb, false); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); return NULL; @@ -1807,7 +1807,7 @@ STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { return pTaskDb; } -void taskDbDestroy(void* pDb) { +void taskDbDestroy(void* pDb, bool flush) { STaskDbWrapper* wrapper = pDb; qDebug("succ to destroy stream backend:%p", wrapper); @@ -1815,24 +1815,33 @@ void taskDbDestroy(void* pDb) { if (wrapper == NULL) return; - if (wrapper->db && wrapper->pCf) { - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - rocksdb_flushoptions_set_wait(flushOpt, 1); + if (flush) { + if (wrapper->db && wrapper->pCf) { + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flushOpt, 1); - char* err = NULL; - for (int i = 0; i < nCf; i++) { - if (wrapper->pCf[i] != NULL) rocksdb_flush_cf(wrapper->db, flushOpt, wrapper->pCf[i], &err); - if (err != NULL) { - stError("failed to flush cf:%s, reason:%s", ginitDict[i].key, err); - taosMemoryFreeClear(err); + char* err = NULL; + rocksdb_column_family_handle_t** cfs = taosMemoryCalloc(1, sizeof(rocksdb_column_family_handle_t*) * nCf); + int numOfFlushCf = 0; + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + cfs[numOfFlushCf++] = wrapper->pCf[i]; + } } + if (numOfFlushCf != 0) { + rocksdb_flush_cfs(wrapper->db, flushOpt, cfs, numOfFlushCf, &err); + if (err != NULL) { + stError("failed to flush all cfs, reason:%s", err); + taosMemoryFreeClear(err); + } + } + taosMemoryFree(cfs); + rocksdb_flushoptions_destroy(flushOpt); } - rocksdb_flushoptions_destroy(flushOpt); - - for (int i = 0; i < nCf; i++) { - if (wrapper->pCf[i] != NULL) { - rocksdb_column_family_handle_destroy(wrapper->pCf[i]); - } + } + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pCf[i]); } } rocksdb_options_destroy(wrapper->dbOpt); @@ -1869,6 +1878,8 @@ void taskDbDestroy(void* pDb) { return; } +void taskDbDestroy2(void* pDb) { taskDbDestroy(pDb, true); } + int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char** path) { int64_t st = taosGetTimestampMs(); int32_t code = -1; @@ -2007,7 +2018,7 @@ int32_t streamStateCvtDataFormat(char* path, char* key, void* pCfInst) { } _EXIT: - taskDbDestroy(pTaskDb); + taskDbDestroy(pTaskDb, true); return code; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 78b4814aa6..0e137a673e 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -57,7 +57,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); static void streamMetaEnvInit() { streamBackendId = taosOpenRef(64, streamBackendCleanup); streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); - taskDbWrapperId = taosOpenRef(64, taskDbDestroy); + taskDbWrapperId = taosOpenRef(64, taskDbDestroy2); streamMetaId = taosOpenRef(64, streamMetaCloseImpl); @@ -1239,8 +1239,8 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { void streamMetaNotifyClose(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; - stDebug("vgId:%d notify all stream tasks that the vnode is closing. isLeader:%d startHb:%" PRId64 ", totalHb:%d", vgId, - (pMeta->role == NODE_ROLE_LEADER), pMeta->pHbInfo->hbStart, pMeta->pHbInfo->hbCount); + stDebug("vgId:%d notify all stream tasks that the vnode is closing. isLeader:%d startHb:%" PRId64 ", totalHb:%d", + vgId, (pMeta->role == NODE_ROLE_LEADER), pMeta->pHbInfo->hbStart, pMeta->pHbInfo->hbCount); streamMetaWLock(pMeta); From e483570cd7a40e6a15c6e0b94dcb02662869b708 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Nov 2023 11:10:56 +0800 Subject: [PATCH 141/151] Merge branch '3.0' into enh/refactorBackend --- source/dnode/mnode/impl/src/mndStream.c | 3 ++- source/dnode/mnode/impl/src/mndStreamTrans.c | 23 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 95ea6070ae..b2c3a07681 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1340,7 +1340,8 @@ static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { int64_t *pId = taosArrayGet(pList, i); - taosHashRemove(execInfo.transMgmt.pWaitingList, &pId, sizeof(*pId)); + + taosHashRemove(execInfo.transMgmt.pWaitingList, pId, sizeof(*pId)); } int32_t remain = taosHashGetSize(execInfo.transMgmt.pWaitingList); diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 3e2afe4ade..c1b1df85c0 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "mndTrans.h" #include "mndStream.h" +#include "mndTrans.h" typedef struct SKeyInfo { void* pKey; @@ -42,8 +42,8 @@ int32_t clearFinishedTrans(SMnode* pMnode) { void* pIter = NULL; while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { - SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter; - STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); + SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter; + STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); // let's clear the finished trans if (pTrans == NULL) { @@ -60,12 +60,12 @@ int32_t clearFinishedTrans(SMnode* pMnode) { } size_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { SKeyInfo* pKey = taosArrayGet(pList, i); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); } - mDebug("clear %d finished stream-trans, remained:%d", (int32_t) num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); + mDebug("clear %d finished stream-trans, remained:%d", (int32_t)num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); taosThreadMutexUnlock(&execInfo.lock); terrno = TSDB_CODE_SUCCESS; @@ -83,7 +83,7 @@ bool mndStreamTransConflictOtherTrans(SMnode* pMnode, const char* pSrcDb, const return false; } - SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb)); + SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, pSrcDb, strlen(pSrcDb)); if (pEntry != NULL) { taosThreadMutexUnlock(&execInfo.lock); mWarn("conflict with other transId:%d in Db:%s, trans:%s", pEntry->transId, pSrcDb, pEntry->name); @@ -112,14 +112,13 @@ int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId) taosHashPut(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid), &entry, sizeof(entry)); int32_t size = taosHashGetSize(execInfo.transMgmt.pWaitingList); - mDebug("stream:%" PRIx64 " add into waiting list due to conflict, ts:%" PRId64 ", total in waitingList:%d", - pStream->uid, entry.checkpointTs, size); + mDebug("stream:%" PRIx64 " add into waiting list due to conflict, ts:%" PRId64 " , checkpointId: %" PRId64 + ", total in waitingList:%d", + pStream->uid, entry.checkpointTs, checkpointId, size); } else { - mDebug("stream:%" PRIx64 " ts:%" PRId64 "already in waiting list, no need to add into", pStream->uid, - pEntry->checkpointTs); + mDebug("stream:%" PRIx64 " ts:%" PRId64 ", checkpointId:%" PRId64 " already in waiting list, no need to add into", + pStream->uid, pEntry->checkpointTs, checkpointId); } return TSDB_CODE_SUCCESS; } - - From 703f323b3f289f16f00dfb4b51e85932ec62d650 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Nov 2023 13:52:28 +0800 Subject: [PATCH 142/151] Merge branch '3.0' into enh/refactorBackend --- source/common/src/tglobal.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a9a8217c31..c72a998f88 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -248,7 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; -int32_t tsStreamCheckpointInterval = 10; +int32_t tsStreamCheckpointInterval = 40; float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 15; int32_t tsTtlUnit = 86400; @@ -354,16 +354,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input char cfgFile[PATH_MAX + 100] = {0}; taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); - char lastC = cfgDir[strlen(cfgDir) - 1]; + char lastC = cfgDir[strlen(cfgDir) - 1]; char *tdDirsep = TD_DIRSEP; if (lastC == '\\' || lastC == '/') { tdDirsep = ""; } if (taosIsDir(cfgDir)) { #ifdef CUS_PROMPT - snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "%s.cfg", cfgDir, tdDirsep, CUS_PROMPT); + snprintf(cfgFile, sizeof(cfgFile), + "%s" + "%s" + "%s.cfg", + cfgDir, tdDirsep, CUS_PROMPT); #else - snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "taos.cfg", cfgDir, tdDirsep); + snprintf(cfgFile, sizeof(cfgFile), + "%s" + "%s" + "taos.cfg", + cfgDir, tdDirsep); #endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); From a4d475b582465d2594a3a22bcc15c249c891afeb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Nov 2023 20:27:53 +0800 Subject: [PATCH 143/151] fix rsma --- include/libs/stream/tstream.h | 23 +++++++++++++---------- source/common/src/tglobal.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 5 ++++- source/libs/stream/inc/streamsm.h | 8 ++++---- source/libs/stream/src/streamCheckpoint.c | 4 ++++ 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2e94bbdacd..f91223b863 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -108,6 +108,7 @@ typedef enum { TASK_LEVEL__SOURCE = 1, TASK_LEVEL__AGG, TASK_LEVEL__SINK, + TASK_LEVEL_SMA, } ETASK_LEVEL; enum { @@ -666,19 +667,19 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea typedef struct STaskStatusEntry { STaskId id; int32_t status; - int32_t statusLastDuration; // to record the last duration of current status + int32_t statusLastDuration; // to record the last duration of current status int64_t stage; int32_t nodeId; - int64_t verStart; // start version in WAL, only valid for source task - int64_t verEnd; // end version in WAL, only valid for source task - int64_t processedVer; // only valid for source task - int32_t relatedHTask; // has related fill-history task - int64_t activeCheckpointId; // current active checkpoint id - bool checkpointFailed; // denote if the checkpoint is failed or not - double inputQUsed; // in MiB + int64_t verStart; // start version in WAL, only valid for source task + int64_t verEnd; // end version in WAL, only valid for source task + int64_t processedVer; // only valid for source task + int32_t relatedHTask; // has related fill-history task + int64_t activeCheckpointId; // current active checkpoint id + bool checkpointFailed; // denote if the checkpoint is failed or not + double inputQUsed; // in MiB double inputRate; - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dst data size + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size } STaskStatusEntry; typedef struct SStreamHbMsg { @@ -864,6 +865,8 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int8_t isSucceed); +SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); +void* streamDestroyStateMachine(SStreamTaskSM* pSM); #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c72a998f88..e8ff530d78 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -248,7 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; -int32_t tsStreamCheckpointInterval = 40; +int32_t tsStreamCheckpointInterval = 10; float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 15; int32_t tsTtlUnit = 86400; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index c7623eca78..abe4c3f2fc 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -297,6 +297,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); + pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); + pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; @@ -1282,10 +1284,11 @@ _checkpoint: if (pItem && pItem->pStreamTask) { SStreamTask *pTask = pItem->pStreamTask; // atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1); - // pTask->checkpointingId = checkpointId; + pTask->chkInfo.checkpointingId = checkpointId; pTask->chkInfo.checkpointId = checkpointId; // 1pTask->checkpointingId; pTask->chkInfo.checkpointVer = pItem->submitReqVer; pTask->info.triggerParam = pItem->fetchResultVer; + pTask->info.taskLevel = TASK_LEVEL_SMA; if (!checkpointBuilt) { // the stream states share one checkpoint diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index be3665fde7..7be655fbed 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -32,8 +32,8 @@ typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); typedef struct SAttachedEventInfo { - ETaskStatus status; // required status that this event can be handled - EStreamTaskEvent event; // the delayed handled event + ETaskStatus status; // required status that this event can be handled + EStreamTaskEvent event; // the delayed handled event } SAttachedEventInfo; typedef struct STaskStateTrans { @@ -64,8 +64,8 @@ typedef struct SStreamEventInfo { const char* name; } SStreamEventInfo; -SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); -void* streamDestroyStateMachine(SStreamTaskSM* pSM); +// SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); +// void* streamDestroyStateMachine(SStreamTaskSM* pSM); #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 7473e7a411..e2561de841 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -291,6 +291,10 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { return code; } + if (p->info.taskLevel > TASK_LEVEL__SINK) { + return code; + } + taosThreadMutexLock(&p->lock); ASSERT(p->chkInfo.checkpointId <= p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId && From 786383118c9f8ee5ee7ec608fe4f0a7a63540e09 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Nov 2023 10:39:17 +0800 Subject: [PATCH 144/151] fix drop stream case --- source/dnode/mnode/impl/src/mndStream.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d2bab4eebf..15cc3eb041 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1337,6 +1337,9 @@ static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { SCheckpointCandEntry *pEntry = pIter; SStreamObj *ps = mndAcquireStream(pMnode, pEntry->pName); + if (ps == NULL) { + continue; + } mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId); code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId); From cc1eb2769f036f3f23e8f32fb81b34a0fe5f193b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Nov 2023 20:29:42 +0800 Subject: [PATCH 145/151] fix crash while quit --- source/libs/stream/src/streamBackendRocksdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ccf166c390..19247166b3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1844,6 +1844,9 @@ void taskDbDestroy(void* pDb, bool flush) { rocksdb_column_family_handle_destroy(wrapper->pCf[i]); } } + + if (wrapper->db) rocksdb_close(wrapper->db); + rocksdb_options_destroy(wrapper->dbOpt); rocksdb_readoptions_destroy(wrapper->readOpt); rocksdb_writeoptions_destroy(wrapper->writeOpt); @@ -1851,7 +1854,6 @@ void taskDbDestroy(void* pDb, bool flush) { rocksdb_cache_destroy(wrapper->cache); taosMemoryFree(wrapper->pCf); - for (int i = 0; i < nCf; i++) { rocksdb_options_t* opt = wrapper->pCfOpts[i]; rocksdb_comparator_t* compare = wrapper->pCompares[i]; @@ -1867,8 +1869,6 @@ void taskDbDestroy(void* pDb, bool flush) { taosThreadMutexDestroy(&wrapper->mutex); - if (wrapper->db) rocksdb_close(wrapper->db); - taskDbDestroyChkpOpt(wrapper); taosMemoryFree(wrapper->idstr); From ade2f322a036047142dd2548b74b20536f70f7f6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 11:28:03 +0800 Subject: [PATCH 146/151] change default config --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e8ff530d78..f4d79d6e70 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -248,7 +248,7 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; -int32_t tsStreamCheckpointInterval = 10; +int32_t tsStreamCheckpointInterval = 60; float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 15; int32_t tsTtlUnit = 86400; @@ -738,7 +738,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 1, 1200, CFG_SCOPE_SERVER, + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; From 03b698d8aee80b5a52fea83d9bc4cbf53f7a307e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 14:50:07 +0800 Subject: [PATCH 147/151] change default config --- source/common/src/tglobal.c | 2 +- source/libs/stream/src/streamSnapshot.c | 6 +++++- tests/system-test/8-stream/snode_restart_with_checkpoint.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f4d79d6e70..9069f5ce99 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1477,7 +1477,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"supportVnodes", &tsNumOfSupportVnodes}, }; - if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { + if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); } } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index f76b49f300..e29f2ba7de 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -366,8 +366,12 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si int32_t code = 0; SStreamSnapHandle* pHandle = &pReader->handle; int32_t idx = pHandle->currIdx; + SBackendSnapFile2* pSnapFile = taosArrayGet(pHandle->pDbSnapSet, idx); - SBackendFileItem* item = NULL; + if (pSnapFile == NULL) { + return 0; + } + SBackendFileItem* item = NULL; _NEXT: diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py index 9567bbe439..d7bfd7b407 100644 --- a/tests/system-test/8-stream/snode_restart_with_checkpoint.py +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.query("use test") tdSql.query("create snode on dnode 4") tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") - tdLog.debug("========create stream useing snode and insert data ok========") + tdLog.debug("========create stream using snode and insert data ok========") time.sleep(60) tdDnodes = cluster.dnodes From 2e1192b97bf4c62863d5c70822f3cef8a8670624 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 16:16:26 +0800 Subject: [PATCH 148/151] change default config --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9069f5ce99..ced68c134e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -738,7 +738,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1477,7 +1477,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"supportVnodes", &tsNumOfSupportVnodes}, }; - if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { + if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); } } From b6881d572aff50811ca4a0a670306efcfc640a5f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 16:52:36 +0800 Subject: [PATCH 149/151] fix compile error on windows/mac --- source/common/src/cos.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 65b0a61590..7c8676e9f5 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1355,5 +1355,6 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, void s3EvictCache(const char *path, long object_size) {} long s3Size(const char *object_name) { return 0; } int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; } +int32_t s3GetObjectToFile(const char *object_name, char *fileName) { return 0; } #endif From a01ccf8bc96b390120501ad83415dcaf1c62b3ed Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 17:14:37 +0800 Subject: [PATCH 150/151] snode case forbidden --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8cb23c081f..eea0904e32 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -22,7 +22,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py From 08cc97697534e69f41d96319cd6f4c5d6d338a93 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Nov 2023 19:57:30 +0800 Subject: [PATCH 151/151] fix crash --- source/libs/stream/src/streamBackendRocksdb.c | 1 + source/libs/stream/src/streamMeta.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 19247166b3..630650025d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1631,6 +1631,7 @@ void* taskDbAddRef(void* pTaskDb) { return taosAcquireRef(taskDbWrapperId, pBackend->refId); } void taskDbRemoveRef(void* pTaskDb) { + if (pTaskDb == NULL) return; STaskDbWrapper* pBackend = pTaskDb; taosReleaseRef(taskDbWrapperId, pBackend->refId); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0e137a673e..bd23e41a84 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -255,6 +255,8 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { pTask->backendRefId = pBackend->refId; pTask->pBackend = pBackend; taosThreadMutexUnlock(&pMeta->backendMutex); + + stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } @@ -272,6 +274,8 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosThreadMutexUnlock(&pMeta->backendMutex); + + stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) {