add stream backend copy
This commit is contained in:
parent
4e68fbd744
commit
87c78919a9
|
@ -255,7 +255,8 @@ typedef struct SStoreMeta {
|
||||||
|
|
||||||
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
|
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
|
||||||
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
|
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
|
||||||
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
|
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
|
||||||
|
int64_t* numOfNormalTables);
|
||||||
|
|
||||||
int64_t (*getNumOfRowsInMem)(void* pVnode);
|
int64_t (*getNumOfRowsInMem)(void* pVnode);
|
||||||
|
|
||||||
|
@ -382,6 +383,7 @@ typedef struct SStateStore {
|
||||||
void (*streamStateDestroy)(SStreamState* pState, bool remove);
|
void (*streamStateDestroy)(SStreamState* pState, bool remove);
|
||||||
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
|
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
|
||||||
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
|
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
|
||||||
|
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
|
||||||
} SStateStore;
|
} SStateStore;
|
||||||
|
|
||||||
typedef struct SStorageAPI {
|
typedef struct SStorageAPI {
|
||||||
|
|
|
@ -96,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
|
||||||
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
|
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
|
||||||
|
|
||||||
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
|
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
|
||||||
|
|
||||||
|
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
|
||||||
|
|
||||||
SStreamStateCur* createStreamStateCursor();
|
SStreamStateCur* createStreamStateCursor();
|
||||||
|
|
||||||
/***compare func **/
|
/***compare func **/
|
||||||
|
|
|
@ -67,6 +67,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
|
||||||
sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
|
sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
|
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
|
||||||
SReadHandle handle = {
|
SReadHandle handle = {
|
||||||
.checkpointId = pTask->chkInfo.checkpointId,
|
.checkpointId = pTask->chkInfo.checkpointId,
|
||||||
|
|
|
@ -14,8 +14,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "storageapi.h"
|
#include "storageapi.h"
|
||||||
#include "tstreamUpdate.h"
|
|
||||||
#include "streamState.h"
|
#include "streamState.h"
|
||||||
|
#include "tstreamUpdate.h"
|
||||||
|
|
||||||
static void initStateStoreAPI(SStateStore* pStore);
|
static void initStateStoreAPI(SStateStore* pStore);
|
||||||
static void initFunctionStateStore(SFunctionStateStore* pStore);
|
static void initFunctionStateStore(SFunctionStateStore* pStore);
|
||||||
|
@ -103,6 +103,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
||||||
pStore->streamStateDestroy = streamStateDestroy;
|
pStore->streamStateDestroy = streamStateDestroy;
|
||||||
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
||||||
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
||||||
|
pStore->streamStateCopyBackend = streamStateCopyBackend;
|
||||||
}
|
}
|
||||||
|
|
||||||
void initFunctionStateStore(SFunctionStateStore* pStore) {
|
void initFunctionStateStore(SFunctionStateStore* pStore) {
|
||||||
|
|
|
@ -736,15 +736,15 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
|
||||||
}
|
}
|
||||||
|
|
||||||
streamMetaWLock(pMeta);
|
streamMetaWLock(pMeta);
|
||||||
code = streamMetaReopen(pMeta);
|
// code = streamMetaReopen(pMeta);
|
||||||
if (code != TSDB_CODE_SUCCESS) {
|
// if (code != TSDB_CODE_SUCCESS) {
|
||||||
tqError("vgId:%d failed to reopen stream meta", vgId);
|
// tqError("vgId:%d failed to reopen stream meta", vgId);
|
||||||
streamMetaWUnLock(pMeta);
|
// streamMetaWUnLock(pMeta);
|
||||||
code = terrno;
|
// code = terrno;
|
||||||
return code;
|
// return code;
|
||||||
}
|
// }
|
||||||
|
|
||||||
streamMetaInitBackend(pMeta);
|
// streamMetaInitBackend(pMeta);
|
||||||
int64_t el = taosGetTimestampMs() - st;
|
int64_t el = taosGetTimestampMs() - st;
|
||||||
|
|
||||||
tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.);
|
tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.);
|
||||||
|
|
|
@ -212,6 +212,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
||||||
pStore->streamStateDestroy = streamStateDestroy;
|
pStore->streamStateDestroy = streamStateDestroy;
|
||||||
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
||||||
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
||||||
|
pStore->streamStateCopyBackend = streamStateCopyBackend;
|
||||||
}
|
}
|
||||||
|
|
||||||
void initMetaReaderAPI(SStoreMetaReader* pMetaReader) {
|
void initMetaReaderAPI(SStoreMetaReader* pMetaReader) {
|
||||||
|
|
|
@ -1465,7 +1465,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
|
||||||
initBasicInfo(&pInfo->binfo, pResBlock);
|
initBasicInfo(&pInfo->binfo, pResBlock);
|
||||||
|
|
||||||
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
|
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
|
||||||
|
// qError("open state %p", pInfo->pState);
|
||||||
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
|
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
|
||||||
|
// qError("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);
|
||||||
|
|
||||||
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
|
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
|
||||||
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
|
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
|
||||||
|
@ -3341,7 +3343,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
||||||
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
|
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
|
||||||
for (int32_t i = 0; i < rows; i += winRows) {
|
for (int32_t i = 0; i < rows; i += winRows) {
|
||||||
if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
|
if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
|
||||||
&pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) || colDataIsNull_s(pKeyColInfo, i)) {
|
&pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) ||
|
||||||
|
colDataIsNull_s(pKeyColInfo, i)) {
|
||||||
i++;
|
i++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,8 +91,8 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
|
||||||
}
|
}
|
||||||
|
|
||||||
SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
|
SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
|
||||||
stDebug("open stream state, %s", path);
|
|
||||||
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
|
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
|
||||||
|
stDebug("open stream state %p, %s", pState, path);
|
||||||
if (pState == NULL) {
|
if (pState == NULL) {
|
||||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||||
return NULL;
|
return NULL;
|
||||||
|
@ -710,7 +710,8 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen) {
|
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
|
||||||
|
const SSessionKey* pKey, void** pVal, int32_t* pVLen) {
|
||||||
#ifdef USE_ROCKSDB
|
#ifdef USE_ROCKSDB
|
||||||
return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen);
|
return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen);
|
||||||
#else
|
#else
|
||||||
|
@ -1115,6 +1116,10 @@ int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) {
|
||||||
|
|
||||||
void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); }
|
void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); }
|
||||||
|
|
||||||
|
void streamStateCopyBackend(SStreamState* src, SStreamState* dst) {
|
||||||
|
dst->pTdbState->pOwner = src->pTdbState->pOwner;
|
||||||
|
return;
|
||||||
|
}
|
||||||
SStreamStateCur* createStreamStateCursor() {
|
SStreamStateCur* createStreamStateCursor() {
|
||||||
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
|
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
|
||||||
pCur->buffIndex = -1;
|
pCur->buffIndex = -1;
|
||||||
|
|
Loading…
Reference in New Issue