From 8c5dbde918299250fc6649c0903cf80bc4dca19c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 6 Jul 2022 14:20:07 +0800 Subject: [PATCH 01/39] refactor(stream): stream reader created in scanner --- examples/c/tmq.c | 6 +- include/common/tmsg.h | 8 +- include/libs/executor/executor.h | 6 +- include/libs/stream/tstream.h | 13 +- include/libs/wal/wal.h | 4 +- include/util/tutil.h | 1 + source/client/src/tmq.c | 27 ++- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tq.c | 25 ++- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/libs/executor/inc/executorimpl.h | 8 +- source/libs/executor/src/executor.c | 14 +- source/libs/executor/src/executorimpl.c | 16 +- source/libs/executor/src/scanoperator.c | 164 +++++++++--------- source/libs/executor/src/timewindowoperator.c | 14 +- source/libs/stream/inc/streamInc.h | 6 +- source/libs/stream/src/stream.c | 3 - source/libs/wal/src/walWrite.c | 2 +- source/util/src/tutil.c | 14 ++ tests/system-test/7-tmq/stbTagFilter.py | 4 +- 20 files changed, 189 insertions(+), 150 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 697a53e570..378f9ffb24 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -137,8 +137,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -199,7 +199,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + /*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/ tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dedc06a2b9..7e6231d7dd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2826,8 +2826,8 @@ typedef struct { static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) { int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + // tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + // tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI16(buf, pRsp->resMsgType); tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen); tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen); @@ -2835,8 +2835,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp } static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + // buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + // buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI16(buf, &pRsp->resMsgType); buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen); buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 45fa94b3bf..7b4565a99f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -30,13 +30,15 @@ struct SRpcMsg; struct SSubplan; typedef struct SReadHandle { - void* reader; + void* streamReader; void* meta; void* config; void* vnode; void* mnd; SMsgCb* pMsgCb; - bool tqReader; + bool initMetaReader; + bool initTableReader; + bool initStreamReader; } SReadHandle; typedef enum { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 67074c789e..52f671e176 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -223,7 +223,7 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -struct SStreamTask { +typedef struct SStreamTask { int64_t streamId; int32_t taskId; int8_t isDataScan; @@ -235,6 +235,11 @@ struct SStreamTask { int8_t taskStatus; int8_t execStatus; + // exec info + int64_t enqueueVer; + int64_t processedVer; + int64_t checkpointVer; + // node info int32_t selfChildId; int32_t nodeId; @@ -277,7 +282,7 @@ struct SStreamTask { // msg handle SMsgCb* pMsgCb; -}; +} SStreamTask; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); @@ -288,6 +293,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { +#if 0 while (1) { int8_t inputStatus = atomic_val_compare_exchange_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL, TASK_INPUT_STATUS__PROCESSING); @@ -296,6 +302,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem } ASSERT(0); } +#endif if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); @@ -316,8 +323,10 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } +#if 0 // TODO: back pressure atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); +#endif return 0; } diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index c11651970c..e32a8d1055 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -178,7 +178,6 @@ void walFsync(SWal *, bool force); // apis for lifecycle management int32_t walCommit(SWal *, int64_t ver); -// truncate after int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); @@ -207,10 +206,11 @@ void walCloseRef(SWalRef *); int32_t walRefVer(SWalRef *, int64_t ver); int32_t walUnrefVer(SWal *); +// help function for raft bool walLogExist(SWal *, int64_t ver); +bool walIsEmpty(SWal *); // lifecycle check -bool walIsEmpty(SWal *); int64_t walGetFirstVer(SWal *); int64_t walGetSnapshotVer(SWal *); int64_t walGetLastVer(SWal *); diff --git a/include/util/tutil.h b/include/util/tutil.h index 6a1a40f14c..2e96c5b88e 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -45,6 +45,7 @@ void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); +char *strDupUnquo(const char *src); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 667f5b1dbc..331f149e13 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -49,19 +49,18 @@ struct tmq_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[TSDB_CGROUP_LEN]; - int8_t autoCommit; - int8_t resetOffset; - int8_t withTbName; - int8_t spEnable; - int32_t spBatchSize; - uint16_t port; - int32_t autoCommitInterval; - char* ip; - char* user; - char* pass; - /*char* db;*/ + char clientId[256]; + char groupId[TSDB_CGROUP_LEN]; + int8_t autoCommit; + int8_t resetOffset; + int8_t withTbName; + int8_t spEnable; + int32_t spBatchSize; + uint16_t port; + int32_t autoCommitInterval; + char* ip; + char* user; + char* pass; tmq_commit_cb* commitCb; void* commitCbUserParam; }; @@ -337,7 +336,7 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; - char* topic = strdup(src); + char* topic = strDupUnquo(src); if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4e1b2db44a..7a147f5ffa 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -332,7 +332,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } SReadHandle handle = { - .reader = pReadHandle, + .streamReader = pReadHandle, .meta = pMeta, .pMsgCb = pMsgCb, .vnode = pVnode, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5ce3cfab45..95ec99c9e5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -28,8 +28,12 @@ int32_t tqInit() { atomic_store_8(&tqMgmt.inited, 0); return -1; } + if (streamInit() < 0) { + return -1; + } atomic_store_8(&tqMgmt.inited, 1); } + return 0; } @@ -361,8 +365,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ASSERT(IS_META_MSG(pHead->msgType)); tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; - metaRsp.reqOffset = pReq->reqOffset.version; - metaRsp.rspOffset = fetchVer; + /*metaRsp.reqOffset = pReq->reqOffset.version;*/ + /*metaRsp.rspOffset = fetchVer;*/ + /*metaRsp.rspOffsetNew.version = fetchVer;*/ + tqOffsetResetToLog(&metaRsp.reqOffsetNew, pReq->reqOffset.version); + tqOffsetResetToLog(&metaRsp.rspOffsetNew, fetchVer); metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; @@ -448,10 +455,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { req.qmsg = NULL; for (int32_t i = 0; i < 5; i++) { SReadHandle handle = { - .reader = pHandle->execHandle.pExecReader[i], + .streamReader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .tqReader = true, + .initTableReader = true, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -522,11 +529,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->execType != TASK_EXEC__NONE) { // expand runners if (pTask->isDataScan) { - SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - SReadHandle handle = { - .reader = pStreamReader, - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, + /*SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);*/ + SReadHandle handle = { + .meta = pTq->pVnode->pMeta, + .vnode = pTq->pVnode, + .initStreamReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 398a09ecbc..17f4dc5426 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -84,7 +84,7 @@ int32_t tqMetaOpen(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { for (int32_t i = 0; i < 5; i++) { SReadHandle reader = { - .reader = handle.execHandle.pExecReader[i], + .streamReader = handle.execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, }; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 00f2e09e0c..9c7dc234ec 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -338,7 +338,7 @@ typedef struct SessionWindowSupporter { uint8_t parentType; } SessionWindowSupporter; -typedef struct SStreamBlockScanInfo { +typedef struct SStreamScanInfo { uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; @@ -355,7 +355,7 @@ typedef struct SStreamBlockScanInfo { int32_t blockType; // current block type int32_t validBlockIndex; // Is current data has returned? uint64_t numOfExec; // execution times - void* streamBlockReader;// stream block reader handle + void* streamReader;// stream block reader handle int32_t tsArrayIndex; SArray* tsArray; @@ -364,7 +364,7 @@ typedef struct SStreamBlockScanInfo { EStreamScanMode scanMode; SOperatorInfo* pStreamScanOp; - SOperatorInfo* pSnapshotReadOp; + SOperatorInfo* pTableScanOp; SArray* childIds; SessionWindowSupporter sessionSup; bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. @@ -373,7 +373,7 @@ typedef struct SStreamBlockScanInfo { SSDataBlock* pPullDataRes; // pull data SSDataBlock SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; -} SStreamBlockScanInfo; +} SStreamScanInfo; typedef struct SSysTableScanInfo { SRetrieveMetaTableRsp* pRsp; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0e76607c8f..3591aaf975 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -37,7 +37,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } else { pOperator->status = OP_NOT_OPENED; - SStreamBlockScanInfo* pInfo = pOperator->info; + SStreamScanInfo* pInfo = pOperator->info; pInfo->assignBlockUid = assignUid; // TODO: if a block was set but not consumed, @@ -45,7 +45,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pInfo->blockType = type; if (type == STREAM_INPUT__DATA_SUBMIT) { - if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) { + if (tqReadHandleSetMsg(pInfo->streamReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } @@ -130,7 +130,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, const SArray* tableIdList) { +static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -168,17 +168,17 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo pInfo = pInfo->pDownstream[0]; } - int32_t code = 0; - SStreamBlockScanInfo* pScanInfo = pInfo->info; + int32_t code = 0; + SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); - code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); + code = tqReadHandleAddTbUidList(pScanInfo->streamReader, qa); taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); - code = tqReadHandleRemoveTbUidList(pScanInfo->streamBlockReader, tableIdList); + code = tqReadHandleRemoveTbUidList(pScanInfo->streamReader, tableIdList); } return code; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 25b61e15c3..c92fc1b1fb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2847,12 +2847,12 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { pOperator->status = OP_OPENED; if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamBlockScanInfo* pScanInfo = pOperator->info; + SStreamScanInfo* pScanInfo = pOperator->info; pScanInfo->blockType = STREAM_INPUT__DATA_SCAN; - pScanInfo->pSnapshotReadOp->status = OP_OPENED; + pScanInfo->pTableScanOp->status = OP_OPENED; - STableScanInfo* pInfo = pScanInfo->pSnapshotReadOp->info; + STableScanInfo* pInfo = pScanInfo->pTableScanOp->info; ASSERT(pInfo->scanMode == TABLE_SCAN__TABLE_ORDER); if (uid == 0) { @@ -2912,8 +2912,8 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) { int32_t type = pOperator->operatorType; if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamBlockScanInfo* pScanInfo = pOperator->info; - STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info; + SStreamScanInfo* pScanInfo = pOperator->info; + STableScanInfo* pSnapShotScanInfo = pScanInfo->pTableScanOp->info; *uid = pSnapShotScanInfo->lastStatus.uid; *ts = pSnapShotScanInfo->lastStatus.ts; } else { @@ -4537,9 +4537,9 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI } return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { - SStreamBlockScanInfo* pInfo = pOperator->info; - ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); - *ppInfo = pInfo->pSnapshotReadOp->info; + SStreamScanInfo* pInfo = pOperator->info; + ASSERT(pInfo->pTableScanOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + *ppInfo = pInfo->pTableScanOp->info; return 0; } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 515efb86f3..2ef0b7470e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -788,7 +788,7 @@ _error: return NULL; } -static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { +static void doClearBufferedBlocks(SStreamScanInfo* pInfo) { size_t total = taosArrayGetSize(pInfo->pBlockLists); pInfo->validBlockIndex = 0; @@ -799,11 +799,11 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); } -static bool isSessionWindow(SStreamBlockScanInfo* pInfo) { +static bool isSessionWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } -static bool isStateWindow(SStreamBlockScanInfo* pInfo) { +static bool isStateWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } @@ -828,23 +828,21 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { */ } -static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { +static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { ASSERT(rowIndex < pBlock->info.rows); - switch (pBlock->info.type) - { - case STREAM_DELETE_DATA: - case STREAM_RETRIEVE: { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); - uint64_t* groupCol = (uint64_t*)pColInfo->pData; - pInfo->groupId = groupCol[rowIndex]; - } - break; - default: - break; + switch (pBlock->info.type) { + case STREAM_DELETE_DATA: + case STREAM_RETRIEVE: { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); + uint64_t* groupCol = (uint64_t*)pColInfo->pData; + pInfo->groupId = groupCol[rowIndex]; + } break; + default: + break; } } -static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { +static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, .ekey = INT64_MAX, @@ -864,11 +862,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3 win = pCurWin->win; (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL); } else { - win = - getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL); + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL); setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); - (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + (*pRowIndex) += + getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } needRead = true; } else if (isStateWindow(pInfo)) { @@ -886,7 +883,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3 if (!needRead) { return false; } - STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -911,14 +908,14 @@ static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRow dest->info.rows++; } -static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { +static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pSnapshotReadOp); + pResult = doTableScan(pInfo->pTableScanOp); if (pResult == NULL) { if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) { // scan next window data - pResult = doTableScan(pInfo->pSnapshotReadOp); + pResult = doTableScan(pInfo->pTableScanOp); } } if (!pResult) { @@ -943,7 +940,8 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, i */ } -static void copyDeleteDataBlock(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) { +static void copyDeleteDataBlock(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, + SSDataBlock* pUpdateRes) { if (pDelBlock->info.rows == 0) { return; } @@ -951,18 +949,20 @@ static void copyDeleteDataBlock(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBl blockDataEnsureCapacity(pUpdateRes, 64); ASSERT(taosArrayGetSize(pDelBlock->pDataBlock) >= 3); SColumnInfoData* pStartTsCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startData = (TSKEY*)pStartTsCol->pData; + TSKEY* startData = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endData = (TSKEY*)pEndTsCol->pData; + TSKEY* endData = (TSKEY*)pEndTsCol->pData; SColumnInfoData* pGpCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* uidCol = (uint64_t*)pGpCol->pData; + uint64_t* uidCol = (uint64_t*)pGpCol->pData; SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, DELETE_GROUPID_COLUMN_INDEX); - for (int32_t i = pInfo->deleteDataIndex ; i < pDelBlock->info.rows && - i < pDelBlock->info.capacity - (endData[i] - startData[i])/pInfo->interval.interval - 1; i++) { + for (int32_t i = pInfo->deleteDataIndex; + i < pDelBlock->info.rows && + i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1; + i++) { uint64_t groupId = getGroupId(pOperator, uidCol[i]); - for (TSKEY startTs = startData[i]; startTs <= endData[i]; ) { + for (TSKEY startTs = startData[i]; startTs <= endData[i];) { colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false); colDataAppend(pDestGpCol, pUpdateRes->info.rows, (const char*)&groupId, false); pUpdateRes->info.rows++; @@ -977,7 +977,7 @@ static void copyDeleteDataBlock(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBl } } -static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { +static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { blockDataCleanup(pUpdateBlock); int32_t size = taosArrayGetSize(pInfo->tsArray); if (pInfo->tsArrayIndex < size) { @@ -986,11 +986,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa blockDataEnsureCapacity(pUpdateBlock, size); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); - pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid); + pInfo->groupId = getGroupId(pInfo->pTableScanOp, pBlock->info.uid); int32_t i = 0; for (; i < size; i++) { rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); - uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid); + uint64_t id = getGroupId(pInfo->pTableScanOp, pBlock->info.uid); if (pInfo->groupId != id) { break; } @@ -1009,12 +1009,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa } if (size == 0) { - copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pUpdateBlock); + copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock); } } -static void checkUpdateData(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, - bool out) { +static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* ts = (TSKEY*)pColDataInfo->pData; @@ -1030,15 +1029,15 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32 SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex); uint64_t* uidCol = (uint64_t*)pColDataInfo->pData; ASSERT(pBlock->info.rows > 0); - for (int32_t i = 0 ; i < pBlock->info.rows; i++) { + for (int32_t i = 0; i < pBlock->info.rows; i++) { uidCol[i] = getGroupId(pOperator, uidCol[i]); } } -static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamBlockScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamScanInfo* pInfo = pOperator->info; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { @@ -1056,30 +1055,29 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { int32_t current = pInfo->validBlockIndex++; SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + // TODO move into scan blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { - case STREAM_RETRIEVE:{ - pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; - copyDataBlock(pInfo->pPullDataRes, pBlock); - pInfo->pullDataResIndex = 0; - prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex); - updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); - } - break; - case STREAM_DELETE_DATA: { - pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - copyDataBlock(pInfo->pDeleteDataRes, pBlock); - copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pInfo->pUpdateRes); - pInfo->updateResIndex = 0; - prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex); - pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; - return pInfo->pUpdateRes; - } - break; - default: - break; + case STREAM_RETRIEVE: { + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; + copyDataBlock(pInfo->pPullDataRes, pBlock); + pInfo->pullDataResIndex = 0; + prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex); + updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + } break; + case STREAM_DELETE_DATA: { + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; + copyDataBlock(pInfo->pDeleteDataRes, pBlock); + copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes); + pInfo->updateResIndex = 0; + prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex); + pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; + return pInfo->pUpdateRes; + } break; + default: + break; } return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { @@ -1128,11 +1126,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->streamBlockReader)) { + while (tqNextDataBlock(pInfo->streamReader)) { SSDataBlock block = {0}; // todo refactor - int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader); + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamReader); uint64_t groupId = block.info.groupId; uint64_t uid = block.info.uid; @@ -1228,11 +1226,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } } + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) { // check reader last status // if not match, reset status - SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); + SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); return pResult && pResult->info.rows > 0 ? pResult : NULL; } else { @@ -1256,8 +1256,8 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { - SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -1295,32 +1295,40 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } if (pHandle) { - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); - STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; + SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); + STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanOp->info; SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0); - if (pHandle->tqReader) { + if (pHandle->initTableReader) { pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pSTInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0); } + if (pHandle->initStreamReader) { + ASSERT(pHandle->streamReader == NULL); + pInfo->streamReader = tqInitSubmitMsgScanner(pHandle->meta); + ASSERT(pInfo->streamReader); + } else { + ASSERT(pHandle->streamReader); + pInfo->streamReader = pHandle->streamReader; + } + if (pSTInfo->interval.interval > 0) { pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark); } else { pInfo->pUpdateInfo = NULL; } - pInfo->pSnapshotReadOp = pTableScanDummy; + + pInfo->pTableScanOp = pTableScanOp; pInfo->interval = pSTInfo->interval; pInfo->readHandle = *pHandle; - ASSERT(pHandle->reader); - pInfo->streamBlockReader = pHandle->reader; pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds); + tqReadHandleSetColIdList((SStreamReader*)pHandle->streamReader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); - int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList); + int32_t code = tqReadHandleSetTbUidList(pHandle->streamReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -1344,7 +1352,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->deleteDataIndex = 0; pInfo->pDeleteDataRes = createPullDataBlock(); - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; @@ -1353,7 +1361,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); return pOperator; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fdef432d95..24749fde2c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2980,7 +2980,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); - SStreamBlockScanInfo* pScanInfo = downstream->info; + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } @@ -3464,8 +3464,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*); SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; } SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, - SArray* pClosed, __get_win_info_ fn, bool delete) { +int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn, + bool delete) { // Todo(liuyao) save window to tdb void** pIte = NULL; size_t keyLen = 0; @@ -3604,8 +3604,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForSession, pInfo->ignoreExpiredData); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession, + pInfo->ignoreExpiredData); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData); copyUpdateResult(pStUpdated, pUpdated); taosHashCleanup(pStUpdated); @@ -4097,8 +4097,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForState, pInfo->ignoreExpiredData); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState, + pInfo->ignoreExpiredData); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData); copyUpdateResult(pSeUpdated, pUpdated); taosHashCleanup(pSeUpdated); diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 2f41c08354..1629c863d5 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -17,6 +17,7 @@ #define _STREAM_INC_H_ #include "executor.h" +#include "tref.h" #include "tstream.h" #ifdef __cplusplus @@ -24,8 +25,9 @@ extern "C" { #endif typedef struct { - int8_t inited; - void* timer; + int8_t inited; + int32_t refPool; + void* timer; } SStreamGlobalEnv; static SStreamGlobalEnv streamEnv; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 56d063ae51..8b8badd67a 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -76,9 +76,6 @@ void streamTriggerByTimer(void* param, void* tmrId) { int32_t streamSetupTrigger(SStreamTask* pTask) { if (pTask->triggerParam != 0) { - if (streamInit() < 0) { - return -1; - } pTask->timer = taosTmrStart(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); pTask->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 27f12259bc..d7fa6695d0 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -79,13 +79,13 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { + taosThreadMutexLock(&pWal->mutex); int64_t code; char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } - taosThreadMutexLock(&pWal->mutex); // find correct file if (ver < walGetLastFileFirstVer(pWal)) { diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index addb9f55ba..7f3728e2ad 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -64,6 +64,20 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } +char *strDupUnquo(const char *src) { + if (src == NULL) return NULL; + if (src[0] != '`') return strdup(src); + int32_t len = (int32_t)strlen(src); + if (src[len - 1] != '`') return NULL; + char *ret = taosMemoryMalloc(len); + if (ret == NULL) return NULL; + for (int32_t i = 0; i < len - 1; i++) { + ret[i] = src[i + 1]; + } + ret[len - 1] = 0; + return ret; +} + size_t strtrim(char *z) { int32_t i = 0; int32_t j = 0; diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 2a2cb40c09..65609629bc 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -25,7 +25,7 @@ class TDTestCase: paraDict = {'dbName': 'db2', 'dropFlag': 1, 'event': '', - 'vgroups': 4, + 'vgroups': 1, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', @@ -44,7 +44,7 @@ class TDTestCase: topicNameList = ['topic1'] expectRowsList = [] tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") From e51e261e01f539162bff176751ab12a394796188 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 6 Jul 2022 21:00:15 +0800 Subject: [PATCH 02/39] fix(stream): create operator --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2ef0b7470e..7d9889e6a1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1326,9 +1326,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList((SStreamReader*)pHandle->streamReader, pColIds); + tqReadHandleSetColIdList(pInfo->streamReader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); - int32_t code = tqReadHandleSetTbUidList(pHandle->streamReader, tableIdList); + int32_t code = tqReadHandleSetTbUidList(pInfo->streamReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; From baa48489b7f901db5ba89c7e0b220eb32dfd4a57 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 6 Jul 2022 21:25:34 +0800 Subject: [PATCH 03/39] merge from 3.0 --- include/common/tmsgdef.h | 1 + include/libs/nodes/plannodes.h | 1 + source/client/src/clientEnv.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 5 +- source/dnode/mnode/impl/src/mndMain.c | 3 +- source/dnode/mnode/impl/src/mndQuery.c | 2 + source/dnode/qnode/src/qnode.c | 1 + source/dnode/vnode/inc/vnode.h | 25 +- source/dnode/vnode/src/inc/vnd.h | 2 + source/dnode/vnode/src/vnd/vnodeSvr.c | 16 +- source/dnode/vnode/src/vnd/vnodeSync.c | 52 +++-- source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 158 +++++++++++-- source/libs/executor/src/timewindowoperator.c | 217 +++++++++++++----- source/libs/function/src/udfd.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 7 + source/libs/qworker/inc/qwInt.h | 1 + source/libs/qworker/inc/qwMsg.h | 3 +- source/libs/qworker/src/qwMsg.c | 6 +- source/libs/qworker/src/qworker.c | 8 +- source/libs/qworker/test/qworkerTests.cpp | 4 +- source/libs/scheduler/inc/schInt.h | 5 +- source/libs/scheduler/src/schRemote.c | 8 +- source/libs/scheduler/src/schTask.c | 6 +- source/libs/sync/src/syncMain.c | 7 +- tests/script/tsim/stream/state0.sim | 49 ++++ 32 files changed, 473 insertions(+), 135 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 806c0b5122..8b39530e84 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -209,6 +209,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_CONTINUE, "query-continue", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_HEARTBEAT, "query-heartbeat", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_FETCH, "fetch", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SCH_MERGE_FETCH, "merge-fetch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_CANCEL_TASK, "cancel-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_DROP_TASK, "drop-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index d3e9840987..6a865b4e2a 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -353,6 +353,7 @@ typedef struct SDownstreamSourceNode { uint64_t taskId; uint64_t schedId; int32_t execId; + int32_t fetchMsgType; } SDownstreamSourceNode; typedef struct SExchangePhysiNode { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 797d58e6ef..89ecf16b40 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -102,7 +102,7 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (NEED_REDIRECT_ERROR(code)) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 59d68b2110..d60b69daba 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -214,6 +214,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 1f22eefddf..14cb1bd533 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -111,6 +111,7 @@ SArray *qmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH_RSP, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1f7347203d..a3df32a08c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -328,6 +328,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 3913e3fda8..1d795c74f2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -238,9 +238,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { } int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { - pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg); + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeWriteMsg); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); - pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyMsg); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 7043991525..ad7fbf4344 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -42,7 +42,7 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; + SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType}; int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); rsp.pCont = rpcMallocCont(contLen); @@ -88,6 +88,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { case TDMT_MND_SYSTABLE_RETRIEVE_RSP: case TDMT_DND_SYSTABLE_RETRIEVE_RSP: case TDMT_SCH_FETCH_RSP: + case TDMT_SCH_MERGE_FETCH_RSP: qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0); return; case TDMT_MND_STATUS_RSP: @@ -253,7 +254,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5e708616fd..f18f3c983e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -531,7 +531,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY || pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT || - pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { + pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || + pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index aec99fa3b7..5a527b994e 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -45,6 +45,7 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg, 0); break; case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg, 0); break; case TDMT_SCH_DROP_TASK: @@ -72,6 +73,7 @@ int32_t mndInitQuery(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_QUERY, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_CONTINUE, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_FETCH, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_FETCH, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_DROP_TASK, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_HEARTBEAT, mndProcessQueryMsg); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index cfc63b083d..723402e639 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -86,6 +86,7 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_FETCH_RSP: diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 0498d889d6..6320f4719d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -51,15 +51,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodeClose(SVnode *pVnode); -int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg); -int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); -int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); -int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); -int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); -int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); + int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); @@ -68,14 +60,25 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever); int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader); int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData); + int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); void *vnodeGetIdx(SVnode *pVnode); void *vnodeGetIvtIdx(SVnode *pVnode); -void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); -void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); +int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); +int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); + +int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg); +int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); + +int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); +int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); +int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); +void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); +void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); // meta typedef struct SMeta SMeta; // todo: remove diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 32be479116..cb25e93cde 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -94,6 +94,8 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path); void vnodeSyncStart(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); +void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); +bool vnodeIsLeader(SVnode* pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8ad92d0478..b3e9f53a5a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -28,7 +28,7 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg) { +int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; SDecoder dc = {0}; @@ -133,7 +133,7 @@ _err: return code; } -int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { +int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { void *ptr = NULL; void *pReq; int32_t len; @@ -261,6 +261,11 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); + if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { + vnodeRedirectRpcMsg(pVnode, pMsg); + return 0; + } + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_SCH_QUERY: @@ -276,11 +281,18 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); + if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG) + && !vnodeIsLeader(pVnode)) { + vnodeRedirectRpcMsg(pVnode, pMsg); + return 0; + } + char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); switch (pMsg->msgType) { case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_SCH_FETCH_RSP: return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg, 0); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6f6102ea14..f75ccba4bb 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -120,7 +120,24 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { return code; } -void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { +void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { + SEpSet newEpSet = {0}; + syncGetRetryEpSet(pVnode->sync, &newEpSet); + + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->config.vgId, pMsg, + newEpSet.numOfEps, newEpSet.inUse); + for (int32_t i = 0; i < newEpSet.numOfEps; ++i) { + vGTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", pVnode->config.vgId, pMsg, i, newEpSet.eps[i].fqdn, + newEpSet.eps[i].port); + } + pMsg->info.hasEpSet = 1; + + SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + tmsgSendRedirectRsp(&rsp, &newEpSet); +} + +void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; int32_t code = 0; @@ -131,7 +148,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { const STraceId *trace = &pMsg->info.traceId; vGTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle); - code = vnodePreProcessReq(pVnode, pMsg); + code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { vError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); } else { @@ -141,7 +158,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType)); if (code > 0) { SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; - if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { + if (vnodeProcessWriteMsg(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { rsp.code = terrno; vError("vgId:%d, msg:%p failed to apply right now since %s", vgId, pMsg, terrstr()); } @@ -156,16 +173,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vnodeAccumBlockMsg(pVnode, pMsg->msgType); } else if (code < 0) { if (terrno == TSDB_CODE_SYN_NOT_LEADER) { - SEpSet newEpSet = {0}; - syncGetRetryEpSet(pVnode->sync, &newEpSet); - vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps, - newEpSet.inUse); - for (int32_t i = 0; i < newEpSet.numOfEps; ++i) { - vGTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port); - } - pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; - tmsgSendRedirectRsp(&rsp, &newEpSet); + vnodeRedirectRpcMsg(pVnode, pMsg); } else { if (terrno != 0) code = terrno; vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code); @@ -185,7 +193,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vnodeWaitBlockMsg(pVnode); } -void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { +void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; int32_t code = 0; @@ -199,7 +207,7 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { - if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { + if (vnodeProcessWriteMsg(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { rsp.code = terrno; vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr()); } @@ -513,3 +521,17 @@ void vnodeSyncStart(SVnode *pVnode) { } void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } + +bool vnodeIsLeader(SVnode *pVnode) { + if (!syncIsReady(pVnode->sync)) { + return false; + } + + // todo + // if (!pVnode->restored) { + // terrno = TSDB_CODE_APP_NOT_READY; + // return false; + // } + + return true; +} \ No newline at end of file diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1d93903019..9b32560041 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -319,8 +319,9 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DATAREADER, + STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it STREAM_SCAN_FROM_DATAREADER_RETRIEVE, + STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; typedef struct SCatchSupporter { @@ -612,6 +613,7 @@ typedef struct SStreamSessionAggOperatorInfo { SSDataBlock* pWinBlock; // window result SqlFunctionCtx* pDummyCtx; // for combine SSDataBlock* pDelRes; // delete result + bool returnDelete; SSDataBlock* pUpdateRes; // update window SHashObj* pStDeleted; void* pDelIterator; @@ -889,6 +891,9 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); +SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, + TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 63f8c9769b..fa84d79f56 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2053,7 +2053,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf pMsgSendInfo->param = pWrapper; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = TDMT_SCH_FETCH; + pMsgSendInfo->msgType = pSource->fetchMsgType; pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 49e79639a8..aae85bbbc1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -792,13 +792,19 @@ static void doClearBufferedBlocks(SStreamScanInfo* pInfo) { } static bool isSessionWindow(SStreamScanInfo* pInfo) { - return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } static bool isStateWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } +static bool isIntervalWindow(SStreamScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; +} + static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); if (groupId) { @@ -834,6 +840,49 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou } } +void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { + pTableScanInfo->cond.twindows[0] = *pWin; + pTableScanInfo->curTWinIdx = 0; + // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // if (!pTableScanInfo->dataReader) { + // return false; + // } + pTableScanInfo->scanTimes = 0; + pTableScanInfo->currentGroupId = -1; +} + +static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { + if ((*pRowIndex) == pBlock->info.rows) { + return false; + } + + ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startData = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endData = (TSKEY*)pEndTsCol->pData; + STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]}; + setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex); + (*pRowIndex)++; + + for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) { + if (win.skey == startData[*pRowIndex]) { + win.ekey = TMAX(win.ekey, endData[*pRowIndex]); + continue; + } + if (win.skey == endData[*pRowIndex]) { + win.skey = TMIN(win.skey, startData[*pRowIndex]); + continue; + } + ASSERT((win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || + (isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); + break; + } + + resetTableScanInfo(pInfo->pTableScanOp->info, &win); + return true; +} + static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, @@ -852,6 +901,7 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[*pRowIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; + setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL); } else { win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL); @@ -875,15 +925,7 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t if (!needRead) { return false; } - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pTableScanInfo->cond.twindows[0] = win; - pTableScanInfo->curTWinIdx = 0; - // tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - // if (!pTableScanInfo->dataReader) { - // return false; - // } - pTableScanInfo->scanTimes = 0; - pTableScanInfo->currentGroupId = -1; + resetTableScanInfo(pInfo->pTableScanOp->info, &win); return true; } @@ -900,6 +942,26 @@ static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRow dest->info.rows++; } +static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { + while (1) { + SSDataBlock* pResult = NULL; + pResult = doTableScan(pInfo->pTableScanOp); + if (!pResult && prepareRangeScan(pInfo, pSDB, pRowIndex)) { + // scan next window data + pResult = doTableScan(pInfo->pTableScanOp); + } + if (!pResult) { + blockDataCleanup(pSDB); + *pRowIndex = 0; + return NULL; + } + + if (pResult->info.groupId == pInfo->groupId) { + return pResult; + } + } +} + static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { SSDataBlock* pResult = NULL; @@ -931,9 +993,8 @@ static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_ return pResult; */ } - -static void copyDeleteDataBlock(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, - SSDataBlock* pUpdateRes) { +static void generateIntervalTs(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, + SSDataBlock* pUpdateRes) { if (pDelBlock->info.rows == 0) { return; } @@ -948,7 +1009,7 @@ static void copyDeleteDataBlock(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, uint64_t* uidCol = (uint64_t*)pGpCol->pData; SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, DELETE_GROUPID_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX); for (int32_t i = pInfo->deleteDataIndex; i < pDelBlock->info.rows && i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1; @@ -969,6 +1030,43 @@ static void copyDeleteDataBlock(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, } } +static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator, + SSDataBlock* pUpdateRes) { + if (pBlock->info.rows == 0) { + return; + } + blockDataCleanup(pUpdateRes); + blockDataEnsureCapacity(pUpdateRes, pBlock->info.rows); + ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startData = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endData = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pGpCol->pData; + + SColumnInfoData* pDestStartCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDestEndCol = taosArrayGet(pUpdateRes->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX); + int32_t dummy = 0; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + uint64_t groupId = getGroupId(pOperator, uidCol[i]); + // gap must be 0. + SResultWindowInfo* pStartWin = + getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); + if (!pStartWin) { + // window has been closed. + continue; + } + SResultWindowInfo* pEndWin = + getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy); + ASSERT(pEndWin); + colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false); + colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false); + colDataAppend(pDestGpCol, i, (const char*)&groupId, false); + pUpdateRes->info.rows++; + } +} static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { blockDataCleanup(pUpdateBlock); int32_t size = taosArrayGetSize(pInfo->tsArray); @@ -1001,7 +1099,7 @@ static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlo } if (size == 0) { - copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock); + generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock); } } @@ -1060,11 +1158,17 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } break; case STREAM_DELETE_DATA: { pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - copyDataBlock(pInfo->pDeleteDataRes, pBlock); - copyDeleteDataBlock(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes); pInfo->updateResIndex = 0; - prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex); + if (isIntervalWindow(pInfo)) { + copyDataBlock(pInfo->pDeleteDataRes, pBlock); + generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes); + prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; + } else { + generateScanRange(pInfo, pBlock, pInfo->pTableScanOp, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + } pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; return pInfo->pUpdateRes; } break; @@ -1078,8 +1182,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - if (!isStateWindow(pInfo)) { + if (isStateWindow(pInfo)) { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } else { + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } return pInfo->pUpdateRes; @@ -1104,11 +1210,19 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return pInfo->pUpdateRes; } pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE) { + SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + if (pSDB) { + pSDB->info.type = STREAM_NORMAL; + checkUpdateData(pInfo, true, pSDB, false); + return pSDB; + } + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } else if (isStateWindow(pInfo)) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; if (prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) { - ASSERT(pInfo->pUpdateRes->info.rows == 0); + blockDataCleanup(pInfo->pUpdateRes); // return empty data blcok return pInfo->pUpdateRes; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index de11b06c17..37e98693cc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1323,13 +1323,13 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, } } -static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex, +static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsCols = (TSKEY*)pTsCol->pData; uint64_t* pGpDatas = NULL; if (pBlock->info.type == STREAM_RETRIEVE) { - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); pGpDatas = (uint64_t*)pGpCol->pData; } int32_t step = 0; @@ -1492,7 +1492,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pBlock, "single interval recv"); if (pBlock->info.type == STREAM_CLEAR) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, + doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; @@ -1710,7 +1710,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes)); pInfo->delIndex = 0; - // pInfo->pDelRes = createDeleteBlock(); todo(liuyao) for delete + // pInfo->pDelRes = createPullDataBlock(); todo(liuyao) for delete pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete @@ -2571,13 +2571,13 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); for (; (*pIndex) < size; (*pIndex)++) { SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); - SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 1); + SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 2); + SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); pBlock->info.rows++; } @@ -2589,9 +2589,9 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB } void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { @@ -2680,7 +2680,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); @@ -2688,7 +2688,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; - doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, + doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL); rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, NULL); @@ -2719,7 +2719,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); removeResults(pUpWins, pUpdated); taosArrayDestroy(pUpWins); if (taosArrayGetSize(pUpdated) > 0) { @@ -2901,7 +2901,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataRes = createPullDataBlock(); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; - // pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete + // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete pInfo->delIndex = 0; @@ -3046,13 +3046,14 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; - // pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete + // pInfo->pDelRes = createPullDataBlock(); pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete pInfo->pChildren = NULL; pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->returnDelete = false; pOperator->name = "StreamSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; @@ -3087,15 +3088,23 @@ int64_t getSessionWindowEndkey(void* data, int32_t index) { SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index); return pWin->win.ekey; } -static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) { - int64_t sGap = ts - pWin->win.skey; - int64_t eGap = pWin->win.ekey - ts; - if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) { + +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { + int64_t sGap = ts - pWin->skey + gap; + int64_t eGap = pWin->ekey - ts + gap; + // if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) { + // return true; + // } + if (sGap >= 0 && eGap >= 0) { return true; } return false; } +bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { + return isInTimeWindow(&pWinInfo->win, ts, gap); +} + static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; return taosArrayInsert(pWinInfos, index, &win); @@ -3118,6 +3127,41 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { return pWinInfos; } +// don't add new window +SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex) { + SArray* pWinInfos = getWinInfos(pAggSup, groupId); + pAggSup->pCurWins = pWinInfos; + + int32_t size = taosArrayGetSize(pWinInfos); + if (size == 0) { + return NULL; + } + // find the first position which is smaller than the key + int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); + SResultWindowInfo* pWin = NULL; + if (index >= 0) { + pWin = taosArrayGet(pWinInfos, index); + if (isInWindow(pWin, startTs, gap)) { + *pIndex = index; + return pWin; + } + } + + if (index + 1 < size) { + pWin = taosArrayGet(pWinInfos, index + 1); + if (isInWindow(pWin, startTs, gap)) { + *pIndex = index + 1; + return pWin; + } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { + *pIndex = index; + return pWin; + } + } + + return NULL; +} + SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) { SArray* pWinInfos = getWinInfos(pAggSup, groupId); @@ -3358,6 +3402,34 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } +void deleteWindow(SArray* pWinInfos, int32_t index) { + ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); + taosArrayRemove(pWinInfos, index); +} + +static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endDatas = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + int32_t winIndex = 0; + while(1) { + SResultWindowInfo* pCurWin = + getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); + if (!pCurWin) { + break; + } + deleteWindow(pAggSup->pCurWins, winIndex); + if (result) { + taosArrayPush(result, pCurWin); + } + } + } +} + static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); @@ -3366,13 +3438,14 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, for (int32_t i = 0; i < pBlock->info.rows; i += step) { int32_t winIndex = 0; SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex); - step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL); - ASSERT(isInWindow(pCurWin, tsCols[i], gap)); - if (pCurWin->pos.pageId == -1) { + getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex); + if (!pCurWin || pCurWin->pos.pageId == -1) { // window has been closed. + step = 1; continue; } + step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL); + ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); if (result) { taosArrayPush(result, pCurWin); @@ -3407,7 +3480,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It blockDataEnsureCapacity(pBlock, size); size_t keyLen = 0; while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); @@ -3495,7 +3568,7 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra pSeWin->isOutput = true; } if (delete) { - taosArrayRemove(pWins, i); + deleteWindow(pWins, i); i--; size = taosArrayGetSize(pWins); } @@ -3535,6 +3608,14 @@ int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ return TSDB_CODE_SUCCESS; } +static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) { + int32_t size = taosArrayGetSize(pResWins); + for (int32_t i = 0; i < size; i++) { + SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i); + taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY)); + } +} + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; @@ -3570,17 +3651,32 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, - pInfo->gap, pWins); + 0, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, 0, pChildOp->exprSupp.numOfExprs, - pChildInfo->gap, NULL); + 0, NULL); rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); } taosArrayDestroy(pWins); continue; + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + // gap must be 0 + doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins); + if (IS_FINAL_OP(pInfo)) { + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; + // gap must be 0 + doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); + rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); + } + copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + taosArrayDestroy(pWins); + continue; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); continue; @@ -3664,26 +3760,29 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); - if (pInfo->pDelRes->info.rows > 0) { + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "Semi Session"); + return pBInfo->pRes; + } + + // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) { + pInfo->returnDelete = true; printDataBlock(pInfo->pDelRes, "Semi Session"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (pInfo->pUpdateRes->info.rows == 0) { - // semi interval operator clear disk buffer - clearStreamSessionOperator(pInfo); - return NULL; - } + + if (pInfo->pUpdateRes->info.rows > 0) { // process the rest of the data pOperator->status = OP_OPENED; printDataBlock(pInfo->pUpdateRes, "Semi Session"); return pInfo->pUpdateRes; } - printDataBlock(pBInfo->pRes, "Semi Session"); - return pBInfo->pRes; + // semi interval operator clear disk buffer + clearStreamSessionOperator(pInfo); + pOperator->status = OP_EXEC_DONE; + return NULL; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -3699,11 +3798,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins); + doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, 0, pWins); removeSessionResults(pStUpdated, pWins); taosArrayDestroy(pWins); copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); break; + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { + // gap must be 0 + doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL); + copyDataBlock(pInfo->pDelRes, pBlock); + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + break; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); continue; @@ -3728,24 +3833,29 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); - if (pInfo->pDelRes->info.rows > 0) { + + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "Semi Session"); + return pBInfo->pRes; + } + + // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) { + pInfo->returnDelete = true; printDataBlock(pInfo->pDelRes, "Semi Session"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (pInfo->pUpdateRes->info.rows == 0) { - return NULL; - } + + if (pInfo->pUpdateRes->info.rows > 0) { // process the rest of the data pOperator->status = OP_OPENED; printDataBlock(pInfo->pUpdateRes, "Semi Session"); return pInfo->pUpdateRes; } - printDataBlock(pBInfo->pRes, "Semi Session"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + pOperator->status = OP_EXEC_DONE; + return NULL; } SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -3971,11 +4081,6 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, S return rows - start; } -void deleteWindow(SArray* pWinInfos, int32_t index) { - ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); - taosArrayRemove(pWinInfos, index); -} - static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol, int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); @@ -4179,7 +4284,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; - // pInfo->pDelRes = createDeleteBlock(); // todo(liuyao) for delete + // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete pInfo->pChildren = NULL; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 708ea4bd38..a412b589a9 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -549,7 +549,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index a1b0cc5947..6c0717e845 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -595,6 +595,7 @@ static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstre COPY_SCALAR_FIELD(taskId); COPY_SCALAR_FIELD(schedId); COPY_SCALAR_FIELD(execId); + COPY_SCALAR_FIELD(fetchMsgType); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0bff063ea1..2a94ee43e3 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3538,6 +3538,7 @@ static const char* jkDownstreamSourceAddr = "Addr"; static const char* jkDownstreamSourceTaskId = "TaskId"; static const char* jkDownstreamSourceSchedId = "SchedId"; static const char* jkDownstreamSourceExecId = "ExecId"; +static const char* jkDownstreamSourceFetchMsgType = "FetchMsgType"; static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; @@ -3552,6 +3553,9 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceExecId, pNode->execId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceFetchMsgType, pNode->fetchMsgType); + } return code; } @@ -3569,6 +3573,9 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkDownstreamSourceExecId, &pNode->execId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDownstreamSourceFetchMsgType, &pNode->fetchMsgType); + } return code; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index eb10a2fdd6..2b1e535e8c 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -123,6 +123,7 @@ typedef struct SQWTaskCtx { int8_t taskType; int8_t explain; int32_t queryType; + int32_t fetchType; int32_t execId; bool queryFetched; diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 75b11c1b0b..7becaf06eb 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -35,8 +35,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes); int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code); int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code); -int32_t qwBuildAndSendFetchRsp(SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, - int32_t code); +int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, STbVerInfo* tbInfo); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 8df3ac90fa..ea5aa3c563 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -104,7 +104,7 @@ int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendFetchRsp(SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { +int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { if (NULL == pRsp) { pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRsp, 0, sizeof(SRetrieveTableRsp)); @@ -112,7 +112,7 @@ int32_t qwBuildAndSendFetchRsp(SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, i } SRpcMsg rpcRsp = { - .msgType = TDMT_SCH_FETCH_RSP, + .msgType = rspType, .pCont = pRsp, .contLen = sizeof(*pRsp) + dataLength, .code = code, @@ -436,7 +436,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int64_t rId = 0; int32_t eId = msg->execId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connInfo = pMsg->info}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connInfo = pMsg->info, .msgType = pMsg->msgType}; QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->info.handle); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 5ba525329f..3f8d62b1aa 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -606,7 +606,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, @@ -628,7 +628,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; - qwBuildAndSendFetchRsp(&qwMsg->connInfo, NULL, 0, code); + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, NULL, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } @@ -661,6 +661,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + ctx->queryType = qwMsg->msgType; + SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); @@ -711,7 +713,7 @@ _return: } if (code || rsp) { - qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index bc37400249..be54db51de 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -214,7 +214,8 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { rpcFreeCont(rsp); break; } - case TDMT_SCH_FETCH_RSP: { + case TDMT_SCH_FETCH_RSP: + case TDMT_SCH_MERGE_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)pRsp->pCont; if (0 == pRsp->code && 0 == rsp->completed) { @@ -815,6 +816,7 @@ void *fetchQueueThread(void *param) { switch (fetchRpc->msgType) { case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0); break; case TDMT_SCH_CANCEL_TASK: diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 8e8652aab5..7289e4b6be 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -35,7 +35,7 @@ extern "C" { #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_TASK_MAX_EXEC_TIMES 5 +#define SCH_TASK_MAX_EXEC_TIMES 8 #define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA enum { @@ -318,6 +318,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) +#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_SRC_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) @@ -327,7 +328,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0)) -#define SCH_NEED_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH) +#define SCH_NEED_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) #define SCH_NEED_REDIRECT(_msgType, _code, _rspLen) (SCH_NEED_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_SUB_TASK_NETWORK_ERR(_code, _rspLen))) #define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_NEED_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index bf85d09e00..e1035c4fca 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -44,6 +44,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy // SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_SCH_FETCH_RSP: + case TDMT_SCH_MERGE_FETCH_RSP: if (lastMsgType != reqMsgType && -1 != lastMsgType) { SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); @@ -304,7 +305,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa } break; } - case TDMT_SCH_FETCH_RSP: { + case TDMT_SCH_FETCH_RSP: + case TDMT_SCH_MERGE_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; SCH_ERR_JRET(rspCode); @@ -558,6 +560,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_DELETE: case TDMT_SCH_EXPLAIN: case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: *fp = schHandleCallback; break; case TDMT_SCH_DROP_TASK: @@ -1016,7 +1019,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, persistHandle = true; break; } - case TDMT_SCH_FETCH: { + case TDMT_SCH_FETCH: + case TDMT_SCH_MERGE_FETCH: { msgSize = sizeof(SResFetchReq); msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index e60006d75c..45cb0ab935 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -258,7 +258,9 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .taskId = pTask->taskId, .schedId = schMgmt.sId, .execId = pTask->execId, - .addr = pTask->succeedAddr}; + .addr = pTask->succeedAddr, + .fetchMsgType = SCH_FETCH_TYPE(pTask), + }; qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &parent->lock); @@ -818,7 +820,7 @@ int32_t schLaunchFetchTask(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_SCH_FETCH)); + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask))); return TSDB_CODE_SUCCESS; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e207d89212..cc6057b031 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -648,8 +648,6 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { taosReleaseRef(tsNodeRefId, rid); @@ -657,8 +655,8 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodePropose(pSyncNode, pMsg, isWeak); + int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -669,15 +667,14 @@ int32_t syncProposeBatch(int64_t rid, SRpcMsg* pMsgArr, bool* pIsWeakArr, int32_ return -1; } - int32_t ret = 0; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); + int32_t ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 2f2038b914..f98e356540 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -449,4 +449,53 @@ if $data26 != 14 then return -1 endi +sql create database test1 vgroups 1 +sql show databases + +print $data00 $data01 $data02 + +sql use test1 + +sql create table t1(ts timestamp, a int, b int , c int, d double, id int); +sql create stream streams2 trigger at_once into streamt1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); + +sql insert into t1 values(1648791212000,2,2,3,1.0,1); +sql insert into t1 values(1648791213000,1,2,3,1.0,1); +sql insert into t1 values(1648791213000,1,2,4,1.0,2); +$loop_count = 0 +loop5: + +sleep 300 +sql select * from streamt1 order by c desc; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop5 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop5 +endi + +if $data05 != 4 then + print =====data05=$data05 + goto loop5 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop5 +endi + +if $data15 != 3 then + print =====data15=$data15 + goto loop5 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From ca3d918004b0aac5a5cdc75311c32585c8afcc3f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 09:58:23 +0800 Subject: [PATCH 04/39] fix(wal): mutex --- source/libs/wal/src/walWrite.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index d7fa6695d0..445cdea45b 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -84,6 +84,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { terrno = TSDB_CODE_WAL_INVALID_VER; + taosThreadMutexUnlock(&pWal->mutex); return -1; } @@ -92,6 +93,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // change current files code = walChangeWrite(pWal, ver); if (code < 0) { + taosThreadMutexUnlock(&pWal->mutex); return -1; } @@ -146,6 +148,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ASSERT(taosValidFile(pLogTFile)); int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalCkHead)); if (size != sizeof(SWalCkHead)) { + taosThreadMutexUnlock(&pWal->mutex); return -1; } code = walValidHeadCksum(&head); @@ -154,11 +157,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (code != 0) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; ASSERT(0); + taosThreadMutexUnlock(&pWal->mutex); return -1; } if (head.head.version != ver) { ASSERT(0); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + taosThreadMutexUnlock(&pWal->mutex); return -1; } @@ -167,12 +172,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (code < 0) { ASSERT(0); terrno = TAOS_SYSTEM_ERROR(errno); + taosThreadMutexUnlock(&pWal->mutex); return -1; } code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { ASSERT(0); terrno = TAOS_SYSTEM_ERROR(errno); + taosThreadMutexUnlock(&pWal->mutex); return -1; } pWal->vers.lastVer = ver - 1; From d9e832997510a6321aac56bdd5bfef8f59e90513 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:05:48 +0800 Subject: [PATCH 05/39] support bool tag --- source/dnode/vnode/src/meta/metaTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 74076d323c..b3cdf47302 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -92,7 +92,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const } else if (type == TSDB_DATA_TYPE_BOOL) { int val = *(int *)(&pTagVal->i64); int len = sizeof(val); - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len); + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); } if (term != NULL) { indexMultiTermAdd(terms, term); From 11a7b1e1b06e82b41f549cecf38daf900881e633 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:18:20 +0800 Subject: [PATCH 06/39] update test case --- tests/system-test/2-query/json_tag.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ae3f130b7e..24e413fd74 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -323,12 +323,12 @@ class TDTestCase: # where json value is bool tdSql.query("select * from jsons1 where jtag->'tag1'=true") tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'=false") - #tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'=false") + tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'!=false") tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'>false") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'>false") + tdSql.checkRows(0) # where json value is null tdSql.query("select * from jsons1 where jtag->'tag1'=null") From 1e7d9ada190d2aa35b990aec9580b823bfd8c1d3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 10:55:22 +0800 Subject: [PATCH 07/39] fix(stream): clean up timer --- source/dnode/vnode/src/tq/tq.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 64ef188870..7b90719e14 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -46,6 +46,7 @@ void tqCleanUp() { if (old == 1) { taosTmrCleanUp(tqMgmt.timer); + streamCleanUp(); atomic_store_8(&tqMgmt.inited, 0); } } @@ -529,19 +530,16 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->execType != TASK_EXEC__NONE) { // expand runners if (pTask->isDataScan) { - /*SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);*/ SReadHandle handle = { .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initStreamReader = 1, }; - /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - ASSERT(pTask->exec.executor); } else { pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); - ASSERT(pTask->exec.executor); } + ASSERT(pTask->exec.executor); } // sink From 061399e43ed74fd0569f759c469034c77dfdb9db Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:56:35 +0800 Subject: [PATCH 08/39] update log --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b771f6f8a..4b6ff43ba8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -573,8 +573,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), - T_REF_VAL_GET(conn)); + tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3f4a5628f9..b805aa929c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -305,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } - tError("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); + tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; if (conn->status == ConnAcquire) { From bd693eaa0287893b798abedbd732541d9b37cc04 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 7 Jul 2022 11:42:01 +0800 Subject: [PATCH 09/39] test: let case run about and or for byte compute --- tests/system-test/2-query/and_or_for_byte.py | 4 ++-- tests/system-test/fulltest.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 28d3e1cf43..78ba0e8dfa 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -230,7 +230,7 @@ class TDTestCase: work_sql += f"cast({arg} as bigint){opera}" if not agg: - work_sql+= f" from {tbname} order by ts" + work_sql+= f" from {tbname} order by tbname ,ts" else: work_sql+= f" from {tbname} " tdSql.query(work_sql) @@ -243,7 +243,7 @@ class TDTestCase: else: origin_sql += f"cast({arg} as bigint)," if not agg: - origin_sql+= f" from {tbname} order by ts" + origin_sql+= f" from {tbname} order by tbname ,ts" else: origin_sql+= f" from {tbname} " tdSql.query(origin_sql) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b91b556cc..67d2065d2c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -117,7 +117,7 @@ python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 2-query/irate.py -#python3 ./test.py -f 2-query/and_or_for_byte.py +python3 ./test.py -f 2-query/and_or_for_byte.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py From 1ec57798d2b17e16743ea904c256e237816384b5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 11:42:45 +0800 Subject: [PATCH 10/39] ci: remove unstable case --- include/libs/wal/wal.h | 44 ++++++----- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/libs/sync/inc/syncRaftLog.h | 4 +- source/libs/sync/src/syncRaftLog.c | 12 +-- source/libs/wal/src/walRead.c | 110 +++++++++------------------ source/libs/wal/test/walMetaTest.cpp | 14 ++-- tests/system-test/failed.txt | 1 + tests/system-test/fulltest.sh | 2 +- 10 files changed, 79 insertions(+), 115 deletions(-) create mode 100644 tests/system-test/failed.txt diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index e32a8d1055..9a3998ed57 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -88,7 +88,7 @@ typedef struct { EWalType level; // wal level } SWalCfg; -typedef struct SWalVer { +typedef struct { int64_t firstVer; int64_t verInSnapshotting; int64_t snapshotVer; @@ -149,17 +149,22 @@ typedef struct SWal { SWalCkHead writeHead; } SWal; // WAL HANDLE -typedef struct SWalReadHandle { - SWal *pWal; - TdFilePtr pReadLogTFile; - TdFilePtr pReadIdxTFile; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t capacity; - int64_t status; // if cursor valid - TdThreadMutex mutex; - SWalCkHead *pHead; -} SWalReadHandle; +typedef struct { + int8_t scanUncommited; + int8_t scanMeta; +} SWalFilterCond; + +typedef struct { + SWal *pWal; + TdFilePtr pLogFile; + TdFilePtr pIdxFile; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + TdThreadMutex mutex; + SWalFilterCond cond; + SWalCkHead *pHead; +} SWalReader; // module initialization int32_t walInit(); @@ -186,15 +191,16 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); // read -SWalReadHandle *walOpenReadHandle(SWal *); -void walCloseReadHandle(SWalReadHandle *); -int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); +void walCloseReader(SWalReader *pRead); +int32_t walReadVer(SWalReader *pRead, int64_t ver); +int32_t walNextValidMsg(SWalReader *pRead, SWalCkHead **ppHead); // only for tq usage -void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity); -int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead); -int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead); -int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead); +void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); +int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); +int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); +int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); typedef struct { int64_t refId; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index d10935c022..6179dd94c8 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -128,7 +128,7 @@ typedef struct { int8_t fetchMeta; // reader - SWalReadHandle* pWalReader; + SWalReader* pWalReader; // push STqPushHandle pushHandle; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7b90719e14..2dd80c5d52 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -149,7 +149,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con SEncoder encoder; tEncoderInit(&encoder, abuf, len); tEncodeSMqDataRsp(&encoder, pRsp); - /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/ SRpcMsg rsp = { .info = pMsg->info, @@ -447,7 +446,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->execHandle.subType = req.subType; pHandle->fetchMeta = req.withMeta; - pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); + pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); for (int32_t i = 0; i < 5; i++) { pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d6ec955f1e..c7e9c8eed6 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -77,7 +77,7 @@ int32_t tqMetaOpen(STQ* pTq) { STqHandle handle; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecodeSTqHandle(&decoder, &handle); - handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal); + handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); for (int32_t i = 0; i < 5; i++) { handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); } diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index f3ed9e302b..65ec77e38f 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -32,8 +32,8 @@ typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; - TdThreadMutex mutex; - SWalReadHandle* pWalHandle; + TdThreadMutex mutex; + SWalReader* pWalHandle; // SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 83495e7486..c5d339b08f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -62,7 +62,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { ASSERT(pData->pWal != NULL); taosThreadMutexInit(&(pData->mutex), NULL); - pData->pWalHandle = walOpenReadHandle(pData->pWal); + pData->pWalHandle = walOpenReader(pData->pWal, NULL); ASSERT(pData->pWalHandle != NULL); pLogStore->appendEntry = logStoreAppendEntry; @@ -95,7 +95,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) { taosThreadMutexLock(&(pData->mutex)); if (pData->pWalHandle != NULL) { - walCloseReadHandle(pData->pWalHandle); + walCloseReader(pData->pWalHandle); pData->pWalHandle = NULL; } taosThreadMutexUnlock(&(pData->mutex)); @@ -255,7 +255,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, *ppEntry = NULL; // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - SWalReadHandle* pWalHandle = pData->pWalHandle; + SWalReader* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; @@ -263,7 +263,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, taosThreadMutexLock(&(pData->mutex)); - code = walReadWithHandle(pWalHandle, index); + code = walReadVer(pWalHandle, index); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); @@ -398,10 +398,10 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { taosThreadMutexLock(&(pData->mutex)); // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - SWalReadHandle* pWalHandle = pData->pWalHandle; + SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); - int32_t code = walReadWithHandle(pWalHandle, index); + int32_t code = walReadVer(pWalHandle, index); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e7f0b31ccc..1967a75ce6 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -16,20 +16,20 @@ #include "taoserror.h" #include "walInt.h" -SWalReadHandle *walOpenReadHandle(SWal *pWal) { - SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle)); +SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { + SWalReader *pRead = taosMemoryMalloc(sizeof(SWalReader)); if (pRead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pRead->pWal = pWal; - pRead->pReadIdxTFile = NULL; - pRead->pReadLogTFile = NULL; + pRead->pIdxFile = NULL; + pRead->pLogFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; - pRead->status = 0; + pRead->cond = *cond; taosThreadMutexInit(&pRead->mutex, NULL); @@ -42,23 +42,24 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { return pRead; } -void walCloseReadHandle(SWalReadHandle *pRead) { - taosCloseFile(&pRead->pReadIdxTFile); - taosCloseFile(&pRead->pReadLogTFile); +void walCloseReader(SWalReader *pRead) { + taosCloseFile(&pRead->pIdxFile); + taosCloseFile(&pRead->pLogFile); taosMemoryFreeClear(pRead->pHead); taosMemoryFree(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { - // TODO +int32_t walNextValidMsg(SWalReader *pRead, SWalCkHead **ppHead) { + // + return 0; } -static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { +static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; - TdFilePtr pIdxTFile = pRead->pReadIdxTFile; - TdFilePtr pLogTFile = pRead->pReadLogTFile; + TdFilePtr pIdxTFile = pRead->pIdxFile; + TdFilePtr pLogTFile = pRead->pLogFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); @@ -90,11 +91,11 @@ static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i return ret; } -static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { +static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - taosCloseFile(&pRead->pReadIdxTFile); - taosCloseFile(&pRead->pReadLogTFile); + taosCloseFile(&pRead->pIdxFile); + taosCloseFile(&pRead->pLogFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); @@ -104,7 +105,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } - pRead->pReadLogTFile = pLogTFile; + pRead->pLogFile = pLogTFile; walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); @@ -114,11 +115,11 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } - pRead->pReadIdxTFile = pIdxTFile; + pRead->pIdxFile = pIdxTFile; return 0; } -static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { +static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { SWal *pWal = pRead->pWal; if (ver == pRead->curVersion) { return 0; @@ -153,9 +154,9 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } -void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; } +void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity = capacity; } -int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) { +int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { int64_t code; // TODO: valid ver @@ -168,9 +169,9 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) { if (code < 0) return -1; } - ASSERT(taosValidFile(pRead->pReadLogTFile) == true); + ASSERT(taosValidFile(pRead->pLogFile) == true); - code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalCkHead)); + code = taosReadFile(pRead->pLogFile, pHead, sizeof(SWalCkHead)); if (code != sizeof(SWalCkHead)) { return -1; } @@ -186,12 +187,12 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) { return 0; } -int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) { +int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { int64_t code; ASSERT(pRead->curVersion == pHead->head.version); - code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR); + code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); pRead->curVersion = -1; @@ -203,7 +204,7 @@ int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) { return 0; } -int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) { +int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { SWalCont *pReadHead = &((*ppHead)->head); int64_t ver = pReadHead->version; @@ -218,7 +219,7 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) { pRead->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { ASSERT(0); return -1; } @@ -241,9 +242,9 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) { return 0; } -int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHead) { +int32_t walReadWithHandle_s(SWalReader *pRead, int64_t ver, SWalCont **ppHead) { taosThreadMutexLock(&pRead->mutex); - if (walReadWithHandle(pRead, ver) < 0) { + if (walReadVer(pRead, ver) < 0) { taosThreadMutexUnlock(&pRead->mutex); return -1; } @@ -257,7 +258,7 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHea return 0; } -int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { +int32_t walReadVer(SWalReader *pRead, int64_t ver) { int64_t code; if (pRead->pWal->vers.firstVer == -1) { @@ -280,9 +281,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } - ASSERT(taosValidFile(pRead->pReadLogTFile) == true); + ASSERT(taosValidFile(pRead->pLogFile) == true); - code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalCkHead)); + code = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead)); if (code != sizeof(SWalCkHead)) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); @@ -310,7 +311,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->capacity = pRead->pHead->head.bodyLen; } - if ((code = taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) != + if ((code = taosReadFile(pRead->pLogFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) != pRead->pHead->head.bodyLen) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); @@ -340,46 +341,3 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return 0; } - -#if 0 -int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { - int code; - code = walSeekVer(pWal, ver); - if (code != 0) { - return code; - } - if (*ppHead == NULL) { - void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead)); - if (ptr == NULL) { - return -1; - } - *ppHead = ptr; - } - if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { - return -1; - } - // TODO: endian compatibility processing after read - if (walValidHeadCksum(*ppHead) != 0) { - return -1; - } - void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); - if (ptr == NULL) { - taosMemoryFree(*ppHead); - *ppHead = NULL; - return -1; - } - if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { - return -1; - } - // TODO: endian compatibility processing after read - if (walValidBodyCksum(*ppHead) != 0) { - return -1; - } - - return 0; -} - -int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { -return 0; -} -#endif diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 89c4fd9ef2..97b9852016 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -292,8 +292,8 @@ TEST_F(WalCleanDeleteEnv, roll) { TEST_F(WalKeepEnv, readHandleRead) { walResetEnv(); - int code; - SWalReadHandle* pRead = walOpenReadHandle(pWal); + int code; + SWalReader* pRead = walOpenReader(pWal, NULL); ASSERT(pRead != NULL); int i; @@ -306,7 +306,7 @@ TEST_F(WalKeepEnv, readHandleRead) { } for (int i = 0; i < 1000; i++) { int ver = taosRand() % 100; - code = walReadWithHandle(pRead, ver); + code = walReadVer(pRead, ver); ASSERT_EQ(code, 0); // printf("rrbody: \n"); @@ -325,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } - walCloseReadHandle(pRead); + walCloseReader(pRead); } TEST_F(WalRetentionEnv, repairMeta1) { @@ -354,12 +354,12 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT_EQ(pWal->vers.lastVer, 99); - SWalReadHandle* pRead = walOpenReadHandle(pWal); + SWalReader* pRead = walOpenReader(pWal, NULL); ASSERT(pRead != NULL); for (int i = 0; i < 1000; i++) { int ver = taosRand() % 100; - code = walReadWithHandle(pRead, ver); + code = walReadVer(pRead, ver); ASSERT_EQ(code, 0); // printf("rrbody: \n"); @@ -389,7 +389,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { for (int i = 0; i < 1000; i++) { int ver = taosRand() % 200; - code = walReadWithHandle(pRead, ver); + code = walReadVer(pRead, ver); ASSERT_EQ(code, 0); // printf("rrbody: \n"); diff --git a/tests/system-test/failed.txt b/tests/system-test/failed.txt new file mode 100644 index 0000000000..d0b66b1769 --- /dev/null +++ b/tests/system-test/failed.txt @@ -0,0 +1 @@ +#python3 ./test.py -f 2-query/last.py -Q 3 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b91b556cc..1e305f2518 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -295,7 +295,7 @@ python3 ./test.py -f 2-query/Today.py -Q 3 python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 -python3 ./test.py -f 2-query/last.py -Q 3 +#python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 python3 ./test.py -f 2-query/To_iso8601.py -Q 3 python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 From ee6343379cd0fb98f7bb1f64b2ad5f5bf1a2832c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 7 Jul 2022 11:04:40 +0530 Subject: [PATCH 11/39] fix(query): fix timezone parsing for /usr/share/localtime TD-16413 TD-16441 --- source/os/src/osTimezone.c | 58 ++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 724f81c66c..2ce2033a00 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -857,19 +857,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { return; } buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { + + char *zi = strstr(buf, "zoneinfo"); + if (!zi) { printf("parsing /etc/localtime failed"); return; } + tz = zi + strlen("zoneinfo") + 1; + + //for (int i = n - 1; i >= 0; --i) { + // if (buf[i] == '/') { + // if (tz) { + // tz = buf + i + 1; + // break; + // } + // tz = buf + i + 1; + // } + //} + //if (!tz || 0 == strchr(tz, '/')) { + // printf("parsing /etc/localtime failed"); + // return; + //} setenv("TZ", tz, 1); tzset(); @@ -900,7 +908,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { int n = readlink("/etc/localtime", buf, sizeof(buf)); if (n < 0) { printf("read /etc/localtime error, reason:%s", strerror(errno)); - + if (taosCheckExistFile("/etc/timezone")) { /* * NOTE: do not remove it. @@ -962,19 +970,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { return; } buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { + + char *zi = strstr(buf, "zoneinfo"); + if (!zi) { printf("parsing /etc/localtime failed"); return; } + tz = zi + strlen("zoneinfo") + 1; + + //for (int i = n - 1; i >= 0; --i) { + // if (buf[i] == '/') { + // if (tz) { + // tz = buf + i + 1; + // break; + // } + // tz = buf + i + 1; + // } + //} + //if (!tz || 0 == strchr(tz, '/')) { + // printf("parsing /etc/localtime failed"); + // return; + //} setenv("TZ", tz, 1); tzset(); From 4a648d711fdf59958e62a489eaacad37467ddade Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 13:56:47 +0800 Subject: [PATCH 12/39] feat(wal): support filter --- include/common/tcommon.h | 12 ++++ include/libs/wal/wal.h | 2 +- source/dnode/vnode/src/inc/tq.h | 9 --- source/libs/wal/inc/walInt.h | 1 + source/libs/wal/src/walRead.c | 123 ++++++++++++++++++++++++++++++-- 5 files changed, 132 insertions(+), 15 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 083ded8887..fd4ed6b180 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -32,6 +32,18 @@ enum { TMQ_CONF__RESET_OFFSET__LATEST = -1, }; +// clang-format off +#define IS_META_MSG(x) ( \ + x == TDMT_VND_CREATE_STB \ + || x == TDMT_VND_ALTER_STB \ + || x == TDMT_VND_DROP_STB \ + || x == TDMT_VND_CREATE_TABLE \ + || x == TDMT_VND_ALTER_TABLE \ + || x == TDMT_VND_DROP_TABLE \ + || x == TDMT_VND_DROP_TTL_TABLE \ +) +// clang-format on + enum { TMQ_MSG_TYPE__DUMMY = 0, TMQ_MSG_TYPE__POLL_RSP, diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 9a3998ed57..43792b5415 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -194,7 +194,7 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver); SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); int32_t walReadVer(SWalReader *pRead, int64_t ver); -int32_t walNextValidMsg(SWalReader *pRead, SWalCkHead **ppHead); +int32_t walNextValidMsg(SWalReader *pRead); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 6179dd94c8..12fb500ba3 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -40,15 +40,6 @@ extern "C" { #define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) -#define IS_META_MSG(x) ( \ - x == TDMT_VND_CREATE_STB \ - || x == TDMT_VND_ALTER_STB \ - || x == TDMT_VND_DROP_STB \ - || x == TDMT_VND_CREATE_TABLE \ - || x == TDMT_VND_ALTER_TABLE \ - || x == TDMT_VND_DROP_TABLE \ - || x == TDMT_VND_DROP_TTL_TABLE \ -) // clang-format on typedef struct STqOffsetStore STqOffsetStore; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index c23d0802c1..2767780ff3 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -19,6 +19,7 @@ #include "taoserror.h" #include "tchecksum.h" #include "tcoding.h" +#include "tcommon.h" #include "tcompare.h" #include "wal.h" diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1967a75ce6..b5c75ce3c4 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -16,6 +16,10 @@ #include "taoserror.h" #include "walInt.h" +static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer); +static int32_t walFetchBodyNew(SWalReader *pRead); +static int32_t walSkipFetchBodyNew(SWalReader *pRead); + SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { SWalReader *pRead = taosMemoryMalloc(sizeof(SWalReader)); if (pRead == NULL) { @@ -29,7 +33,12 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; - pRead->cond = *cond; + if (cond) + pRead->cond = *cond; + else { + pRead->cond.scanMeta = 0; + pRead->cond.scanUncommited = 0; + } taosThreadMutexInit(&pRead->mutex, NULL); @@ -39,6 +48,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { taosMemoryFree(pRead); return NULL; } + return pRead; } @@ -49,10 +59,28 @@ void walCloseReader(SWalReader *pRead) { taosMemoryFree(pRead); } -int32_t walNextValidMsg(SWalReader *pRead, SWalCkHead **ppHead) { - // - - return 0; +int32_t walNextValidMsg(SWalReader *pRead) { + int64_t fetchVer = pRead->curVersion; + int64_t endVer = pRead->cond.scanUncommited ? walGetLastVer(pRead->pWal) : walGetCommittedVer(pRead->pWal); + while (fetchVer <= endVer) { + if (walFetchHeadNew(pRead, fetchVer) < 0) { + return -1; + } + if (pRead->pHead->head.msgType == TDMT_VND_SUBMIT || + (IS_META_MSG(pRead->pHead->head.msgType) && pRead->cond.scanMeta)) { + if (walFetchBodyNew(pRead) < 0) { + return -1; + } + return 0; + } else { + if (walSkipFetchBodyNew(pRead) < 0) { + return -1; + } + fetchVer++; + ASSERT(fetchVer == pRead->curVersion); + } + } + return -1; } static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64_t ver) { @@ -156,6 +184,91 @@ static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity = capacity; } +static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { + int64_t contLen; + if (pRead->curVersion != fetchVer) { + if (walReadSeekVer(pRead, fetchVer) < 0) return -1; + } + contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead)); + if (contLen != sizeof(SWalCkHead)) { + if (contLen < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } + pRead->curVersion = -1; + return -1; + } + return 0; +} + +static int32_t walFetchBodyNew(SWalReader *pRead) { + SWalCont *pReadHead = &pRead->pHead->head; + int64_t ver = pReadHead->version; + + if (pRead->capacity < pReadHead->bodyLen) { + void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + return -1; + } + pRead->pHead = ptr; + pReadHead = &pRead->pHead->head; + pRead->capacity = pReadHead->bodyLen; + } + + if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since %s", + pRead->pHead->head.version, ver, tstrerror(terrno)); + } else { + wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since file corrupted", + pRead->pHead->head.version, ver); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } + pRead->curVersion = -1; + ASSERT(0); + return -1; + } + + if (pReadHead->version != ver) { + wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + return -1; + } + + if (walValidBodyCksum(pRead->pHead) != 0) { + wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + return -1; + } + + pRead->curVersion = ver + 1; + return 0; +} + +static int32_t walSkipFetchBodyNew(SWalReader *pRead) { + int64_t code; + + ASSERT(pRead->curVersion == pRead->pHead->head.version); + + code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + pRead->curVersion = -1; + return -1; + } + + pRead->curVersion++; + + return 0; +} + int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { int64_t code; From 3f2ce5985b75b1fd4dfcb89049823f3d3fe055b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 06:12:56 +0000 Subject: [PATCH 13/39] fix: delete file commit bug --- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d498fa71ab..4a33dab08c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -520,7 +520,7 @@ static int32_t tsdbScanAndTryFixFS(STsdbFS *pFS, int8_t deepScan) { return code; _err: - tsdbError("vgId:%d tsdb can and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c22d1a4064..2bd62348c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -49,7 +49,7 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb } pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.size = 0; + pDelFWriter->fDel.offset = 0; *ppWriter = pDelFWriter; return code; From 3c9119b9905a6b52b443bdbe1f897d20fde633da Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Jul 2022 13:49:36 +0800 Subject: [PATCH 14/39] ci(stream): stream session ci --- tests/script/tsim/stream/session0.sim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 24ffe76d2b..eb440d78c4 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -141,7 +141,7 @@ if $data01 != 7 then goto loop1 endi -if $data02 != 9 then +if $data02 != 18 then print =====data02=$data02 goto loop1 endi @@ -151,22 +151,22 @@ if $data03 != 4 then goto loop1 endi -if $data04 != 1.100000000 then +if $data04 != 1.000000000 then print ======$data04 return -1 endi -if $data05 != 0.816496581 then +if $data05 != 1.154700538 then print ======$data05 return -1 endi -if $data06 != 3 then +if $data06 != 4 then print ======$data06 return -1 endi -if $data07 != 1.100000000 then +if $data07 != 1.000000000 then print ======$data07 return -1 endi @@ -235,7 +235,7 @@ sql create stream streams4 trigger at_once watermark 1d into streamt4 as select # sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s); # sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s); sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), hyperloglog(a) from t1 session(ts,10s); -sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); +# sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); sql insert into t1 values(1648791213001,1,1,1,1.0); sql insert into t1 values(1648791213002,2,3,2,3.4); sql insert into t1 values(1648791213003,4,9,3,4.8); @@ -288,10 +288,10 @@ if $rows == 0 then goto loop3 endi -sql select * from streamt8; -if $rows == 0 then - print ======$rows - goto loop3 -endi +#sql select * from streamt8; +#if $rows == 0 then +# print ======$rows +# goto loop3 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From cddc989808793b4b62ff6797f510d685e72cada0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 7 Jul 2022 14:30:08 +0800 Subject: [PATCH 15/39] feat: add filter to fill operator --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 42 +++++++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8994c9d192..514620fdf7 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -528,6 +528,7 @@ typedef struct SFillOperatorInfo { SSDataBlock* existNewGroupBlock; bool multigroupResult; STimeWindow win; + SNode* pCondition; } SFillOperatorInfo; typedef struct SGroupbyOperatorInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3bf6c9fa0a..8742b5830b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3364,18 +3364,13 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf } } -static SSDataBlock* doFill(SOperatorInfo* pOperator) { +static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultInfo* pResultInfo = &pOperator->resultInfo; SSDataBlock* pResBlock = pInfo->pRes; - blockDataCleanup(pResBlock); - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - // todo handle different group data interpolation bool n = false; bool* newgroup = &n; @@ -3440,6 +3435,40 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } } +static SSDataBlock* doFill(SOperatorInfo* pOperator) { + SFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SResultInfo* pResultInfo = &pOperator->resultInfo; + SSDataBlock* pResBlock = pInfo->pRes; + + blockDataCleanup(pResBlock); + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (true) { + SSDataBlock* fillResult = doFillImpl(pOperator); + if (fillResult != NULL) { + doFilter(pInfo->pCondition, fillResult); + } + + if (fillResult == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + if (fillResult->info.rows > 0) { + break; + } + } + + size_t rows = pResBlock->info.rows; + pOperator->resultInfo.totalRows += rows; + + return (rows == 0)? NULL:pResBlock; +} + static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { for (int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExprInfo = &pExpr[i]; @@ -3958,6 +3987,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pInfo->pRes = pResBlock; pInfo->multigroupResult = multigroupResult; + pInfo->pCondition = pPhyFillNode->node.pConditions; pOperator->name = "FillOperator"; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; From 124125677f6199defd1adc2c6c398ecebbc24e6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 06:41:05 +0000 Subject: [PATCH 16/39] fix: another delete file commit bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6489a1b955..b60a4697fa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -168,8 +168,6 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel tb_uid_t suid; tb_uid_t uid; - taosArrayClear(pCommitter->aDelData); - if (pTbData) { suid = pTbData->suid; uid = pTbData->uid; @@ -185,6 +183,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL); if (code) goto _err; + } else { + taosArrayClear(pCommitter->aDelData); } if (pTbData == NULL && pDelIdx == NULL) goto _exit; @@ -205,7 +205,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel if (code) goto _err; // put delIdx - if (taosArrayPush(pCommitter->aDelIdx, &delIdx) == NULL) { + if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } From 9d0c772b40a4d84cba9b1b826317801ed9f987fd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Jul 2022 14:50:37 +0800 Subject: [PATCH 17/39] test: valgrind case --- tests/script/tsim/valgrind/basic1.sim | 26 +++++----------------- tests/script/tsim/valgrind/basic2.sim | 6 ++++- tests/script/tsim/valgrind/checkError1.sim | 2 +- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 1827e8d3c0..69aea5f024 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 +#system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect @@ -19,43 +19,29 @@ if $rows != 1 then return -1 endi -print =============== step2: create alter drop show user -sql create user u1 pass 'taosdata' -sql show users -sql alter user u1 sysinfo 1 -sql alter user u1 enable 1 -sql alter user u1 pass 'taosdata' -sql drop user u1 -sql_error alter user u2 sysinfo 0 - -print =============== step3: create drop dnode -sql create dnode $hostname port 7200 -sql drop dnode 2 -sql alter dnode 1 'debugflag 143' - -print =============== step4: create show database +print =============== step2: create show database sql create database d1 vgroups 1 buffer 3 sql show databases sql use d1 sql show vgroups -print =============== step5: create show stable +print =============== step3: create show stable sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) sql show stables if $rows != 1 then return -1 endi -print =============== step6: create show table +print =============== step4: create show table sql create table ct1 using stb tags(1000) sql show tables if $rows != 1 then return -1 endi -print =============== step7: insert data +print =============== step5: insert data -print =============== step7: select data +print =============== step6: select data _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index a592d48d66..157ac002e6 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -19,10 +19,14 @@ if $rows != 1 then return -1 endi -print =============== step2: create db +print =============== step2 sql create database db vgroups 1 buffer 3 sql use db sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table ct1 using stb tags(1000) +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index edbb28de4b..e3ac537b39 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -32,7 +32,7 @@ sql_error alter user u2 sysinfo 0 print =============== step3: create drop dnode sql create dnode $hostname port 7200 sql drop dnode 2 -sql alter dnode 1 'debugflag 143' +sql alter dnode 1 'debugflag 131' print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT From 4288ab412eb0510cf371121829ac950acf3b747d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 7 Jul 2022 14:58:59 +0800 Subject: [PATCH 18/39] fix: fix tsdb cache del index reading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index dc6fc36035..2086d94099 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -309,9 +309,11 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t if (code) goto _err; // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); - pDelIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); + SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); if (code) goto _err; + *pDelIdx = *pIdx; + if (pDelIdxArray) { taosArrayDestroy(pDelIdxArray); } From 9eb4f16792089bed159f07f26a4eb3394d6a4f84 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 7 Jul 2022 14:59:52 +0800 Subject: [PATCH 19/39] feat: add filter to fill operator --- source/libs/executor/src/executorimpl.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8742b5830b..181ec499ec 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3371,6 +3371,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SResultInfo* pResultInfo = &pOperator->resultInfo; SSDataBlock* pResBlock = pInfo->pRes; + blockDataCleanup(pResBlock); + // todo handle different group data interpolation bool n = false; bool* newgroup = &n; @@ -3439,16 +3441,13 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SResultInfo* pResultInfo = &pOperator->resultInfo; - SSDataBlock* pResBlock = pInfo->pRes; - - blockDataCleanup(pResBlock); if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SSDataBlock* fillResult = NULL; while (true) { - SSDataBlock* fillResult = doFillImpl(pOperator); + fillResult = doFillImpl(pOperator); if (fillResult != NULL) { doFilter(pInfo->pCondition, fillResult); } @@ -3463,10 +3462,12 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } } - size_t rows = pResBlock->info.rows; - pOperator->resultInfo.totalRows += rows; + if (fillResult != NULL) { + size_t rows = fillResult->info.rows; + pOperator->resultInfo.totalRows += rows; + } - return (rows == 0)? NULL:pResBlock; + return fillResult; } static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { From 4482b2dda758d5b4cc0b4236299f607983231100 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 7 Jul 2022 15:26:29 +0800 Subject: [PATCH 20/39] feat: add doFilter to indef operator --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 514620fdf7..7515f3e4de 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -518,6 +518,7 @@ typedef struct SIndefOperatorInfo { SAggSupporter aggSup; SArray* pPseudoColInfo; SExprSupp scalarSup; + SNode* pCondition; } SIndefOperatorInfo; typedef struct SFillOperatorInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 181ec499ec..5d92adfaaa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3862,6 +3862,8 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } + doFilter(pIndefInfo->pCondition, pInfo->pRes); + size_t rows = pInfo->pRes->info.rows; pOperator->resultInfo.totalRows += rows; @@ -3915,6 +3917,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); + pInfo->pCondition = pPhyNode->node.pConditions; pOperator->name = "IndefinitOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; From a09d46247856b6b3c8a78fbaf483a685e91c521b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 7 Jul 2022 16:04:12 +0800 Subject: [PATCH 21/39] fix: resolve valgrind memory issues --- source/dnode/vnode/src/meta/metaQuery.c | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a198f52bee..db959a83b0 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -439,6 +439,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv tDecoderInit(&dc, pData, nData); tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecoderClear(&dc); + tdbFree(pData); // convert STSchemaBuilder sb = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a50125cc4b..9381f673d8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -105,7 +105,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI // check if table exists (todo: refact) SMetaReader mr = {0}; - SMetaEntry me = {0}; + // SMetaEntry me = {0}; metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) { metaReaderClear(&mr); @@ -117,6 +117,8 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schemaRow.version; } else { + tDecoderClear(&mr.coder); + metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid); sverNew = mr.me.stbEntry.schemaRow.version; } From 0691c55e2e3e52a2bacc91e871e5bc71fefaf8bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Jul 2022 16:33:32 +0800 Subject: [PATCH 22/39] test: valgrind case --- tests/script/tsim/user/basic.sim | 2 +- tests/script/tsim/valgrind/basic1.sim | 2 +- tests/script/tsim/valgrind/checkError1.sim | 8 +- tests/script/tsim/valgrind/checkError2.sim | 7 +- tests/script/tsim/valgrind/checkError3.sim | 156 +++++++++++++++++++++ 5 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 tests/script/tsim/valgrind/checkError3.sim diff --git a/tests/script/tsim/user/basic.sim b/tests/script/tsim/user/basic.sim index 85d5f8375e..298e0cf6e3 100644 --- a/tests/script/tsim/user/basic.sim +++ b/tests/script/tsim/user/basic.sim @@ -22,7 +22,7 @@ sql_error ALTER USER root SYSINFO 1 sql_error ALTER USER root enable 0 sql_error ALTER USER root enable 1 -sql_error create database db vgroups 1; +#sql_error create database db vgroups 1; sql_error GRANT read ON db.* to root; sql_error GRANT read ON *.* to root; sql_error REVOKE read ON db.* from root; diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 69aea5f024..7d8991c095 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -#system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index e3ac537b39..62653bbdd3 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -5,20 +5,22 @@ system sh/exec.sh -n dnode1 -s start -v sql connect print =============== step1: show dnodes - $x = 0 step1: $x = $x + 1 sleep 1000 if $x == 10 then - print ----> dnode not ready! + print ---> dnode not ready! return -1 endi sql show dnodes -print ----> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data00 $data01 $data02 $data03 $data04 $data05 if $rows != 1 then return -1 endi +if $data(1)[4] != ready then + goto step1 +endi print =============== step2: create alter drop show user sql create user u1 pass 'taosdata' diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 0cb5ea3c25..eabe20b875 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -10,14 +10,17 @@ step1: $x = $x + 1 sleep 1000 if $x == 10 then - print ----> dnode not ready! + print ---> dnode not ready! return -1 endi sql show dnodes -print ----> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data00 $data01 $data02 $data03 $data04 $data05 if $rows != 1 then return -1 endi +if $data(1)[4] != ready then + goto step1 +endi print =============== step2: create db sql create database d1 vgroups 1 buffer 3 diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim new file mode 100644 index 0000000000..6bf82b4ef4 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -0,0 +1,156 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode2 -s start -v +sql connect + +print =============== add dnode2 into cluster +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== create database, stable, table +sql create database db vgroups 3 +sql use db +sql create table stb (ts timestamp, c int) tags (t int) +sql create table t0 using stb tags (0) +sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); + +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi + +sql show databases +if $rows != 3 then + return -1 +endi + +sql show stables +if $rows != 1 then + return -1 +endi + +sql show tables +if $rows != 2 then + return -1 +endi + +sql show users +if $rows != 1 then + return -1 +endi + +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi + +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_databases +if $rows != 3 then + return -1 +endi + +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_tables +if $rows != 31 then + return -1 +endi + +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +sql show variables; +if $rows != 4 then + return -1 +endi + +sql show dnode 1 variables; +if $rows <= 0 then + return -1 +endi + +sql show local variables; +if $rows <= 0 then + return -1 +endi + +print ==== stop dnode1 and dnode2, and restart dnodes +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +print =============== check dnode1 +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 + +print =============== check dnode2 +system_content sh/checkValgrind.sh -n dnode2 +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 \ No newline at end of file From 1c3c270307cc9df5abbc9ed68f27605c6de3596f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 7 Jul 2022 17:00:01 +0800 Subject: [PATCH 23/39] fix(sync): snapshot strategy wal first --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 + source/libs/sync/inc/syncInt.h | 1 - source/libs/sync/src/syncAppendEntries.c | 17 +++---- source/libs/sync/src/syncAppendEntriesReply.c | 48 +++++++++++++++++-- source/libs/sync/src/syncMain.c | 19 ++++---- source/libs/sync/src/syncMessage.c | 2 +- source/libs/sync/src/syncRaftCfg.c | 8 ++-- source/libs/sync/src/syncReplication.c | 19 +++++++- tests/script/tsim/sync/vnodesnapshot.sim | 4 +- 10 files changed, 91 insertions(+), 31 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 59d68b2110..f4ff64520e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -237,10 +237,12 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1f7347203d..c81f77fa56 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -376,10 +376,12 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8936cd6ed9..9742f93824 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -67,7 +67,6 @@ typedef struct SSyncNode { char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; char configPath[TSDB_FILENAME_LEN * 2]; - int32_t batchSize; // sync io SWal* pWal; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 885ab1acae..2d511b6003 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -842,8 +842,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + "recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -876,7 +876,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); - syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); } // fsync once @@ -931,8 +931,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, not match, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + "recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -976,8 +976,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -999,7 +1000,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); - syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); } // fsync once diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index e4724fc999..3f19b6759e 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -174,8 +174,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SyncIndex newNextIndex = pMsg->matchIndex + 1; SyncIndex newMatchIndex = pMsg->matchIndex; - if (ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex) && - ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex - 1)) { + bool needStartSnapshot = false; + if (newMatchIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, newMatchIndex)) { + needStartSnapshot = true; + } + + if (!needStartSnapshot) { // update next-index, match-index syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); @@ -197,15 +201,36 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); } + // event log, update next-index + do { + char host[64]; + int16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, + host, port); + syncNodeEventLog(ths, logBuf); + + } while (0); + } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); if (nextIndex > SYNC_INDEX_BEGIN) { --nextIndex; - if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex) && - ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) { + bool needStartSnapshot = false; + if (nextIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex)) { + needStartSnapshot = true; + } + if (nextIndex - 1 >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) { + needStartSnapshot = true; + } + + if (!needStartSnapshot) { // do nothing + } else { SSyncRaftEntry* pEntry; int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry); @@ -227,6 +252,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie nextIndex = SYNC_INDEX_BEGIN; } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + + // event log, update next-index + do { + char host[64]; + int16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + + SyncIndex newNextIndex = nextIndex; + SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, + host, port); + syncNodeEventLog(ths, logBuf); + + } while (0); } return 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e207d89212..5951078822 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -648,8 +648,6 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { taosReleaseRef(tsNodeRefId, rid); @@ -657,8 +655,8 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodePropose(pSyncNode, pMsg, isWeak); + int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -669,15 +667,14 @@ int32_t syncProposeBatch(int64_t rid, SRpcMsg* pMsgArr, bool* pIsWeakArr, int32_ return -1; } - int32_t ret = 0; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); + int32_t ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -1529,12 +1526,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "strategy:%d, batch:%d, " "replica-num:%d, " "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pSyncNode->restoreFinish, printStr); + pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, + pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, + pSyncNode->restoreFinish, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1546,12 +1545,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "strategy:%d, batch:%d, " "replica-num:%d, " "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pSyncNode->restoreFinish, printStr); + pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, + pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, + pSyncNode->restoreFinish, printStr); } else { snprintf(s, len, "%s", str); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index e53b9d5e36..697c25f304 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1605,7 +1605,7 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) { ASSERT(entryPArr != NULL); - ASSERT(arrSize > 0); + ASSERT(arrSize >= 0); int32_t dataLen = 0; int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index ead1168632..6c381f6e7d 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -206,7 +206,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -285,7 +285,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index da31e9c4c4..17d9aeaea0 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -148,9 +148,17 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { // get entry batch int32_t getCount = 0; SyncIndex getEntryIndex = nextIndex; - for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry); + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "get index:%d, code:%d, %s", getEntryIndex, code, tstrerror(terrno)); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + if (code == 0) { ASSERT(pEntry != NULL); entryPArr[i] = pEntry; @@ -162,12 +170,19 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { } } + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "build batch:%d", getCount); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + // build msg SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId); ASSERT(pMsg != NULL); // free entries - for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { SSyncRaftEntry* pEntry = entryPArr[i]; if (pEntry != NULL) { syncEntryDestory(pEntry); diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index 7ad3afe309..da97f40f1a 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -145,9 +145,9 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT sql insert into ct1 values(now+0s, 10, 2.0, 3.0) sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql flush database db; +#sql flush database db; -system sh/exec.sh -n dnode4 -s start +#system sh/exec.sh -n dnode4 -s start sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3) From 806c89cdfe82ab1aa3de479434beac906408ec80 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Jul 2022 17:06:19 +0800 Subject: [PATCH 24/39] test: adjust valgrind error num --- tests/script/tsim/valgrind/checkError2.sim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index eabe20b875..f98cd0df1d 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -35,6 +35,21 @@ if $rows != 1 then return -1 endi +print =============== step3: create show table +sql create table ct1 using stb tags(1000) +#sql show tables +#if $rows != 1 then +# return -1 +#endi + +print =============== step5: insert data +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) + +print =============== step6: select data +#sql select * from ct1 +#sql select * from stb + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 16c1df91ebd2fa31cbc636b6d578fbac7fc51a37 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 09:07:14 +0000 Subject: [PATCH 25/39] fix: tsdb commit merge bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 ++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b60a4697fa..8a189d98b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -602,7 +602,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { iter.pRow = NULL; while (true) { - pRow = tsdbTbDataIterGet(pIter); + pRow = tsdbTbDataIterGet(&iter); if (pRow == NULL) break; key = TSDBROW_KEY(pRow); @@ -610,7 +610,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock); if (c == 0) { nRow++; - tsdbTbDataIterNext(pIter); + tsdbTbDataIterNext(&iter); } else if (c > 0) { break; } else { @@ -635,7 +635,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; while (true) { - if (pRow) break; + if (pRow == NULL) break; code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 2bd62348c9..8e9cd5268e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1106,6 +1106,8 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; + tBlockDataInit(pBlockData1); + tBlockDataInit(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f906ef1b54..24b7c08476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -821,16 +821,20 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { int32_t code = 0; int32_t size; + ASSERT(pColDataSrc->nVal > 0); + pColDataDest->cid = pColDataSrc->cid; pColDataDest->type = pColDataSrc->type; pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->nVal = pColDataSrc->nVal; pColDataDest->flag = pColDataSrc->flag; - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { + size = BIT2_SIZE(pColDataSrc->nVal); + code = tRealloc(&pColDataDest->pBitMap, size); + if (code) goto _exit; + memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + } if (IS_VAR_DATA_TYPE(pColDataDest->type)) { size = sizeof(int32_t) * pColDataSrc->nVal; From ce6e505084748dfb43db0a348c53825a20bbf2c7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Jul 2022 17:27:49 +0800 Subject: [PATCH 26/39] feat(stream): stream state support delete --- source/libs/executor/src/timewindowoperator.c | 16 +++++++++++----- source/util/src/tpagedbuf.c | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a89e166e8a..9763f86838 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3189,7 +3189,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start *pIndex = index + 1; return pWin; } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index; + *pIndex = index + 1; return pWin; } } @@ -3442,7 +3442,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index) { taosArrayRemove(pWinInfos, index); } -static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { +static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -3700,13 +3700,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); // gap must be 0 - doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); + doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); @@ -3840,7 +3840,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { // gap must be 0 - doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL); copyDataBlock(pInfo->pDelRes, pBlock); pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; break; @@ -4232,6 +4232,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; + } else if (pBlock->info.type == STREAM_DELETE_DATA) { + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins); + copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); + taosArrayDestroy(pWins); + continue; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState); continue; diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index f50fd22ba6..0e608d0da2 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -518,8 +518,8 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { #ifdef BUF_PAGE_DEBUG uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%"PRId64, pi->pageId, pi->used, pi->offset); #endif - assert(pi->pData != NULL && pi->used == true); -// assert(pi->pData != NULL); + // assert(pi->pData != NULL && pi->used == true); + assert(pi->pData != NULL); pi->used = false; pBuf->statis.releasePages += 1; } From 7afdd3603d3103ce23e92320e1f7e98c2c822ffe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Jul 2022 17:20:03 +0800 Subject: [PATCH 27/39] fix: definite lost while send msg on dnode exiting --- .../{common => libs/transport}/src/tmsgcb.c | 17 ++- tests/script/tsim/valgrind/basic2.sim | 100 ++++++++++++++++-- 2 files changed, 104 insertions(+), 13 deletions(-) rename source/{common => libs/transport}/src/tmsgcb.c (79%) diff --git a/source/common/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c similarity index 79% rename from source/common/src/tmsgcb.c rename to source/libs/transport/src/tmsgcb.c index b8eec655b1..aefc25fab4 100644 --- a/source/common/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -16,20 +16,33 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" #include "taoserror.h" +#include "trpc.h" static SMsgCb defaultMsgCb; void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { - return (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg); + int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; } int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) { return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype); } -int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return (*defaultMsgCb.sendReqFp)(epSet, pMsg); } +int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { + int32_t code = (*defaultMsgCb.sendReqFp)(epSet, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); } diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index 157ac002e6..1a76d8ce5c 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -4,29 +4,107 @@ system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect -print =============== step1: create drop show dnodes +print =============== step1: show dnodes $x = 0 step1: $x = $x + 1 sleep 1000 if $x == 10 then - print ----> dnode not ready! + print ---> dnode not ready! return -1 endi sql show dnodes -print ----> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data00 $data01 $data02 $data03 $data04 $data05 if $rows != 1 then return -1 endi +if $data(1)[4] != ready then + goto step1 +endi -print =============== step2 -sql create database db vgroups 1 buffer 3 -sql use db -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) -sql create table ct1 using stb tags(1000) -sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +print =============== step2: create alter drop show user +sql create user u1 pass 'taosdata' +sql show users +sql alter user u1 sysinfo 1 +sql alter user u1 enable 1 +sql alter user u1 pass 'taosdata' +sql drop user u1 +sql_error alter user u2 sysinfo 0 +print =============== step3: create drop dnode +sql create dnode $hostname port 7200 +sql drop dnode 2 +sql alter dnode 1 'debugflag 131' -_OVER: +print =============== step4: + +print =============== run show xxxx +sql show dnodes +if $rows != 1 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi + +sql show databases +if $rows != 2 then + return -1 +endi + +sql show users +if $rows != 1 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi + +sql show variables; +if $rows != 4 then + return -1 +endi + +sql show dnode 1 variables; +if $rows <= 0 then + return -1 +endi + +sql show local variables; +if $rows <= 0 then + return -1 +endi + +print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT + +print =============== check +print ----> start to check if there are ERRORS in vagrind log file for each dnode +system_content sh/checkValgrind.sh -n dnode1 + +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 From 3ed0280025aeb43a667584b9b0e672cc7e086d0b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 17:39:49 +0800 Subject: [PATCH 28/39] feat: support del index when drop child table --- source/dnode/vnode/src/meta/metaTable.c | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index b3cdf47302..341173103c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -16,6 +16,7 @@ #include "meta.h" static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -103,6 +104,64 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const #endif return 0; } +int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return -1; + } + void * data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + if (tTagToValArray((const STag *)data, &pTagVals) != 0) { + return -1; + } + + SIndexMultiTerm *terms = indexMultiTermCreate(); + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + + char * key = pTagVal->pKey; + int32_t nKey = strlen(key); + + SIndexTerm *term = NULL; + if (type == TSDB_DATA_TYPE_NULL) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); + } else if (pTagVal->nData == 0) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); + } + if (term != NULL) { + indexMultiTermAdd(terms, term); + } + } + indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); +#endif + return 0; +} int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; @@ -434,9 +493,33 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (type) *type = e.type; + if (e.type == TSDB_CHILD_TABLE) { + void *tData = NULL; + int tLen = 0; + + if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { + version = *(int64_t *)tData; + STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version}; + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { + SDecoder tdc = {0}; + SMetaEntry stbEntry = {0}; + + tDecoderInit(&tdc, tData, tLen); + metaDecodeEntry(&tdc, &stbEntry); + const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + if (pTagColumn->type == TSDB_DATA_TYPE_JSON) { + metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); + } + tDecoderClear(&tdc); + } + tdbFree(tData); + } + } + tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); + if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); if (e.type == TSDB_CHILD_TABLE) { From 74501e8fd5fe2412134bf54cace4c3ffd76a66f3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 09:48:54 +0000 Subject: [PATCH 29/39] fix: merge commit --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8e9cd5268e..98decfe002 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -969,6 +969,8 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; + tBlockDataInit(pBlockData1); + tBlockDataInit(pBlockData2); for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); if (code) goto _err; From 866e4c4be0471ec85798de5c5ab2c893305b4621 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 17:16:12 +0800 Subject: [PATCH 30/39] refactor(stream): internal refactor --- include/common/tcommon.h | 14 ++ include/libs/executor/executor.h | 9 +- include/libs/stream/tstream.h | 5 - source/dnode/vnode/inc/vnode.h | 64 +++++---- source/dnode/vnode/src/inc/tq.h | 23 +--- source/dnode/vnode/src/sma/smaRollup.c | 8 +- source/dnode/vnode/src/tq/tq.c | 24 +++- source/dnode/vnode/src/tq/tqExec.c | 8 +- source/dnode/vnode/src/tq/tqMeta.c | 4 +- source/dnode/vnode/src/tq/tqRead.c | 172 ++++++++++++++++-------- source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/executor.c | 10 +- source/libs/executor/src/executorMain.c | 31 +++++ source/libs/executor/src/executorimpl.c | 20 +-- source/libs/executor/src/scanoperator.c | 41 +++--- 15 files changed, 276 insertions(+), 164 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index fd4ed6b180..63167bd834 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -117,6 +117,20 @@ typedef struct SSDataBlock { SDataBlockInfo info; } SSDataBlock; +enum { + FETCH_TYPE__DATA = 1, + FETCH_TYPE__META, + FETCH_TYPE__NONE, +}; + +typedef struct { + int8_t fetchType; + union { + SSDataBlock data; + void* meta; + }; +} SFetchRet; + typedef struct SVarColAttr { int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7b4565a99f..0094afef12 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -30,7 +30,7 @@ struct SRpcMsg; struct SSubplan; typedef struct SReadHandle { - void* streamReader; + void* tqReader; void* meta; void* config; void* vnode; @@ -38,7 +38,7 @@ typedef struct SReadHandle { SMsgCb* pMsgCb; bool initMetaReader; bool initTableReader; - bool initStreamReader; + bool initTqReader; } SReadHandle; typedef enum { @@ -52,7 +52,7 @@ typedef enum { * @param streamReadHandle * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); /** * Switch the stream scan to snapshot mode @@ -176,6 +176,9 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +void* qExtractReaderFromStreamScanner(void* scanner); +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 52f671e176..d6cb2c27b0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -235,11 +235,6 @@ typedef struct SStreamTask { int8_t taskStatus; int8_t execStatus; - // exec info - int64_t enqueueVer; - int64_t processedVer; - int64_t checkpointVer; - // node info int32_t selfChildId; int32_t nodeId; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 722087ea45..b381046ecf 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -64,8 +64,8 @@ int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); -void * vnodeGetIdx(SVnode *pVnode); -void * vnodeGetIvtIdx(SVnode *pVnode); +void *vnodeGetIdx(SVnode *pVnode); +void *vnodeGetIvtIdx(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); @@ -95,7 +95,7 @@ typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; int16_t type; - char * val; + char *val; bool reverse; int (*filterFunc)(void *a, void *b, int16_t type); @@ -136,8 +136,8 @@ SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdLis int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void * tsdbGetIdx(SMeta *pMeta); -void * tsdbGetIvtIdx(SMeta *pMeta); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols, void **pReader); @@ -146,19 +146,37 @@ int32_t tsdbLastrowReaderClose(void *pReader); // tq -typedef struct STqReadHandle SStreamReader; +typedef struct STqReader { + int64_t ver; + const SSubmitReq *pMsg; + SSubmitBlk *pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; -SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta); + SWalReader *pWalReader; -void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList); -int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList); + SMeta *pVnodeMeta; + SHashObj *tbIdHash; + SArray *pColIdList; // SArray -int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(SStreamReader *pHandle); -bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle); + int32_t cachedSchemaVer; + int64_t cachedSchemaSuid; + SSchemaWrapper *pSchemaWrapper; + STSchema *pSchema; +} STqReader; + +STqReader *tqOpenReader(SVnode *pVnode); +void tqCloseReader(STqReader *); + +void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); + +int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock(STqReader *pReader); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); @@ -214,7 +232,7 @@ struct SMetaEntry { int8_t type; int8_t flags; // TODO: need refactor? tb_uid_t uid; - char * name; + char *name; union { struct { SSchemaWrapper schemaRow; @@ -225,7 +243,7 @@ struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char *comment; tb_uid_t suid; uint8_t *pTags; } ctbEntry; @@ -233,7 +251,7 @@ struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char *comment; int32_t ncid; // next column id SSchemaWrapper schemaRow; } ntbEntry; @@ -247,17 +265,17 @@ struct SMetaEntry { struct SMetaReader { int32_t flags; - SMeta * pMeta; + SMeta *pMeta; SDecoder coder; SMetaEntry me; - void * pBuf; + void *pBuf; int32_t szBuf; }; struct SMTbCursor { - TBC * pDbc; - void * pKey; - void * pVal; + TBC *pDbc; + void *pKey; + void *pVal; int32_t kLen; int32_t vLen; SMetaReader mr; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 12fb500ba3..9c3bd85c71 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -44,25 +44,6 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; -// tqRead - -struct STqReadHandle { - int64_t ver; - const SSubmitReq* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - - SMeta* pVnodeMeta; - SHashObj* tbIdHash; - SArray* pColIdList; // SArray - - int32_t cachedSchemaVer; - int64_t cachedSchemaSuid; - SSchemaWrapper* pSchemaWrapper; - STSchema* pSchema; -}; - // tqPush typedef struct { @@ -102,7 +83,7 @@ typedef struct { typedef struct { int8_t subType; - SStreamReader* pExecReader[5]; + STqReader* pExecReader[5]; union { STqExecCol execCol; STqExecTb execTb; @@ -118,7 +99,7 @@ typedef struct { int32_t epoch; int8_t fetchMeta; - // reader + // TODO remove SWalReader* pWalReader; // push diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 92168a9b92..125490ecd5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -325,14 +325,14 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_FAILED; } - SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta); - if (!pReadHandle) { + STqReader *pReader = tqOpenReader(pVnode); + if (!pReader) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } SReadHandle handle = { - .streamReader = pReadHandle, + .tqReader = pReader, .meta = pMeta, .pMsgCb = pMsgCb, .vnode = pVnode, @@ -364,7 +364,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; _err: tdFreeRSmaInfo(pRSmaInfo); - taosMemoryFree(pReadHandle); + taosMemoryFree(pReader); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2dd80c5d52..e21b0fe9e8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -447,26 +447,38 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->fetchMeta = req.withMeta; pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - for (int32_t i = 0; i < 5; i++) { - pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - } + /*for (int32_t i = 0; i < 5; i++) {*/ + /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/ + /*}*/ if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { pHandle->execHandle.execCol.qmsg = req.qmsg; req.qmsg = NULL; for (int32_t i = 0; i < 5; i++) { SReadHandle handle = { - .streamReader = pHandle->execHandle.pExecReader[i], + .tqReader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTableReader = true, + .initTqReader = true, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); + void* scanner = NULL; + qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner); + ASSERT(scanner); + pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner); + ASSERT(pHandle->execHandle.pExecReader[i]); } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { + for (int32_t i = 0; i < 5; i++) { + pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); + } pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + for (int32_t i = 0; i < 5; i++) { + pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); + } pHandle->execHandle.execTb.suid = req.suid; SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList); @@ -476,7 +488,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid); } for (int32_t i = 0; i < 5; i++) { - tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); + tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); } taosArrayDestroy(tbUidList); } @@ -532,7 +544,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { SReadHandle handle = { .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initStreamReader = 1, + .initTqReader = 1, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } else { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 9dd2a0258f..16822e6003 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -135,8 +135,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { pRsp->withSchema = 1; - SStreamReader* pReader = pExec->pExecReader[workerId]; - tqReadHandleSetMsg(pReader, pReq, 0); + STqReader* pReader = pExec->pExecReader[workerId]; + tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block, pReader) < 0) { @@ -153,8 +153,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { pRsp->withSchema = 1; - SStreamReader* pReader = pExec->pExecReader[workerId]; - tqReadHandleSetMsg(pReader, pReq, 0); + STqReader* pReader = pExec->pExecReader[workerId]; + tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block, pReader) < 0) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index c7e9c8eed6..67fa4ed166 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -79,12 +79,12 @@ int32_t tqMetaOpen(STQ* pTq) { tDecodeSTqHandle(&decoder, &handle); handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); for (int32_t i = 0; i < 5; i++) { - handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); } if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { for (int32_t i = 0; i < 5; i++) { SReadHandle reader = { - .streamReader = handle.execHandle.pExecReader[i], + .tqReader = handle.execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, .vnode = pTq->pVnode, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index bc992b2211..53ef17a6ba 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -15,6 +15,11 @@ #include "tq.h" +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) { + /*if ()*/ + return 0; +} + int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t code = 0; taosThreadMutexLock(&pHandle->pWalReader->mutex); @@ -73,53 +78,107 @@ END: return code; } -SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) { - SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader)); - if (pReadHandle == NULL) { +STqReader* tqOpenReader(SVnode* pVnode) { + STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + if (pReader == NULL) { return NULL; } - pReadHandle->pVnodeMeta = pMeta; - pReadHandle->pMsg = NULL; - pReadHandle->ver = -1; - pReadHandle->pColIdList = NULL; - pReadHandle->cachedSchemaVer = 0; - pReadHandle->cachedSchemaSuid = 0; - pReadHandle->pSchema = NULL; - pReadHandle->pSchemaWrapper = NULL; - pReadHandle->tbIdHash = NULL; - return pReadHandle; + + // TODO open + /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/ + + pReader->pVnodeMeta = pVnode->pMeta; + pReader->pMsg = NULL; + pReader->ver = -1; + pReader->pColIdList = NULL; + pReader->cachedSchemaVer = 0; + pReader->cachedSchemaSuid = 0; + pReader->pSchema = NULL; + pReader->pSchemaWrapper = NULL; + pReader->tbIdHash = NULL; + return pReader; } -int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) { - pReadHandle->pMsg = pMsg; +void tqCloseReader(STqReader* pReader) { + // close wal reader + // free cached schema + // free hash + taosMemoryFree(pReader); +} - if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; +int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { + bool fromProcessedMsg = pReader->pMsg != NULL; + + while (1) { + if (!fromProcessedMsg) { + if (walNextValidMsg(pReader->pWalReader) < 0) { + ret->fetchType = FETCH_TYPE__NONE; + return -1; + } + void* body = pReader->pWalReader->pHead->head.body; + if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { + // TODO do filter + ret->fetchType = FETCH_TYPE__META; + ret->meta = pReader->pWalReader->pHead->head.body; + return 0; + } else { + tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + } + } + + while (tqNextDataBlock(pReader)) { + memset(&ret->data, 0, sizeof(SSDataBlock)); + int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + if (code != 0 || ret->data.info.rows == 0) { + if (fromProcessedMsg) { + ret->fetchType = FETCH_TYPE__NONE; + return 0; + } else { + break; + } + } + + ret->fetchType = FETCH_TYPE__DATA; + return 0; + } + + if (fromProcessedMsg) { + ret->fetchType = FETCH_TYPE__NONE; + return 0; + } + } +} + +int32_t tqReaderSetDataMsg(STqReader* pReader, SSubmitReq* pMsg, int64_t ver) { + pReader->pMsg = pMsg; + + if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; - if (pReadHandle->pBlock == NULL) break; + if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1; + if (pReader->pBlock == NULL) break; } - if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; - pReadHandle->ver = ver; - memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); + if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; + pReader->ver = ver; + memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } -bool tqNextDataBlock(SStreamReader* pHandle) { - if (pHandle->pMsg == NULL) return false; +bool tqNextDataBlock(STqReader* pReader) { + if (pReader->pMsg == NULL) return false; while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) { return false; } - if (pHandle->pBlock == NULL) { - pHandle->pMsg = NULL; + if (pReader->pBlock == NULL) { + pReader->pMsg = NULL; return false; } - if (pHandle->tbIdHash == NULL) { + if (pReader->tbIdHash == NULL) { return true; } - void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t)); + void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/ if (ret != NULL) { /*tqDebug("find uid %ld", pHandle->msgIter.uid);*/ @@ -129,7 +188,7 @@ bool tqNextDataBlock(SStreamReader* pHandle) { return false; } -bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { +bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; @@ -145,38 +204,38 @@ bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema - int32_t sversion = htonl(pHandle->pBlock->sversion); - if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || - pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { - if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema); - pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); - if (pHandle->pSchema == NULL) { + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + if (pReader->pSchema) taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion); + if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", - pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->cachedSchemaVer); + pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } - if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper); - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); - if (pHandle->pSchemaWrapper == NULL) { + if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true); + if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", - pHandle->msgIter.uid, pHandle->cachedSchemaVer); + pReader->msgIter.uid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } - pHandle->cachedSchemaVer = sversion; - pHandle->cachedSchemaSuid = pHandle->msgIter.suid; + pReader->cachedSchemaVer = sversion; + pReader->cachedSchemaSuid = pReader->msgIter.suid; } - STSchema* pTschema = pHandle->pSchema; - SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); if (colNumNeed == 0) { int32_t colMeta = 0; @@ -199,7 +258,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); if (colIdSchema < colIdNeed) { colMeta++; } else if (colIdSchema > colIdNeed) { @@ -216,7 +275,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { } } - if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) { + if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) { goto FAIL; } @@ -227,13 +286,12 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { STSRow* row; int32_t curRow = 0; - tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - pBlock->info.groupId = 0; - pBlock->info.uid = pHandle->msgIter.uid; - pBlock->info.rows = pHandle->msgIter.numOfRows; + pBlock->info.uid = pReader->msgIter.uid; + pBlock->info.rows = pReader->msgIter.numOfRows; - while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { + while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { @@ -255,9 +313,9 @@ FAIL: return -1; } -void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } +void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } -int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash) { taosHashClear(pHandle->tbIdHash); } @@ -276,7 +334,7 @@ int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash == NULL) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { @@ -293,7 +351,7 @@ int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) { ASSERT(pHandle->tbIdHash != NULL); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9b32560041..85ddfc4b2b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -365,7 +365,7 @@ typedef struct SStreamScanInfo { int32_t blockType; // current block type int32_t validBlockIndex; // Is current data has returned? uint64_t numOfExec; // execution times - void* streamReader;// stream block reader handle + STqReader* tqReader; int32_t tsArrayIndex; SArray* tsArray; @@ -383,6 +383,11 @@ typedef struct SStreamScanInfo { SSDataBlock* pPullDataRes; // pull data SSDataBlock SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; + + // status for tmq + //SSchemaWrapper schema; + STqOffset offset; + } SStreamScanInfo; typedef struct SSysTableScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0155cdd416..a83565cbe0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -45,7 +45,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pInfo->blockType = type; if (type == STREAM_INPUT__DATA_SUBMIT) { - if (tqReadHandleSetMsg(pInfo->streamReader, input, 0) < 0) { + if (tqReaderSetDataMsg(pInfo->tqReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } @@ -105,7 +105,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO return code; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { if (msg == NULL) { return NULL; } @@ -120,7 +120,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; @@ -174,11 +174,11 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); - code = tqReadHandleAddTbUidList(pScanInfo->streamReader, qa); + code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); - code = tqReadHandleRemoveTbUidList(pScanInfo->streamReader, tableIdList); + code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); } return code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 7f84b4c174..d5e546f466 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -236,6 +236,37 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le return decodeOperator(pTaskInfo->pRoot, pInput, len); } +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + + while (1) { + uint8_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + *scanner = pOperator->info; + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } +} + +void* qExtractReaderFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return (void*)pInfo->tqReader; +} + +const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return pInfo->tqReader->pSchemaWrapper; +} + +const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return &pInfo->offset; +} + int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fa84d79f56..9865fbc5cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2842,7 +2842,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { - int32_t type = pOperator->operatorType; + uint8_t type = pOperator->operatorType; pOperator->status = OP_OPENED; @@ -4309,11 +4309,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; -// int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); -// if (code) { -// pTaskInfo->code = code; -// return NULL; -// } + // int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + // if (code) { + // pTaskInfo->code = code; + // return NULL; + // } int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { @@ -4370,8 +4370,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo); } else { - pOptr = - createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, + pScalarExprInfo, numOfScalarExpr, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -4538,7 +4538,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } -STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) { +STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, const char* idstr) { int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4782,7 +4783,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); - if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index aae85bbbc1..68cda52b10 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1232,38 +1232,33 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->streamReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; // todo refactor - int32_t code = tqRetrieveDataBlock(&block, pInfo->streamReader); + int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); - uint64_t groupId = block.info.groupId; - uint64_t uid = block.info.uid; - int32_t numOfRows = block.info.rows; - - if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { + if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { pTaskInfo->code = code; return NULL; } - pInfo->pRes->info.groupId = groupId; - pInfo->pRes->info.rows = numOfRows; - pInfo->pRes->info.uid = uid; + pInfo->pRes->info.rows = block.info.rows; + pInfo->pRes->info.uid = block.info.uid; pInfo->pRes->info.type = STREAM_NORMAL; - pInfo->pRes->info.capacity = numOfRows; + pInfo->pRes->info.capacity = block.info.rows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this if (pInfo->assignBlockUid) { - pInfo->pRes->info.groupId = uid; - } else { - pInfo->pRes->info.groupId = groupId; + pInfo->pRes->info.groupId = block.info.uid; } - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; + } else { + pInfo->pRes->info.groupId = 0; } // todo extract method @@ -1413,13 +1408,13 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - if (pHandle->initStreamReader) { - ASSERT(pHandle->streamReader == NULL); - pInfo->streamReader = tqInitSubmitMsgScanner(pHandle->meta); - ASSERT(pInfo->streamReader); + if (pHandle->initTqReader) { + ASSERT(pHandle->tqReader == NULL); + pInfo->tqReader = tqOpenReader(pHandle->vnode); + ASSERT(pInfo->tqReader); } else { - ASSERT(pHandle->streamReader); - pInfo->streamReader = pHandle->streamReader; + ASSERT(pHandle->tqReader); + pInfo->tqReader = pHandle->tqReader; } if (pSTInfo->interval.interval > 0) { @@ -1435,9 +1430,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList(pInfo->streamReader, pColIds); + tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); - int32_t code = tqReadHandleSetTbUidList(pInfo->streamReader, tableIdList); + int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; From e0532065f089e4820d7a0969a3b82af5ce0444ba Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 7 Jul 2022 17:58:03 +0800 Subject: [PATCH 31/39] other: comment out the test case and add debug logs --- source/dnode/vnode/src/sma/smaRollup.c | 2 ++ tests/script/jenkins/basic.txt | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 92168a9b92..669fbe1b49 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%"PRIi32, SMA_VID(pSma), + pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; _err: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e009e702d..0783aa0fd1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -163,8 +163,8 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From fe87085b213f1f2b1ae0d3d2753f78f45d724d1f Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 7 Jul 2022 23:09:27 +0000 Subject: [PATCH 32/39] Minor adjustments to crash_gen tool, so that it can work with 3.0 code base, now can do simple -t 4 -s 25 --- tests/pytest/crash_gen.sh | 7 ++++++- tests/pytest/crash_gen/crash_gen_main.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 127e13c5be..539314dea4 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -48,7 +48,12 @@ fi PYTHON_EXEC=python3.8 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) +# export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) +# NOTE: we are now pointing outside the current github, per Wade on 7/7/2022, we'll be keeping connectors outside +# and there does not seem to be a module to reference that. +PROJECT_PARENT=$(pwd)/../../.. +TAOS_PYTHON_PROJECT_DIR=$PROJECT_PARENT/taos-connector-python +export PYTHONPATH=$TAOS_PYTHON_PROJECT_DIR:$(pwd) # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index b743eee2ef..08155f656b 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -466,6 +466,7 @@ class ThreadCoordinator: self._te = None # No more executor, time to end Logging.debug("Main thread tapping all threads one last time...") self.tapAllThreads() # Let the threads run one last time + #TODO: looks like we are not capturing the failures for the last step yet (i.e. calling registerFailure if neccessary) Logging.debug("\r\n\n--> Main thread ready to finish up...") Logging.debug("Main thread joining all threads") @@ -1290,6 +1291,7 @@ class Task(): def _isErrAcceptable(self, errno, msg): if errno in [ + # TDengine 2.x Error Codes: 0x05, # TSDB_CODE_RPC_NOT_READY 0x0B, # Unable to establish connection, more details in TD-1648 # 0x200, # invalid SQL， TODO: re-examine with TD-934 @@ -1310,6 +1312,18 @@ class Task(): 0x14, # db not ready, errno changed 0x600, # Invalid table ID, why? 0x218, # Table does not exist + + # TDengine 3.0 Error Codes: + 0x0333, # Object is creating # TODO: this really is NOT an acceptable error + 0x03A0, # STable already exists + 0x03A1, # STable [does] not exist + 0x03AA, # Tag already exists + 0x0603, # Table already exists + 0x2602, # Table does not exist + 0x260d, # Tags number not matched + + + 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones @@ -1749,6 +1763,8 @@ class TdSuperTable: tagType = tags[tagName] if tagType == 'BINARY': tagStrs.append("'Beijing-Shanghai-LosAngeles'") + elif tagType== 'VARCHAR': + tagStrs.append("'London-Paris-Berlin'") elif tagType == 'FLOAT': tagStrs.append('9.9') elif tagType == 'INT': From c591d16e29e1345f077b225c78b8e08167aca47b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 09:24:53 +0800 Subject: [PATCH 33/39] test: add loop script for multi run use case --- tests/script/loop.sh | 8 +++++- tests/system-test/loop.sh | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 tests/system-test/loop.sh diff --git a/tests/script/loop.sh b/tests/script/loop.sh index b435eef123..b3bc9728de 100755 --- a/tests/script/loop.sh +++ b/tests/script/loop.sh @@ -13,7 +13,7 @@ CMD_NAME= LOOP_TIMES=5 SLEEP_TIME=0 -while getopts "f:t:s:" arg +while getopts "hf:t:s:" arg do case $arg in f) @@ -25,6 +25,12 @@ do s) SLEEP_TIME=$OPTARG ;; + h) + echo "Usage: $(basename $0) -f [cmd name] " + echo " -t [loop times] " + echo " -s [sleep time] " + exit 0 + ;; ?) echo "unknow argument" ;; diff --git a/tests/system-test/loop.sh b/tests/system-test/loop.sh new file mode 100755 index 0000000000..b3bc9728de --- /dev/null +++ b/tests/system-test/loop.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +################################################## +# +# Do simulation test +# +################################################## + +set -e +#set -x + +CMD_NAME= +LOOP_TIMES=5 +SLEEP_TIME=0 + +while getopts "hf:t:s:" arg +do + case $arg in + f) + CMD_NAME=$OPTARG + ;; + t) + LOOP_TIMES=$OPTARG + ;; + s) + SLEEP_TIME=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -f [cmd name] " + echo " -t [loop times] " + echo " -s [sleep time] " + exit 0 + ;; + ?) + echo "unknow argument" + ;; + esac +done + +echo LOOP_TIMES ${LOOP_TIMES} +echo CMD_NAME ${CMD_NAME} +echo SLEEP_TIME ${SLEEP_TIME} + +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +for ((i=0; i<$LOOP_TIMES; i++ )) +do + echo -e $GREEN loop $i $NC + echo -e $GREEN cmd $CMD_NAME $NC + $CMD_NAME + sleep ${SLEEP_TIME} +done From 2339f9a5b0c79d3d345fc73e24ba09872153569c Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 10:39:38 +0800 Subject: [PATCH 34/39] test: use flush data into disk,not restart taosd --- .../system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 5 +++-- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 6 +++--- .../system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 7 ++++--- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py | 7 ++++--- tests/system-test/7-tmq/tmqConsFromTsdb.py | 5 +++-- .../system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py | 5 +++-- .../7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 5 +++-- .../system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 5 +++-- 12 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index bd8531d7c8..d22d183e86 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index 1d2aaccda5..951a747069 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 2720b7cdce..6ee089af4e 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -64,9 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index 0b2dddd24a..882989bfb6 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -64,9 +64,10 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # # tdDnodes.start(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index a4d6648276..c334ff752b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -64,9 +64,10 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # # tdDnodes.start(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index c18474dcc3..a4a242365a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 540c9dbbe1..ec11b3286c 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 5cb373092b..4878652593 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index eaef134845..451dc43343 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index c1b327e5f1..3b3467f9f9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index dd8a0ad33a..d1fe69f0b7 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index a183eda1c7..597f7968ae 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): From ae19c8f63e2674e36afdc64e293b3013fe0c55b4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 11:49:32 +0800 Subject: [PATCH 35/39] test: comment 2 rsma cases --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0783aa0fd1..4e009e702d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -163,8 +163,8 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From 09c095a91cfbef049dd7751e4b20979baf4e9e0f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 8 Jul 2022 11:51:35 +0800 Subject: [PATCH 36/39] fix: asan error --- cmake/cmake.define | 4 ++-- cmake/cmake.options | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index b5d5c6d957..1cc28ec6fd 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -97,10 +97,10 @@ ELSE () SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") ENDIF () - IF (${SANITIZER} MATCHES "true") + IF (${BUILD_SANITIZER}) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - MESSAGE(STATUS "Will compile with Address Sanitizer!") + MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0") diff --git a/cmake/cmake.options b/cmake/cmake.options index 09236a5b87..e013ff7592 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -77,6 +77,12 @@ ELSE () ENDIF () ENDIF () +option( + BUILD_SANITIZER + "If build addr2line" + OFF + ) + option( BUILD_ADDR2LINE "If build addr2line" diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e53a1b43af..8e5b135858 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1940,6 +1940,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } ASSERT(0); + return -1; } static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, From 74eaa512d367a1affdb848a34b5c5a050c38b5d8 Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:04:33 +0800 Subject: [PATCH 37/39] docs: Update 03-immigrate.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 存储资源估算的计算公式有误 --- docs/zh/25-application/03-immigrate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/25-application/03-immigrate.md b/docs/zh/25-application/03-immigrate.md index 9d8946bc4a..d1c9caea09 100644 --- a/docs/zh/25-application/03-immigrate.md +++ b/docs/zh/25-application/03-immigrate.md @@ -367,10 +367,10 @@ WHERE ts>=1510560000 AND ts<=1515000009 ### 存储资源估算 -假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源： +假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `86400×n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(86400×n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源： ```matlab -(n×t×L)×(365×1.5)×(1+20%)/C +(86400×n×t×L)×(365×1.5)×(1+20%)/C ``` 结合以上的计算公式，将参数带入计算公式，在不考虑标签信息的情况下，每年产生的原始数据规模是 11.8TB。需要注意的是，由于标签信息在 TDengine 中关联到每个时间线，并不是每条记录。所以需要记录的数据量规模相对于产生的数据有一定的降低，而这部分标签数据整体上可以忽略不记。假设压缩比为 5，则保留的数据规模最终为 2.56 TB。 From 81205dedb1fed578fa54818847e38e690c89441a Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:07:21 +0800 Subject: [PATCH 38/39] Update 03-immigrate.md --- docs/en/25-application/03-immigrate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/25-application/03-immigrate.md b/docs/en/25-application/03-immigrate.md index 4d47aec1d7..fe67f97389 100644 --- a/docs/en/25-application/03-immigrate.md +++ b/docs/en/25-application/03-immigrate.md @@ -379,11 +379,11 @@ We still use the hypothetical environment from Chapter 4. There are three measur ### Storage resource estimation -Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7. +Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `86400 * n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(86400 * n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7. With additional 20% redundancy, you can calculate the required storage resources: ```matlab -(n * t * L) * (365 * 1.5) * (1+20%)/C +(86400 * n * t * L) * (365 * 1.5) * (1+20%)/C ```` Substituting in the above formula, the raw data generated every year is 11.8TB without considering the label information. Note that tag information is associated with each timeline in TDengine, not every record. The amount of data to be recorded is somewhat reduced relative to the generated data, and label data can be ignored as a whole. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB. From adb90b68e3f41cb978c640533c3d86c44c659d3c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 8 Jul 2022 09:47:41 +0800 Subject: [PATCH 39/39] fix(stream): window_close crash --- source/libs/executor/src/timewindowoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9763f86838..f2a9a13c52 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1316,7 +1316,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) // window has been closed return false; } - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); + // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return true; @@ -1419,7 +1419,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, ASSERT(pRecyPages != NULL); taosArrayPush(pRecyPages, &pPos->pageId); } else { - SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); + // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); // dBufSetBufPageRecycled(pDiscBuf, bufPage); } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; @@ -1446,7 +1446,7 @@ static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) { int32_t size = taosArrayGetSize(pageIds); for (int32_t i = 0; i < size; i++) { int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i); - SFilePage* bufPage = getBufPage(pDiskBuf, pageId); + // SFilePage* bufPage = getBufPage(pDiskBuf, pageId); // dBufSetBufPageRecycled(pDiskBuf, bufPage); } taosArrayClear(pageIds);