xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feature/3_liaohj' into feature/3_liaohj

This commit is contained in:
Haojun Liao 2023-04-06 15:09:06 +08:00
parent 665ba407cf 06c7bbe91b
commit 7e734e3088
46 changed files with 2259 additions and 2809 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.0.3.1") SET(TD_VER_NUMBER "3.0.3.2")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 273a3fe GIT_TAG 6ae60ac
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

5
include/common/tcommon.h
View File

@ -208,15 +208,12 @@ typedef struct SSDataBlock {
} SSDataBlock; } SSDataBlock;
enum { enum {
FETCH_TYPE__DATA = 1, FETCH_TYPE__DATA = 0,
FETCH_TYPE__META,
FETCH_TYPE__SEP,
FETCH_TYPE__NONE, FETCH_TYPE__NONE,
}; };
typedef struct { typedef struct {
int8_t fetchType; int8_t fetchType;
STqOffsetVal offset;
union { union {
SSDataBlock data; SSDataBlock data;
void* meta; void* meta;

36
include/common/ttokendef.h
View File

@ -127,24 +127,24 @@
#define TK_NK_EQ 109 #define TK_NK_EQ 109
#define TK_USING 110 #define TK_USING 110
#define TK_TAGS 111 #define TK_TAGS 111
#define TK_COMMENT 112 #define TK_BOOL 112
#define TK_BOOL 113 #define TK_TINYINT 113
#define TK_TINYINT 114 #define TK_SMALLINT 114
#define TK_SMALLINT 115 #define TK_INT 115
#define TK_INT 116 #define TK_INTEGER 116
#define TK_INTEGER 117 #define TK_BIGINT 117
#define TK_BIGINT 118 #define TK_FLOAT 118
#define TK_FLOAT 119 #define TK_DOUBLE 119
#define TK_DOUBLE 120 #define TK_BINARY 120
#define TK_BINARY 121 #define TK_NCHAR 121
#define TK_NCHAR 122 #define TK_UNSIGNED 122
#define TK_UNSIGNED 123 #define TK_JSON 123
#define TK_JSON 124 #define TK_VARCHAR 124
#define TK_VARCHAR 125 #define TK_MEDIUMBLOB 125
#define TK_MEDIUMBLOB 126 #define TK_BLOB 126
#define TK_BLOB 127 #define TK_VARBINARY 127
#define TK_VARBINARY 128 #define TK_DECIMAL 128
#define TK_DECIMAL 129 #define TK_COMMENT 129
#define TK_MAX_DELAY 130 #define TK_MAX_DELAY 130
#define TK_WATERMARK 131 #define TK_WATERMARK 131
#define TK_ROLLUP 132 #define TK_ROLLUP 132

6
include/libs/executor/executor.h
View File

@ -198,12 +198,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
// //
int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit);
int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); void qStreamSetOpen(qTaskInfo_t tinfo);
void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo);
const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo); const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);

4
include/libs/wal/wal.h
View File

@ -146,8 +146,8 @@ typedef struct {
int64_t curFileFirstVer; int64_t curFileFirstVer;
int64_t curVersion; int64_t curVersion;
int64_t capacity; int64_t capacity;
int8_t curInvalid; // int8_t curInvalid;
int8_t curStopped; // int8_t curStopped;
TdThreadMutex mutex; TdThreadMutex mutex;
SWalFilterCond cond; SWalFilterCond cond;
// TODO remove it // TODO remove it

1
include/util/taoserror.h
View File

@ -758,6 +758,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000)
#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001)
#define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002)
#define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003)
// stream // stream
#define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100)

3
source/client/src/clientSml.c
View File

@ -533,8 +533,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL;
if (index) { if (index) {
if (colField[*index].type != kv->type) { if (colField[*index].type != kv->type) {
uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key);
kv->key, colField[*index].type, kv->type);
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }

2
source/client/src/clientTmq.c
View File

@ -1264,6 +1264,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP;
taosWriteQitem(tmq->mqueue, pRspWrapper); taosWriteQitem(tmq->mqueue, pRspWrapper);
}else if(code == TSDB_CODE_WAL_LOG_NOT_EXIST){ //poll data while insert
taosMsleep(500);
} }
goto CREATE_MSG_FAIL; goto CREATE_MSG_FAIL;

1
source/common/src/tmsg.c
View File

@ -7212,6 +7212,7 @@ void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) {
tDestroySSubmitTbData(&aSubmitTbData[i], flag); tDestroySSubmitTbData(&aSubmitTbData[i], flag);
} }
taosArrayDestroy(pReq->aSubmitTbData); taosArrayDestroy(pReq->aSubmitTbData);
pReq->aSubmitTbData = NULL;
} }
int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) {

4
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -24,10 +24,10 @@ extern "C" {
enum { enum {
MQ_CONSUMER_STATUS__MODIFY = 1, MQ_CONSUMER_STATUS__MODIFY = 1,
MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore // MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__READY,
MQ_CONSUMER_STATUS__LOST, MQ_CONSUMER_STATUS__LOST,
MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore // MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__LOST_REBD, MQ_CONSUMER_STATUS__LOST_REBD,
MQ_CONSUMER_STATUS__REMOVED, MQ_CONSUMER_STATUS__REMOVED,
}; };

14
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -337,7 +337,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
} }
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} else if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
taosRLockLatch(&pConsumer->lock); taosRLockLatch(&pConsumer->lock);
int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
@ -876,17 +876,11 @@ static void updateConsumerStatus(SMqConsumerObj* pConsumer) {
int32_t status = pConsumer->status; int32_t status = pConsumer->status;
if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) {
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { if (status == MQ_CONSUMER_STATUS__MODIFY) {
pConsumer->status = MQ_CONSUMER_STATUS__READY; pConsumer->status = MQ_CONSUMER_STATUS__READY;
} else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { } else if (status == MQ_CONSUMER_STATUS__LOST) {
pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
} }
} else {
if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
} else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
pConsumer->status = MQ_CONSUMER_STATUS__LOST;
}
} }
} }
@ -1195,10 +1189,8 @@ static const char *mndConsumerStatusName(int status) {
return "ready"; return "ready";
case MQ_CONSUMER_STATUS__LOST: case MQ_CONSUMER_STATUS__LOST:
case MQ_CONSUMER_STATUS__LOST_REBD: case MQ_CONSUMER_STATUS__LOST_REBD:
case MQ_CONSUMER_STATUS__LOST_IN_REB:
return "lost"; return "lost";
case MQ_CONSUMER_STATUS__MODIFY: case MQ_CONSUMER_STATUS__MODIFY:
case MQ_CONSUMER_STATUS__MODIFY_IN_REB:
return "rebalancing"; return "rebalancing";
default: default:
return "unknown"; return "unknown";

11
source/dnode/vnode/inc/vnode.h
View File

@ -228,19 +228,12 @@ typedef struct SSnapContext {
SArray *idList; SArray *idList;
int32_t index; int32_t index;
bool withMeta; bool withMeta;
bool queryMetaOrData; // true-get meta, false-get data bool queryMeta; // true-get meta, false-get data
} SSnapContext; } SSnapContext;
typedef struct STqReader { typedef struct STqReader {
// const SSubmitReq *pMsg;
// SSubmitBlk *pBlock;
// SSubmitMsgIter msgIter;
// SSubmitBlkIter blkIter;
int64_t ver;
SPackedData msg2; SPackedData msg2;
int8_t setMsg;
SSubmitReq2 submit; SSubmitReq2 submit;
int32_t nextBlk; int32_t nextBlk;
@ -267,7 +260,7 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); void tqNextBlock(STqReader *pReader, SFetchRet *ret);
int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver);

4
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -268,7 +268,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
ctx->snapVersion = snapVersion; ctx->snapVersion = snapVersion;
ctx->suid = suid; ctx->suid = suid;
ctx->subType = subType; ctx->subType = subType;
ctx->queryMetaOrData = withMeta; ctx->queryMeta = withMeta;
ctx->withMeta = withMeta; ctx->withMeta = withMeta;
ctx->idVersion = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); ctx->idVersion = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (ctx->idVersion == NULL) { if (ctx->idVersion == NULL) {
@ -475,7 +475,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
if (ctx->index >= taosArrayGetSize(ctx->idList)) { if (ctx->index >= taosArrayGetSize(ctx->idList)) {
metaDebug("tmqsnap get meta end"); metaDebug("tmqsnap get meta end");
ctx->index = 0; ctx->index = 0;
ctx->queryMetaOrData = false; // change to get data ctx->queryMeta = false; // change to get data
return 0; return 0;
} }

157
source/dnode/vnode/src/tq/tq.c
View File

@ -236,21 +236,6 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
} }
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type) { int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type) {
#if 0
A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
A(!pRsp->withSchema);
A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
if (pRsp->blockNum > 0) {
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
} else {
A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
}
}
#endif
doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type); doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type);
char buf1[80] = {0}; char buf1[80] = {0};
@ -352,17 +337,6 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
} }
pRsp->withTbName = 0; pRsp->withTbName = 0;
#if 0
pRsp->withTbName = pReq->withTbName;
if (pRsp->withTbName) {
pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
if (pRsp->blockTbName == NULL) {
// TODO free
return -1;
}
}
#endif
pRsp->withSchema = false; pRsp->withSchema = false;
return 0; return 0;
} }
@ -421,6 +395,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
return -1; return -1;
} }
// offset set to previous version when init
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1);
} }
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
@ -461,7 +436,6 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* pOffset) { SRpcMsg* pMsg, STqOffsetVal* pOffset) {
int32_t code = 0;
uint64_t consumerId = pRequest->consumerId; uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
@ -472,10 +446,9 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
code = tqScanData(pTq, pHandle, &dataRsp, pOffset); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
if (code != TSDB_CODE_SUCCESS) { if(code != 0) {
taosWUnLockLatch(&pTq->lock); goto end;
return code;
} }
// till now, all data has been transferred to consumer, new data needs to push client once arrived. // till now, all data has been transferred to consumer, new data needs to push client once arrived.
@ -486,89 +459,60 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
return code; return code;
} }
taosWUnLockLatch(&pTq->lock);
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
// NOTE: this pHandle->consumerId may have been changed already. // NOTE: this pHandle->consumerId may have been changed already.
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
", ts:%" PRId64 ", reqId:0x%" PRIx64,
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
dataRsp.rspOffset.ts, pRequest->reqId);
tDeleteSMqDataRsp(&dataRsp); end:
{
char buf[80] = {0};
tFormatOffset(buf, 80, &dataRsp.rspOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d",
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code);
taosWUnLockLatch(&pTq->lock);
tDeleteSMqDataRsp(&dataRsp);
}
return code; return code;
} }
static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
int32_t code = -1; static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) {
STqOffsetVal offset = {0}; int code = 0;
SWalCkHead* pCkHead = NULL;
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SWalCkHead* pCkHead = NULL;
STqOffsetVal reqOffset = pRequest->reqOffset;
uint64_t consumerId = pRequest->consumerId;
// 1. reset the offset if needed
if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
// handle the reset offset cases, according to the consumer's choice.
bool blockReturned = false;
code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
if (code != 0) {
return code;
}
// empty block returned, quit
if (blockReturned) {
return 0;
}
} else { // use the consumer specified offset
// the offset value can not be monotonious increase??
offset = reqOffset;
}
// this is a normal subscribe requirement
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
// todo handle the case where re-balance occurs.
// for taosx
SMqMetaRsp metaRsp = {0}; SMqMetaRsp metaRsp = {0};
STaosxRsp taosxRsp = {0}; STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, pRequest); tqInitTaosxRsp(&taosxRsp, pRequest);
if (offset.type != TMQ_OFFSET__LOG) { if (offset->type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &offset) < 0) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
return -1; return -1;
} }
if (metaRsp.metaRspLen > 0) { if (metaRsp.metaRspLen > 0) {
code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64,
",ts:%" PRId64, pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts);
consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
metaRsp.rspOffset.ts);
taosMemoryFree(metaRsp.metaRsp); taosMemoryFree(metaRsp.metaRsp);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
return code; return code;
} }
tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts);
if (taosxRsp.blockNum > 0) { if (taosxRsp.blockNum > 0) {
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
return code; return code;
} else { }else {
offset = taosxRsp.rspOffset; *offset = taosxRsp.rspOffset;
} }
tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
",version:%" PRId64,
consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,
taosxRsp.rspOffset.version);
} }
if (offset.type == TMQ_OFFSET__LOG) {
int64_t fetchVer = offset.version + 1; if (offset->type == TMQ_OFFSET__LOG) {
int64_t fetchVer = offset->version + 1;
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
if (pCkHead == NULL) { if (pCkHead == NULL) {
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
@ -578,12 +522,10 @@ static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* p
walSetReaderCapacity(pHandle->pWalReader, 2048); walSetReaderCapacity(pHandle->pWalReader, 2048);
int totalRows = 0; int totalRows = 0;
while (1) { while (1) {
// todo refactor: this is not correct.
int32_t savedEpoch = atomic_load_32(&pHandle->epoch); int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
if (savedEpoch > pRequest->epoch) { if (savedEpoch > pRequest->epoch) {
tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64
", found new consumer epoch %d, discard req epoch %d", ", found new consumer epoch %d, discard req epoch %d", pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch);
consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch);
break; break;
} }
@ -596,7 +538,7 @@ static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* p
} }
SWalCont* pHead = &pCkHead->head; SWalCont* pHead = &pCkHead->head;
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId,
pRequest->epoch, vgId, fetchVer, pHead->msgType); pRequest->epoch, vgId, fetchVer, pHead->msgType);
// process meta // process meta
@ -634,7 +576,7 @@ static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* p
}; };
if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) {
tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId, tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId,
pRequest->subKey); pRequest->subKey);
taosMemoryFreeClear(pCkHead); taosMemoryFreeClear(pCkHead);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
@ -658,6 +600,39 @@ static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* p
return 0; return 0;
} }
static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
int32_t code = -1;
STqOffsetVal offset = {0};
STqOffsetVal reqOffset = pRequest->reqOffset;
// 1. reset the offset if needed
if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
// handle the reset offset cases, according to the consumer's choice.
bool blockReturned = false;
code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
if (code != 0) {
return code;
}
// empty block returned, quit
if (blockReturned) {
return 0;
}
} else { // use the consumer specified offset
// the offset value can not be monotonious increase??
offset = reqOffset;
}
// this is a normal subscribe requirement
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
// todo handle the case where re-balance occurs.
// for taosx
return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq req = {0}; SMqPollReq req = {0};
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
@ -1557,6 +1532,8 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
return 0; return 0;
} else {
tDeleteStreamDispatchReq(&req);
} }
code = TSDB_CODE_STREAM_TASK_NOT_EXIST; code = TSDB_CODE_STREAM_TASK_NOT_EXIST;

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -238,7 +238,7 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6
if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { if (qStreamSetScanMemData(pTaskInfo, submit) != 0) {
return; return;
} }
qStreamSetOpen(pTaskInfo);
// here start to scan submit block to extract the subscribed data // here start to scan submit block to extract the subscribed data
int32_t totalRows = 0; int32_t totalRows = 0;

448
source/dnode/vnode/src/tq/tqRead.c
View File

@ -263,7 +263,7 @@ STqReader* tqOpenReader(SVnode* pVnode) {
pReader->pVnodeMeta = pVnode->pMeta; pReader->pVnodeMeta = pVnode->pMeta;
/*pReader->pMsg = NULL;*/ /*pReader->pMsg = NULL;*/
pReader->ver = -1; // pReader->ver = -1;
pReader->pColIdList = NULL; pReader->pColIdList = NULL;
pReader->cachedSchemaVer = 0; pReader->cachedSchemaVer = 0;
pReader->cachedSchemaSuid = 0; pReader->cachedSchemaSuid = 0;
@ -290,41 +290,26 @@ void tqCloseReader(STqReader* pReader) {
} }
// free hash // free hash
taosHashCleanup(pReader->tbIdHash); taosHashCleanup(pReader->tbIdHash);
tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE);
taosMemoryFree(pReader); taosMemoryFree(pReader);
} }
int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
// todo set the correct vgId
tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
if (walReadSeekVer(pReader->pWalReader, ver) < 0) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
return -1; return -1;
} else {
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
return 0;
} }
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
return 0;
} }
int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { void tqNextBlock(STqReader* pReader, SFetchRet* ret) {
bool fromProcessedMsg = pReader->msg2.msgStr != NULL;
while (1) { while (1) {
if (!fromProcessedMsg) { if (pReader->msg2.msgStr == NULL) {
if (walNextValidMsg(pReader->pWalReader) < 0) { if (walNextValidMsg(pReader->pWalReader) < 0) {
// pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
if(pReader->pWalReader->curInvalid == 0){
pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
}else{
pReader->ver = walGetLastVer(pReader->pWalReader->pWal);
}
ret->offset.type = TMQ_OFFSET__LOG;
ret->offset.version = pReader->ver;
ret->fetchType = FETCH_TYPE__NONE; ret->fetchType = FETCH_TYPE__NONE;
tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); return;
return -1;
} }
void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
int64_t ver = pReader->pWalReader->pHead->head.version; int64_t ver = pReader->pWalReader->pHead->head.version;
@ -333,109 +318,44 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
} }
while (tqNextDataBlock2(pReader)) { while (tqNextDataBlock2(pReader)) {
// TODO mem free
memset(&ret->data, 0, sizeof(SSDataBlock)); memset(&ret->data, 0, sizeof(SSDataBlock));
int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL);
if (code != 0 || ret->data.info.rows == 0) { if (code != 0 || ret->data.info.rows == 0) {
continue; continue;
} }
ret->fetchType = FETCH_TYPE__DATA; ret->fetchType = FETCH_TYPE__DATA;
tqDebug("return data rows %" PRId64, ret->data.info.rows); return;
return 0;
}
if (fromProcessedMsg) {
ret->offset.type = TMQ_OFFSET__LOG;
ret->offset.version = pReader->ver;
ret->fetchType = FETCH_TYPE__SEP;
tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version);
return 0;
} }
} }
} }
#if 0
int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) {
pReader->pMsg = pMsg;
// if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
// while (true) {
// if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1;
// tqDebug("submitnext vgId:%d, block:%p, dataLen:%d, len:%d, uid:%"PRId64, pWalReader->pWal->cfg.vgId, pReader->pBlock, pReader->msgIter.dataLen,
// pReader->msgIter.len, pReader->msgIter.uid);
// if (pReader->pBlock == NULL) break;
// }
if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
pReader->ver = ver;
memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter));
return 0;
}
#endif
int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0));
pReader->msg2.msgStr = msgStr; pReader->msg2.msgStr = msgStr;
pReader->msg2.msgLen = msgLen; pReader->msg2.msgLen = msgLen;
pReader->msg2.ver = ver; pReader->msg2.ver = ver;
pReader->ver = ver;
tqDebug("tq reader set msg %p %d", msgStr, msgLen); tqDebug("tq reader set msg %p %d", msgStr, msgLen);
SDecoder decoder;
if (pReader->setMsg == 0) { tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen);
SDecoder decoder; if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) {
tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen);
if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) {
ASSERT(0);
}
tDecoderClear(&decoder); tDecoderClear(&decoder);
pReader->setMsg = 1; tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver);
return -1;
} }
tDecoderClear(&decoder);
return 0; return 0;
} }
#if 0
bool tqNextDataBlock(STqReader* pReader) {
if (pReader->pMsg == NULL) return false;
while (1) {
if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) {
return false;
}
if (pReader->pBlock == NULL) {
pReader->pMsg = NULL;
return false;
}
if (pReader->tbIdHash == NULL) {
return true;
}
void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
/*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/
if (ret != NULL) {
/*tqDebug("find uid %" PRId64, pHandle->msgIter.uid);*/
return true;
}
}
return false;
}
#endif
bool tqNextDataBlock2(STqReader* pReader) { bool tqNextDataBlock2(STqReader* pReader) {
if (pReader->msg2.msgStr == NULL) { if (pReader->msg2.msgStr == NULL) {
return false; return false;
} }
ASSERT(pReader->setMsg == 1);
tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen,
pReader->msg2.ver, pReader->nextBlk);
int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < blockSz) { while (pReader->nextBlk < blockSz) {
tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen,
pReader->msg2.ver, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
ASSERT(pSubmitTbData->uid);
if (pReader->tbIdHash == NULL) return true; if (pReader->tbIdHash == NULL) return true;
void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
@ -446,7 +366,6 @@ bool tqNextDataBlock2(STqReader* pReader) {
} }
tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->setMsg = 0;
pReader->nextBlk = 0; pReader->nextBlk = 0;
pReader->msg2.msgStr = NULL; pReader->msg2.msgStr = NULL;
@ -455,7 +374,6 @@ bool tqNextDataBlock2(STqReader* pReader) {
bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) {
if (pReader->msg2.msgStr == NULL) return false; if (pReader->msg2.msgStr == NULL) return false;
ASSERT(pReader->setMsg == 1);
int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < blockSz) { while (pReader->nextBlk < blockSz) {
@ -470,7 +388,6 @@ bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) {
} }
tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->setMsg = 0;
pReader->nextBlk = 0; pReader->nextBlk = 0;
pReader->msg2.msgStr = NULL; pReader->msg2.msgStr = NULL;
@ -506,63 +423,8 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap
return 0; return 0;
} }
#if 0
bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
}
if (pHandle->pBlock == NULL) return false;
void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t));
if (ret == NULL) {
return true;
}
}
return false;
}
int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) {
//
int32_t sversion = htonl(pReader->pBlock->sversion);
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
taosMemoryFree(pReader->pSchema);
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchema == NULL) {
tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
"), version %d, possibly dropped table",
pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
}
return 0;
}
#endif
int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) {
int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk);
ASSERT(pReader->nextBlk < blockSz);
tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
pReader->nextBlk++; pReader->nextBlk++;
@ -666,33 +528,27 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
int32_t targetIdx = 0; int32_t targetIdx = 0;
int32_t sourceIdx = 0; int32_t sourceIdx = 0;
while (targetIdx < colActual) { while (targetIdx < colActual) {
ASSERT(sourceIdx < numOfCols); if(sourceIdx >= numOfCols){
tqError("tqRetrieveDataBlock2 sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols);
goto FAIL;
}
SColData* pCol = taosArrayGet(pCols, sourceIdx); SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
SColVal colVal; SColVal colVal;
ASSERT(pCol->nVal == numOfRows); if(pCol->nVal != numOfRows){
tqError("tqRetrieveDataBlock2 pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows);
goto FAIL;
}
if (pCol->cid < pColData->info.colId) { if (pCol->cid < pColData->info.colId) {
sourceIdx++; sourceIdx++;
} else if (pCol->cid == pColData->info.colId) { } else if (pCol->cid == pColData->info.colId) {
for (int32_t i = 0; i < pCol->nVal; i++) { for (int32_t i = 0; i < pCol->nVal; i++) {
tColDataGetValue(pCol, i, &colVal); tColDataGetValue(pCol, i, &colVal);
#if 0
void* val = NULL;
if (IS_STR_DATA_TYPE(colVal.type)) {
val = colVal.value.pData;
} else {
val = &colVal.value.val;
}
if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
goto FAIL;
}
#endif
if (IS_STR_DATA_TYPE(colVal.type)) { if (IS_STR_DATA_TYPE(colVal.type)) {
if (colVal.value.pData != NULL) { if (colVal.value.pData != NULL) {
char val[65535 + 2]; char val[65535 + 2] = {0};
memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
varDataSetLen(val, colVal.value.nData); varDataSetLen(val, colVal.value.nData);
if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
@ -726,8 +582,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
for (int32_t j = 0; j < colActual; j++) { for (int32_t j = 0; j < colActual; j++) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j);
while (1) { while (1) {
ASSERT(sourceIdx < pTschema->numOfCols);
SColVal colVal; SColVal colVal;
tRowGet(pRow, pTschema, sourceIdx, &colVal); tRowGet(pRow, pTschema, sourceIdx, &colVal);
if (colVal.cid < pColData->info.colId) { if (colVal.cid < pColData->info.colId) {
@ -736,7 +590,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
} else if (colVal.cid == pColData->info.colId) { } else if (colVal.cid == pColData->info.colId) {
if (IS_STR_DATA_TYPE(colVal.type)) { if (IS_STR_DATA_TYPE(colVal.type)) {
if (colVal.value.pData != NULL) { if (colVal.value.pData != NULL) {
char val[65535 + 2]; char val[65535 + 2] = {0};
memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
varDataSetLen(val, colVal.value.nData); varDataSetLen(val, colVal.value.nData);
if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
@ -745,7 +599,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
} else { } else {
colDataSetNULL(pColData, i); colDataSetNULL(pColData, i);
} }
/*val = colVal.value.pData;*/
} else { } else {
if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
goto FAIL; goto FAIL;
@ -771,253 +624,6 @@ FAIL:
return -1; return -1;
} }
#if 0
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
// TODO: cache multiple schema
int32_t sversion = htonl(pReader->pBlock->sversion);
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->msgIter.uid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
pReader->cachedSchemaVer = sversion;
pReader->cachedSchemaSuid = pReader->msgIter.suid;
}
STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
if (colNumNeed == 0) {
int32_t colMeta = 0;
while (colMeta < pSchemaWrapper->nCols) {
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
goto FAIL;
}
colMeta++;
}
} else {
if (colNumNeed > pSchemaWrapper->nCols) {
colNumNeed = pSchemaWrapper->nCols;
}
int32_t colMeta = 0;
int32_t colNeed = 0;
while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
col_id_t colIdSchema = pColSchema->colId;
col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
if (colIdSchema < colIdNeed) {
colMeta++;
} else if (colIdSchema > colIdNeed) {
colNeed++;
} else {
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
goto FAIL;
}
colMeta++;
colNeed++;
}
}
}
if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
int32_t colActual = blockDataGetNumOfCols(pBlock);
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
int32_t curRow = 0;
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
pBlock->info.id.uid = pReader->msgIter.uid;
pBlock->info.rows = pReader->msgIter.numOfRows;
pBlock->info.version = pReader->pMsg->version;
pBlock->info.dataLoad = 1;
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row);
// get all wanted col of that block
for (int32_t i = 0; i < colActual; i++) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SCellVal sVal = {0};
if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) {
goto FAIL;
}
}
curRow++;
}
return 0;
FAIL:
blockDataFreeRes(pBlock);
return -1;
}
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) {
int32_t sversion = htonl(pReader->pBlock->sversion);
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->msgIter.uid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
pReader->cachedSchemaVer = sversion;
pReader->cachedSchemaSuid = pReader->msgIter.suid;
}
STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t colAtMost = pSchemaWrapper->nCols;
int32_t curRow = 0;
int32_t lastRow = 0;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
if (assigned == NULL) return -1;
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
bool buildNew = false;
tdSTSRowIterReset(&iter, row);
tqDebug("vgId:%d, row of block %d", pReader->pWalReader->pWal->cfg.vgId, curRow);
for (int32_t i = 0; i < colAtMost; i++) {
SCellVal sVal = {0};
if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) {
break;
}
tqDebug("vgId:%d, %d col, type %d", pReader->pWalReader->pWal->cfg.vgId, i, sVal.valType);
if (curRow == 0) {
assigned[i] = sVal.valType != TD_VTYPE_NONE;
buildNew = true;
} else {
bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE;
if (currentRowAssigned != assigned[i]) {
assigned[i] = currentRowAssigned;
buildNew = true;
}
}
}
if (buildNew) {
if (taosArrayGetSize(blocks) > 0) {
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
lastRow = curRow;
}
SSDataBlock* pBlock = createDataBlock();
SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
if (tqMaskBlock(pSW, pBlock, pSchemaWrapper, assigned) < 0) {
blockDataDestroy(pBlock);
goto FAIL;
}
SSDataBlock block = {0};
assignOneDataBlock(&block, pBlock);
blockDataDestroy(pBlock);
tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
(int32_t)taosArrayGetSize(block.pDataBlock));
taosArrayPush(blocks, &block);
taosArrayPush(schemas, &pSW);
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
pBlock->info.id.uid = pReader->msgIter.uid;
pBlock->info.rows = 0;
pBlock->info.version = pReader->pMsg->version;
tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId,
(int32_t)taosArrayGetSize(blocks));
if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
tdSTSRowIterReset(&iter, row);
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SCellVal sVal = {0};
if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
ASSERT(sVal.valType != TD_VTYPE_NONE);
if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
goto FAIL;
}
tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i,
sVal.valType == TD_VTYPE_NULL);
}
curRow++;
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
taosMemoryFree(assigned);
return 0;
FAIL:
taosMemoryFree(assigned);
return -1;
}
#endif
int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk);

121
source/dnode/vnode/src/tq/tqScan.c
View File

@ -71,37 +71,25 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
int32_t totalRows = 0; int32_t totalRows = 0;
const STqExecHandle* pExec = &pHandle->execHandle; const STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task; qTaskInfo_t task = pExec->task;
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
tqDebug("prepare scan failed, vgId:%d, consumer:0x%"PRIx64, vgId, pHandle->consumerId); tqError("prepare scan failed, return");
if (pOffset->type == TMQ_OFFSET__LOG) { return -1;
pRsp->rspOffset = *pOffset;
return code;
} else {
tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
tqDebug("prepare scan failed, vgId:%d, consumer:0x%"PRIx64, vgId, pHandle->consumerId);
pRsp->rspOffset = *pOffset;
return code;
}
}
} }
while (1) { while (1) {
SSDataBlock* pDataBlock = NULL; SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0; uint64_t ts = 0;
qStreamSetOpen(task);
tqDebug("vgId:%d, tmq task start to execute, consumer:0x%"PRIx64, vgId, pHandle->consumerId); tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq one task start execute", pHandle->consumerId, vgId);
if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) {
code = qExecTask(task, &pDataBlock, &ts); tqError("consumer:0x%"PRIx64" vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr());
if (code != TSDB_CODE_SUCCESS) { return -1;
tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(),
pHandle->consumerId);
return code;
} }
// current scan should be stopped ASAP, since the re-balance occurs. tqDebug("consumer:0x%"PRIx64" vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, pDataBlock);
// current scan should be stopped asap, since the rebalance occurs.
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
break; break;
} }
@ -113,37 +101,15 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
} }
pRsp->blockNum++; pRsp->blockNum++;
totalRows += pDataBlock->info.rows;
tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId, pHandle->consumerId, if (totalRows >= MAX_ROWS_TO_RETURN) {
pDataBlock->info.rows, pRsp->blockNum); break;
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
totalRows += pDataBlock->info.rows;
if (totalRows >= MAX_ROWS_TO_RETURN) {
break;
}
} }
} }
tqDebug("consumer:0x%"PRIx64" vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", pHandle->consumerId, vgId, pRsp->blockNum, totalRows);
qStreamExtractOffset(task, &pRsp->rspOffset); qStreamExtractOffset(task, &pRsp->rspOffset);
return 0;
if (pRsp->rspOffset.type == 0) {
code = TSDB_CODE_INVALID_PARA;
tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type,
pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version);
return code;
}
if (pRsp->withTbName || pRsp->withSchema) {
code = TSDB_CODE_INVALID_PARA;
tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema);
return code;
}
tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, total blocks:%d, rows:%d", vgId, pHandle->consumerId,
pRsp->blockNum, totalRows);
return code;
} }
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
@ -151,18 +117,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
qTaskInfo_t task = pExec->task; qTaskInfo_t task = pExec->task;
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
tqDebug("prepare scan failed, return"); tqDebug("tqScanTaosx prepare scan failed, return");
if (pOffset->type == TMQ_OFFSET__LOG) { return -1;
pRsp->rspOffset = *pOffset;
return 0;
} else {
tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
tqDebug("prepare scan failed, return");
pRsp->rspOffset = *pOffset;
return 0;
}
}
} }
int32_t rowCnt = 0; int32_t rowCnt = 0;
@ -208,42 +164,31 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
} }
} }
if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { // get meta
if (qStreamExtractPrepareUid(task) != 0) { SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
if (tmp->metaRspLen > 0) {
qStreamExtractOffset(task, &tmp->rspOffset);
*pMetaRsp = *tmp;
tqDebug("tmqsnap task get data");
break;
}
if (pDataBlock == NULL) {
qStreamExtractOffset(task, pOffset);
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
continue; continue;
} }
tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1);
pHandle->snapshotVer + 1); qStreamExtractOffset(task, &pRsp->rspOffset);
break; break;
} }
if (pRsp->blockNum > 0) { if (pRsp->blockNum > 0) {
tqDebug("tmqsnap task exec exited, get data"); tqDebug("tmqsnap task exec exited, get data");
qStreamExtractOffset(task, &pRsp->rspOffset);
break; break;
} }
SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
tqDebug("tmqsnap task exec change to get data");
continue;
}
*pMetaRsp = *tmp;
tqDebug("tmqsnap task exec exited, get meta");
tqDebug("task exec exited");
break;
}
qStreamExtractOffset(task, &pRsp->rspOffset);
if (pRsp->rspOffset.type == 0) {
tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts,
pRsp->rspOffset.uid, pRsp->rspOffset.version);
return -1;
} }
return 0; return 0;

8
source/libs/executor/inc/executorimpl.h
View File

@ -128,10 +128,10 @@ enum {
typedef struct { typedef struct {
// TODO remove prepareStatus // TODO remove prepareStatus
STqOffsetVal prepareStatus; // for tmq // STqOffsetVal prepareStatus; // for tmq
STqOffsetVal lastStatus; // for tmq STqOffsetVal currentOffset; // for tmq
SMqMetaRsp metaRsp; // for tmq fetching meta SMqMetaRsp metaRsp; // for tmq fetching meta
int8_t returned; // int8_t returned;
int64_t snapshotVer; int64_t snapshotVer;
// const SSubmitReq* pReq; // const SSubmitReq* pReq;
@ -191,7 +191,7 @@ enum {
OP_OPENED = 0x1, OP_OPENED = 0x1,
OP_RES_TO_RETURN = 0x5, OP_RES_TO_RETURN = 0x5,
OP_EXEC_DONE = 0x9, OP_EXEC_DONE = 0x9,
OP_EXEC_RECV = 0x11, // OP_EXEC_RECV = 0x11,
}; };
typedef struct SOperatorFpSet { typedef struct SOperatorFpSet {

29
source/libs/executor/src/executor.c
View File

@ -1022,15 +1022,9 @@ SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
return &pTaskInfo->streamInfo.metaRsp; return &pTaskInfo->streamInfo.metaRsp;
} }
int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) { void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
return pTaskInfo->streamInfo.prepareStatus.uid; memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal));
}
int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal));
return 0;
} }
int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) {
@ -1076,22 +1070,27 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
return 0; return 0;
} }
void qStreamSetOpen(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
pOperator->status = OP_NOT_OPENED;
}
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot; SOperatorInfo* pOperator = pTaskInfo->pRoot;
const char* id = GET_TASKID(pTaskInfo); const char* id = GET_TASKID(pTaskInfo);
pTaskInfo->streamInfo.prepareStatus = *pOffset; // if pOffset equal to current offset, means continue consume
pTaskInfo->streamInfo.returned = 0; if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) {
if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
return 0; return 0;
} }
if (subType == TOPIC_SUB_TYPE__COLUMN) { if (subType == TOPIC_SUB_TYPE__COLUMN) {
pOperator->status = OP_OPENED;
pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id);
if (pOperator == NULL) {
return -1;
}
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pScanInfo = pInfo->pTableScanOp->info;
STableScanBase* pScanBaseInfo = &pScanInfo->base; STableScanBase* pScanBaseInfo = &pScanInfo->base;
@ -1130,6 +1129,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} }
} }
qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows);
pInfo->pTableScanOp->resultInfo.totalRows = 0; pInfo->pTableScanOp->resultInfo.totalRows = 0;
// start from current accessed position // start from current accessed position
@ -1234,6 +1234,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
} }
} }
pTaskInfo->streamInfo.currentOffset = *pOffset;
return 0; return 0;
} }

6
source/libs/executor/src/executorimpl.c
View File

@ -1970,11 +1970,11 @@ void qStreamCloseTsdbReader(void* task) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task;
SOperatorInfo* pOp = pTaskInfo->pRoot; SOperatorInfo* pOp = pTaskInfo->pRoot;
qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.lastStatus.uid, qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid,
pTaskInfo->streamInfo.lastStatus.ts); pTaskInfo->streamInfo.currentOffset.ts);
// todo refactor, other thread may already use this read to extract data. // todo refactor, other thread may already use this read to extract data.
pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0};
while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) {
SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; SOperatorInfo* pDownstreamOp = pOp->pDownstream[0];
if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

28
source/libs/executor/src/projectoperator.c
View File

@ -227,17 +227,8 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
blockDataCleanup(pFinalRes); blockDataCleanup(pFinalRes);
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pTaskInfo->streamInfo.submit.msgStr) {
pOperator->status = OP_OPENED;
}
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {
pOperator->status = OP_OPENED;
qDebug("projection in queue model, set status open and return null");
return NULL;
}
return NULL; return NULL;
} }
@ -263,23 +254,14 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
// The downstream exec may change the value of the newgroup, so use a local variable instead. // The downstream exec may change the value of the newgroup, so use a local variable instead.
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows);
pOperator->status = OP_OPENED;
if (pOperator->status == OP_EXEC_RECV) {
continue;
} else {
return NULL;
}
}
qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status,
pFinalRes->info.rows);
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
break; break;
} }
if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { // if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {
qDebug("set status recv"); // qDebug("set status recv");
pOperator->status = OP_EXEC_RECV; // pOperator->status = OP_EXEC_RECV;
} // }
// for stream interval // for stream interval
if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT ||

117
source/libs/executor/src/scanoperator.c
View File

@ -707,9 +707,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
// todo refactor // todo refactor
/*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/
/*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
return pBlock; return pBlock;
} }
@ -899,9 +899,11 @@ static void destroyTableScanOperatorInfo(void* param) {
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
int32_t code = 0;
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error; goto _error;
} }
@ -909,8 +911,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo);
extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -1632,14 +1633,11 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
qDebug("start to exec queue scan, %s", id); qDebug("start to exec queue scan, %s", id);
if (pTaskInfo->streamInfo.submit.msgStr != NULL) { if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->tqReader->msg2.msgStr == NULL) {
SPackedData submit = pTaskInfo->streamInfo.submit; SPackedData submit = pTaskInfo->streamInfo.submit;
if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
qError("submit msg messed up when initing stream submit block %p, %s", submit.msgStr, id); qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
pInfo->tqReader->msg2 = (SPackedData){0}; return NULL;
pInfo->tqReader->setMsg = 0;
ASSERT(0);
} }
} }
@ -1662,73 +1660,49 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
} }
pInfo->tqReader->msg2 = (SPackedData){0}; pInfo->tqReader->msg2 = (SPackedData){0};
pInfo->tqReader->setMsg = 0;
pTaskInfo->streamInfo.submit = (SPackedData){0}; pTaskInfo->streamInfo.submit = (SPackedData){0};
return NULL; return NULL;
} }
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) { if (pResult && pResult->info.rows > 0) {
qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64" %s", pResult->info.rows, qDebug("queue scan tsdb return %"PRId64" rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows,
pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion, id); pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion);
pTaskInfo->streamInfo.returned = 1; tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey);
return pResult; return pResult;
} else {
// no data has return already, try to extract data in the WAL
if (!pTaskInfo->streamInfo.returned) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
qDebug("queue scan tsdb over, switch to wal ver:%" PRId64 " %s", pTaskInfo->streamInfo.snapshotVer + 1, id);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer);
return NULL;
}
} else {
return NULL;
}
} }
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
return NULL;
}
tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer);
} }
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {
while (1) { while (1) {
SFetchRet ret = {0}; SFetchRet ret = {0};
terrno = 0; tqNextBlock(pInfo->tqReader, &ret);
tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); //curVersion move to next, so currentOffset = curVersion - 1
if (tqNextBlock(pInfo->tqReader, &ret) < 0) {
// if the end is reached, terrno is 0
if (terrno != 0) {
qError("failed to get next log block since %s, %s", terrstr(), id);
}
}
if (ret.fetchType == FETCH_TYPE__DATA) { if (ret.fetchType == FETCH_TYPE__DATA) {
qDebug("doQueueScan get data from log %"PRId64" rows, version:%" PRId64, ret.data.info.rows, pTaskInfo->streamInfo.currentOffset.version);
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
setBlockIntoRes(pInfo, &ret.data, true); setBlockIntoRes(pInfo, &ret.data, true);
if (pInfo->pRes->info.rows > 0) { if (pInfo->pRes->info.rows > 0) {
pOperator->status = OP_EXEC_RECV; qDebug("doQueueScan get data from log %"PRId64" rows, return, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version);
qDebug("queue scan log return %" PRId64 " rows", pInfo->pRes->info.rows);
return pInfo->pRes; return pInfo->pRes;
} }
} else if (ret.fetchType == FETCH_TYPE__META) { }else if(ret.fetchType == FETCH_TYPE__NONE){
qError("unexpected ret.fetchType:%d", ret.fetchType); qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version);
continue;
} else if (ret.fetchType == FETCH_TYPE__NONE ||
(ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) {
pTaskInfo->streamInfo.lastStatus = ret.offset;
char formatBuf[80];
tFormatOffset(formatBuf, 80, &ret.offset);
qDebug("queue scan log return null, offset %s", formatBuf);
pOperator->status = OP_OPENED;
return NULL; return NULL;
} }
} }
} else { } else {
qError("unexpected streamInfo prepare type: %d %s", pTaskInfo->streamInfo.prepareStatus.type, id); qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type);
return NULL; return NULL;
} }
} }
@ -2083,7 +2057,6 @@ FETCH_NEXT_BLOCK:
int32_t current = pInfo->validBlockIndex++; int32_t current = pInfo->validBlockIndex++;
SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current);
/*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/
if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current,
totBlockNum); totBlockNum);
@ -2175,7 +2148,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; pTaskInfo->streamInfo.metaRsp.metaRsp = NULL;
qDebug("tmqsnap doRawScan called"); qDebug("tmqsnap doRawScan called");
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
bool hasNext = false; bool hasNext = false;
if (pInfo->dataReader) { if (pInfo->dataReader) {
code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); code = tsdbNextDataBlock(pInfo->dataReader, &hasNext);
@ -2197,28 +2170,23 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
} }
qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid);
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pBlock->info.id.uid, pBlock->info.window.ekey);
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
return pBlock; return pBlock;
} }
SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext);
STqOffsetVal offset = {0};
if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal
qDebug("tmqsnap read snapshot done, change to get data from wal"); qDebug("tmqsnap read snapshot done, change to get data from wal");
pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion);
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG;
pTaskInfo->streamInfo.lastStatus.version = pInfo->sContext->snapVersion;
} else { } else {
pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN);
pTaskInfo->streamInfo.prepareStatus.ts = INT64_MIN;
qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid);
qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType);
} }
qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
tDeleteSSchemaWrapper(mtInfo.schema); tDeleteSSchemaWrapper(mtInfo.schema);
qDebug("tmqsnap stream scan tsdb return null");
return NULL; return NULL;
} else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) { } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
SSnapContext* sContext = pInfo->sContext; SSnapContext* sContext = pInfo->sContext;
void* data = NULL; void* data = NULL;
int32_t dataLen = 0; int32_t dataLen = 0;
@ -2230,16 +2198,12 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
if (!sContext->queryMetaOrData) { // change to get data next poll request if (!sContext->queryMeta) { // change to get data next poll request
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; STqOffsetVal offset = {0};
pTaskInfo->streamInfo.lastStatus.uid = uid; tqOffsetResetToData(&offset, 0, INT64_MIN);
pTaskInfo->streamInfo.metaRsp.rspOffset.type = TMQ_OFFSET__SNAPSHOT_DATA; qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
pTaskInfo->streamInfo.metaRsp.rspOffset.uid = 0;
pTaskInfo->streamInfo.metaRsp.rspOffset.ts = INT64_MIN;
} else { } else {
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
pTaskInfo->streamInfo.lastStatus.uid = uid;
pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.lastStatus;
pTaskInfo->streamInfo.metaRsp.resMsgType = type; pTaskInfo->streamInfo.metaRsp.resMsgType = type;
pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen;
pTaskInfo->streamInfo.metaRsp.metaRsp = data; pTaskInfo->streamInfo.metaRsp.metaRsp = data;
@ -2453,7 +2417,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pHandle->initTableReader) { if (pHandle->initTableReader) {
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
pTaskInfo->streamInfo.lastStatus.uid = -1;
} }
if (pHandle->initTqReader) { if (pHandle->initTqReader) {

2
source/libs/parser/inc/sql.y
View File

@ -330,7 +330,7 @@ column_def_list(A) ::= column_def(B).
column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); }
column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); }
column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } //column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); }
%type type_name { SDataType } %type type_name { SDataType }
%destructor type_name { } %destructor type_name { }

6
source/libs/parser/src/parInsertUtil.c
View File

@ -470,11 +470,11 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SVgroupDataCxt* pVgCxt = NULL; SVgroupDataCxt* pVgCxt = NULL;
int32_t vgId = pTableCxt->pMeta->vgId; int32_t vgId = pTableCxt->pMeta->vgId;
void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId));
if (NULL == p) { if (NULL == pp) {
code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt);
} else { } else {
pVgCxt = *(SVgroupDataCxt**)p; pVgCxt = *(SVgroupDataCxt**)pp;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupDataCxt(pTableCxt, pVgCxt); code = fillVgroupDataCxt(pTableCxt, pVgCxt);

3592
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

10
source/libs/parser/test/parInitialCTest.cpp
View File

@ -742,10 +742,10 @@ TEST_F(ParserInitialCTest, createStable) {
addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE);
run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" run("CREATE STABLE IF NOT EXISTS rollup_db.t1("
"ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), "
"c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, "
"c13 NCHAR(30), c14 VARCHAR(50)) " "c13 NCHAR(30), c14 VARCHAR(50)) "
"TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, "
"a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, "
"a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) "
"TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m "
"DELETE_MARK 1000s,200m"); "DELETE_MARK 1000s,200m");
@ -1005,16 +1005,16 @@ TEST_F(ParserInitialCTest, createTable) {
run("CREATE TABLE IF NOT EXISTS test.t1(" run("CREATE TABLE IF NOT EXISTS test.t1("
"ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), "
"c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, "
"c13 NCHAR(30), c15 VARCHAR(50)) " "c13 NCHAR(30), c15 VARCHAR(50)) "
"TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)");
run("CREATE TABLE IF NOT EXISTS rollup_db.t1(" run("CREATE TABLE IF NOT EXISTS rollup_db.t1("
"ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), "
"c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, "
"c13 NCHAR(30), c14 VARCHAR(50)) " "c13 NCHAR(30), c14 VARCHAR(50)) "
"TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), "
"a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, "
"a14 NCHAR(30), a15 VARCHAR(50)) " "a14 NCHAR(30), a15 VARCHAR(50)) "
"TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)");

17
source/libs/wal/src/walRead.c
View File

@ -33,7 +33,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
pReader->pLogFile = NULL; pReader->pLogFile = NULL;
pReader->curVersion = -1; pReader->curVersion = -1;
pReader->curFileFirstVer = -1; pReader->curFileFirstVer = -1;
pReader->curInvalid = 1;
pReader->capacity = 0; pReader->capacity = 0;
if (cond) { if (cond) {
pReader->cond = *cond; pReader->cond = *cond;
@ -81,7 +80,6 @@ int32_t walNextValidMsg(SWalReader *pReader) {
wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64
", applied index:%" PRId64 ", end index:%" PRId64, ", applied index:%" PRId64 ", end index:%" PRId64,
pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer);
pReader->curStopped = 0;
while (fetchVer <= endVer) { while (fetchVer <= endVer) {
if (walFetchHeadNew(pReader, fetchVer) < 0) { if (walFetchHeadNew(pReader, fetchVer) < 0) {
return -1; return -1;
@ -99,7 +97,6 @@ int32_t walNextValidMsg(SWalReader *pReader) {
fetchVer = pReader->curVersion; fetchVer = pReader->curVersion;
} }
} }
pReader->curStopped = 1;
return -1; return -1;
} }
@ -196,17 +193,16 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
return -1; return -1;
} }
wDebug("vgId:%d, wal version reset from %" PRId64 "(invalid:%d) to %" PRId64, pReader->pWal->cfg.vgId, wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId,
pReader->curVersion, pReader->curInvalid, ver); pReader->curVersion, ver);
pReader->curVersion = ver; pReader->curVersion = ver;
pReader->curInvalid = 0;
return 0; return 0;
} }
int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal; SWal *pWal = pReader->pWal;
if (!pReader->curInvalid && ver == pReader->curVersion) { if (ver == pReader->curVersion) {
wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver);
return 0; return 0;
} }
@ -238,7 +234,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer); wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer);
if (pRead->curInvalid || pRead->curVersion != fetchVer) { if (pRead->curVersion != fetchVer) {
if (walReadSeekVer(pRead, fetchVer) < 0) { if (walReadSeekVer(pRead, fetchVer) < 0) {
// pRead->curVersion = fetchVer; // pRead->curVersion = fetchVer;
// pRead->curInvalid = 1; // pRead->curInvalid = 1;
@ -344,7 +340,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
return -1; return -1;
} }
if (pRead->curInvalid || pRead->curVersion != ver) { if (pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver); code = walReadSeekVer(pRead, ver);
if (code < 0) { if (code < 0) {
// pRead->curVersion = ver; // pRead->curVersion = ver;
@ -479,7 +475,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
taosThreadMutexLock(&pReader->mutex); taosThreadMutexLock(&pReader->mutex);
if (pReader->curInvalid || pReader->curVersion != ver) { if (pReader->curVersion != ver) {
if (walReadSeekVer(pReader, ver) < 0) { if (walReadSeekVer(pReader, ver) < 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
@ -575,7 +571,6 @@ void walReadReset(SWalReader *pReader) {
taosThreadMutexLock(&pReader->mutex); taosThreadMutexLock(&pReader->mutex);
taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pIdxFile);
taosCloseFile(&pReader->pLogFile); taosCloseFile(&pReader->pLogFile);
pReader->curInvalid = 1;
pReader->curFileFirstVer = -1; pReader->curFileFirstVer = -1;
pReader->curVersion = -1; pReader->curVersion = -1;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);

1
source/util/src/terror.c
View File

@ -628,6 +628,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is inval
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed")
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log")
// stream // stream
TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist")

2
tests/parallel_test/cases.task
View File

@ -94,7 +94,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py #,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py

18
tests/script/tsim/tmq/basic1.sim
View File

@ -111,8 +111,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -134,7 +134,7 @@ if $data[0][2] != $expectmsgcnt then
print expect $expectmsgcnt , actual $data02 print expect $expectmsgcnt , actual $data02
return -1 return -1
endi endi
if $data[0][3] != $expectmsgcnt then if $data[0][3] != $totalMsgOfStb then
return -1 return -1
endi endi
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
@ -183,8 +183,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -202,7 +202,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfCtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfCtb then if $data[0][3] != $totalMsgOfCtb then
@ -254,8 +254,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -273,7 +273,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfNtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfNtb then if $data[0][3] != $totalMsgOfNtb then

42
tests/script/tsim/tmq/basic1Of2Cons.sim
View File

@ -111,11 +111,11 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -142,14 +142,14 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfStb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_0 goto check_ok_0
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfStb then if $data[1][2] == $expectmsgcnt then
goto check_ok_0 goto check_ok_0
endi endi
endi endi
@ -214,10 +214,10 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -244,14 +244,14 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfCtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_2 goto check_ok_2
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_2 goto check_ok_2
endi endi
endi endi
@ -316,10 +316,10 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -346,14 +346,14 @@ if $data[1][1] == 0 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfNtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_4 goto check_ok_4
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_4 goto check_ok_4
endi endi
endi endi

18
tests/script/tsim/tmq/basic2.sim
View File

@ -83,8 +83,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum $totalMsgOfStb = $totalMsgOfStb * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -105,7 +105,7 @@ endi
if $data[0][2] != $expectmsgcnt then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $expectmsgcnt then if $data[0][3] != $totalMsgOfStb then
return -1 return -1
endi endi
@ -140,8 +140,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -159,7 +159,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfCtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfCtb then if $data[0][3] != $totalMsgOfCtb then
@ -197,8 +197,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -216,7 +216,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfNtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfNtb then if $data[0][3] != $totalMsgOfNtb then

42
tests/script/tsim/tmq/basic2Of2Cons.sim
View File

@ -82,10 +82,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum $totalMsgOfStb = $totalMsgOfStb * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -112,14 +112,14 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfStb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_0 goto check_ok_0
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfStb then if $data[1][2] == $expectmsgcnt then
goto check_ok_0 goto check_ok_0
endi endi
endi endi
@ -169,10 +169,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -199,14 +199,14 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfCtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_2 goto check_ok_2
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_2 goto check_ok_2
endi endi
endi endi
@ -256,10 +256,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -286,14 +286,14 @@ if $data[1][1] == 0 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfNtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_4 goto check_ok_4
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_4 goto check_ok_4
endi endi
endi endi

92
tests/script/tsim/tmq/basic2Of2ConsOverlap.sim
View File

@ -80,8 +80,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 3
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_stb_all $topicList = ' . topic_stb_all
@ -89,7 +89,7 @@ $topicList = $topicList . ,
$topicList = $topicList . topic_stb_function $topicList = $topicList . topic_stb_function
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -118,40 +118,40 @@ endi
# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. # $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][2] < $totalMsgOfOneTopic then #if $data[0][2] < $totalMsgOfOneTopic then
# return -1
#endi
if $data[0][2] > $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][2] > $totalMsgOfStb then #if $data[1][2] < $totalMsgOfOneTopic then
return -1 # return -1
endi #endi
if $data[1][2] < $totalMsgOfOneTopic then if $data[1][2] > $expectmsgcnt then
return -1
endi
if $data[1][2] > $totalMsgOfStb then
return -1 return -1
endi endi
$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb #$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfMsgCnt = $data[0][2] + $data[1][2] $sumOfMsgCnt = $data[0][2] + $data[1][2]
if $sumOfMsgCnt != $totalMsgCons then if $sumOfMsgCnt != $expectmsgcnt then
print total: $totalMsgCons print total: $expectmsgcnt
print sum: $sumOfMsgCnt print sum: $sumOfMsgCnt
return -1 return -1
endi endi
# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. # $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][3] < $totalMsgOfOneTopic then #if $data[0][3] < $totalMsgOfStb then
return -1 # return -1
endi #endi
if $data[0][3] > $totalMsgOfStb then #if $data[0][3] > $totalMsgOfStb then
return -1 # return -1
endi #endi
if $data[1][3] < $totalMsgOfOneTopic then #if $data[1][3] < $totalMsgOfStb then
return -1 # return -1
endi #endi
if $data[1][3] > $totalMsgOfStb then #if $data[1][3] > $totalMsgOfStb then
return -1 # return -1
endi #endi
$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfRows = $data[0][3] + $data[1][3] $sumOfRows = $data[0][3] + $data[1][3]
@ -189,15 +189,15 @@ $consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_ctb_function $topicList = ' . topic_ctb_function
$topicList = $topicList . , $topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all $topicList = $topicList . topic_ctb_all
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -226,24 +226,24 @@ endi
# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb
# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic # or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then if $data[0][2] == $topicNum then
if $data[1][2] == $totalMsgOfCtb then if $data[1][2] == 1 then
goto check_ok_0 goto check_ok_0
endi endi
elif $data[1][2] == $totalMsgOfOneTopic then elif $data[0][2] == 1 then
if $data[0][2] == $totalMsgOfCtb then if $data[1][2] == $topicNum then
goto check_ok_0 goto check_ok_0
endi endi
endi endi
return -1 return -1
check_ok_0: check_ok_0:
if $data[0][3] == $totalMsgOfOneTopic then if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == $totalMsgOfCtb then if $data[1][3] == $totalMsgOfOneTopic then
goto check_ok_1 goto check_ok_1
endi endi
elif $data[1][3] == $totalMsgOfOneTopic then elif $data[0][3] == $totalMsgOfOneTopic then
if $data[0][3] == $totalMsgOfCtb then if $data[1][3] == $totalMsgOfCtb then
goto check_ok_1 goto check_ok_1
endi endi
endi endi
@ -280,8 +280,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
$topicList = ' . topic_ntb_function $topicList = ' . topic_ntb_function
@ -289,7 +289,7 @@ $topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all $topicList = $topicList . topic_ntb_all
$topicList = $topicList . ' $topicList = $topicList . '
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -318,12 +318,12 @@ endi
# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb
# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic # or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == $totalMsgOfNtb then if $data[1][2] == 1 then
goto check_ok_2 goto check_ok_2
endi endi
elif $data[1][2] == $totalMsgOfOneTopic then elif $data[0][2] == 1 then
if $data[0][2] == $totalMsgOfNtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_2 goto check_ok_2
endi endi
endi endi
@ -334,8 +334,8 @@ if $data[0][3] == $totalMsgOfOneTopic then
if $data[1][3] == $totalMsgOfNtb then if $data[1][3] == $totalMsgOfNtb then
goto check_ok_3 goto check_ok_3
endi endi
elif $data[1][3] == $totalMsgOfOneTopic then elif $data[0][3] == $totalMsgOfNtb then
if $data[0][3] == $totalMsgOfNtb then if $data[1][3] == $totalMsgOfOneTopic then
goto check_ok_3 goto check_ok_3
endi endi
endi endi

24
tests/script/tsim/tmq/basic3.sim
View File

@ -111,8 +111,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -130,10 +130,10 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $expectmsgcnt then #if $data[0][2] != $expectmsgcnt then
return -1 # return -1
endi #endi
if $data[0][3] != $expectmsgcnt then if $data[0][3] != $totalMsgOfStb then
return -1 return -1
endi endi
$loop_cnt = $loop_cnt + 1 $loop_cnt = $loop_cnt + 1
@ -182,8 +182,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -201,7 +201,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfCtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfCtb then if $data[0][3] != $totalMsgOfCtb then
@ -253,8 +253,8 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -272,7 +272,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfNtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfNtb then if $data[0][3] != $totalMsgOfNtb then

40
tests/script/tsim/tmq/basic3Of2Cons.sim
View File

@ -110,10 +110,10 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 3
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -163,19 +163,19 @@ endi
if $data[0][3] <= 0 then if $data[0][3] <= 0 then
return -1 return -1
endi endi
if $data[0][3] >= $expectmsgcnt then if $data[0][3] >= $totalMsgOfStb then
return -1 return -1
endi endi
if $data[1][3] <= 0 then if $data[1][3] <= 0 then
return -1 return -1
endi endi
if $data[1][3] >= $expectmsgcnt then if $data[1][3] >= $totalMsgOfStb then
return -1 return -1
endi endi
$sumOfMsgRows = $data[0][3] + $data[1][3] $sumOfMsgRows = $data[0][3] + $data[1][3]
if $sumOfMsgRows != $expectmsgcnt then if $sumOfMsgRows != $totalMsgOfStb then
return -1 return -1
endi endi
@ -225,10 +225,10 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -255,13 +255,13 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfCtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_0 goto check_ok_0
endi endi
elif $data[1][2] == $totalMsgOfCtb then elif $data[1][2] == $expectmsgcnt then
if $data[0][2] == 0 then if $data[0][2] == 0 then
goto check_ok_0 goto check_ok_0
endi endi
@ -327,10 +327,10 @@ endi
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -357,13 +357,13 @@ if $data[1][1] == 0 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfNtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_2 goto check_ok_2
endi endi
elif $data[1][2] == $totalMsgOfNtb then elif $data[1][2] == $expectmsgcnt then
if $data[0][2] == 0 then if $data[0][2] == 0 then
goto check_ok_2 goto check_ok_2
endi endi

18
tests/script/tsim/tmq/basic4.sim
View File

@ -80,8 +80,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum $totalMsgOfStb = $totalMsgOfStb * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 9
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -102,7 +102,7 @@ endi
if $data[0][2] != $expectmsgcnt then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $expectmsgcnt then if $data[0][3] != $totalMsgOfStb then
return -1 return -1
endi endi
@ -137,8 +137,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -156,7 +156,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfCtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfCtb then if $data[0][3] != $totalMsgOfCtb then
@ -194,8 +194,8 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -213,7 +213,7 @@ endi
if $data[0][1] != $consumerId then if $data[0][1] != $consumerId then
return -1 return -1
endi endi
if $data[0][2] != $totalMsgOfNtb then if $data[0][2] != $expectmsgcnt then
return -1 return -1
endi endi
if $data[0][3] != $totalMsgOfNtb then if $data[0][3] != $totalMsgOfNtb then

40
tests/script/tsim/tmq/basic4Of2Cons.sim
View File

@ -79,10 +79,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfStb * $topicNum $totalMsgOfStb = $totalMsgOfStb * $topicNum
$expectmsgcnt = $totalMsgOfStb $expectmsgcnt = 9
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from stb print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
@ -131,19 +131,19 @@ endi
if $data[0][3] <= 0 then if $data[0][3] <= 0 then
return -1 return -1
endi endi
if $data[0][3] >= $expectmsgcnt then if $data[0][3] >= $totalMsgOfStb then
return -1 return -1
endi endi
if $data[1][3] <= 0 then if $data[1][3] <= 0 then
return -1 return -1
endi endi
if $data[1][3] >= $expectmsgcnt then if $data[1][3] >= $totalMsgOfStb then
return -1 return -1
endi endi
$sumOfConsRow = $data[0][3] + $data[1][3] $sumOfConsRow = $data[0][3] + $data[1][3]
if $sumOfConsRow != $expectmsgcnt then if $sumOfConsRow != $totalMsgOfStb then
return -1 return -1
endi endi
@ -178,10 +178,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfCtb = $rowsPerCtb * $topicNum $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ctb print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -208,14 +208,14 @@ if $data[0][1] == 1 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfCtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_0 goto check_ok_0
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_0 goto check_ok_0
endi endi
endi endi
@ -266,10 +266,10 @@ $topicList = $topicList . '
$consumerId = 0 $consumerId = 0
$totalMsgOfNtb = $rowsPerCtb * $topicNum $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb $expectmsgcnt = $topicNum
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
$consumerId = 1 $consumerId = 1
sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit )
print == start consumer to pull msgs from ntb print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
@ -296,14 +296,14 @@ if $data[1][1] == 0 then
endi endi
endi endi
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 # either $data[0][2] == $expectmsgcnt and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb # or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt
if $data[0][2] == $totalMsgOfNtb then if $data[0][2] == $expectmsgcnt then
if $data[1][2] == 0 then if $data[1][2] == 0 then
goto check_ok_2 goto check_ok_2
endi endi
elif $data[0][2] == 0 then elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then if $data[1][2] == $expectmsgcnt then
goto check_ok_2 goto check_ok_2
endi endi
endi endi

4
tests/system-test/7-tmq/subscribeStb.py
View File

@ -226,7 +226,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -307,7 +307,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)

14
tests/system-test/7-tmq/subscribeStb0.py
View File

@ -228,7 +228,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -303,7 +303,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -315,7 +315,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt/4: if totalConsumeRows < expectrowcnt/4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -333,7 +333,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt: if totalConsumeRows < expectrowcnt:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -386,7 +386,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -398,7 +398,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt/4: if totalConsumeRows < expectrowcnt/4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -416,7 +416,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != (expectrowcnt * (1 + 1/4)): if totalConsumeRows < (expectrowcnt * (1 + 1/4)):
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")

4
tests/system-test/7-tmq/subscribeStb1.py
View File

@ -233,7 +233,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -320,7 +320,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)

9
tests/system-test/7-tmq/subscribeStb2.py
View File

@ -233,7 +233,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume 0 processor") tdLog.info("start consume 0 processor")
pollDelay = 100 pollDelay = 10
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -251,6 +251,7 @@ class TDTestCase:
tdLog.info("start consume 1 processor") tdLog.info("start consume 1 processor")
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
tdLog.sleep(2)
tdLog.info("start one new thread to insert data") tdLog.info("start one new thread to insert data")
parameterDict['actionType'] = actionType.INSERT_DATA parameterDict['actionType'] = actionType.INSERT_DATA
@ -271,6 +272,7 @@ class TDTestCase:
tdLog.info("start consume 2 processor") tdLog.info("start consume 2 processor")
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
tdLog.sleep(2)
tdLog.info("start one new thread to insert data") tdLog.info("start one new thread to insert data")
parameterDict['actionType'] = actionType.INSERT_DATA parameterDict['actionType'] = actionType.INSERT_DATA
@ -338,7 +340,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume 0 processor") tdLog.info("start consume 0 processor")
pollDelay = 100 pollDelay = 20
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -380,6 +382,7 @@ class TDTestCase:
tdLog.info("start consume 2 processor") tdLog.info("start consume 2 processor")
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
tdLog.sleep(2)
tdLog.info("start one new thread to insert data") tdLog.info("start one new thread to insert data")
parameterDict['actionType'] = actionType.INSERT_DATA parameterDict['actionType'] = actionType.INSERT_DATA
@ -394,7 +397,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt*2: if totalConsumeRows < expectrowcnt*2:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")

30
tests/system-test/7-tmq/subscribeStb3.py
View File

@ -215,7 +215,8 @@ class TDTestCase:
parameterDict["stbName"],\ parameterDict["stbName"],\
parameterDict["ctbNum"],\ parameterDict["ctbNum"],\
parameterDict["rowsPerTbl"],\ parameterDict["rowsPerTbl"],\
parameterDict["batchNum"]) parameterDict["batchNum"],\
parameterDict["startTs"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
@ -233,7 +234,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume 0 processor") tdLog.info("start consume 0 processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -269,7 +270,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt-10000: if totalConsumeRows < expectrowcnt-10000:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -328,7 +329,8 @@ class TDTestCase:
parameterDict["stbName"],\ parameterDict["stbName"],\
parameterDict["ctbNum"],\ parameterDict["ctbNum"],\
parameterDict["rowsPerTbl"],\ parameterDict["rowsPerTbl"],\
parameterDict["batchNum"]) parameterDict["batchNum"],\
parameterDict["startTs"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
@ -346,7 +348,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -415,7 +417,8 @@ class TDTestCase:
parameterDict["stbName"],\ parameterDict["stbName"],\
parameterDict["ctbNum"],\ parameterDict["ctbNum"],\
parameterDict["rowsPerTbl"],\ parameterDict["rowsPerTbl"],\
parameterDict["batchNum"]) parameterDict["batchNum"],\
parameterDict["startTs"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
@ -433,7 +436,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -445,7 +448,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt/4: if totalConsumeRows < expectrowcnt/4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -467,7 +470,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt/4: if totalConsumeRows < expectrowcnt/4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -502,7 +505,8 @@ class TDTestCase:
parameterDict["stbName"],\ parameterDict["stbName"],\
parameterDict["ctbNum"],\ parameterDict["ctbNum"],\
parameterDict["rowsPerTbl"],\ parameterDict["rowsPerTbl"],\
parameterDict["batchNum"]) parameterDict["batchNum"],\
parameterDict["startTs"])
tdLog.info("create topics from stb1") tdLog.info("create topics from stb1")
topicFromStb1 = 'topic_stb1' topicFromStb1 = 'topic_stb1'
@ -520,7 +524,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -532,7 +536,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt/4: if totalConsumeRows < expectrowcnt/4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
@ -555,7 +559,7 @@ class TDTestCase:
for i in range(expectRows): for i in range(expectRows):
totalConsumeRows += resultList[i] totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt*(1/2+1/4): if totalConsumeRows < expectrowcnt*(1/2+1/4):
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4))) tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4)))
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")

4
tests/system-test/7-tmq/subscribeStb4.py
View File

@ -231,7 +231,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -305,7 +305,7 @@ class TDTestCase:
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 5
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)

28
tests/system-test/7-tmq/tmqDelete-1ctb.py
View File

@ -53,7 +53,7 @@ class TDTestCase:
paraDict['rowsPerTbl'] = self.rowsPerTbl paraDict['rowsPerTbl'] = self.rowsPerTbl
tmqCom.initConsumerTable() tmqCom.initConsumerTable()
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
tdLog.info("create stb") tdLog.info("create stb")
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
tdLog.info("create ctb") tdLog.info("create ctb")
@ -237,10 +237,10 @@ class TDTestCase:
if self.snapshot == 0: if self.snapshot == 0:
consumerId = 2 consumerId = 2
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4)) expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
elif self.snapshot == 1: elif self.snapshot == 1:
consumerId = 3 consumerId = 3
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4))
topicList = topicFromStb1 topicList = topicFromStb1
ifcheckdata = 1 ifcheckdata = 1
@ -270,7 +270,7 @@ class TDTestCase:
if totalConsumeRows != expectrowcnt: if totalConsumeRows != expectrowcnt:
tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.exit("tmq consume rows error with snapshot = 0!")
elif self.snapshot == 1: elif self.snapshot == 1:
if totalConsumeRows != totalRowsFromQuery: if totalConsumeRows != expectrowcnt:
tdLog.exit("tmq consume rows error with snapshot = 1!") tdLog.exit("tmq consume rows error with snapshot = 1!")
# tmqCom.checkFileContent(consumerId, queryString) # tmqCom.checkFileContent(consumerId, queryString)
@ -323,7 +323,7 @@ class TDTestCase:
if self.snapshot == 0: if self.snapshot == 0:
consumerId = 4 consumerId = 4
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/4 + 3/4)) expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
elif self.snapshot == 1: elif self.snapshot == 1:
consumerId = 5 consumerId = 5
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))
@ -369,11 +369,7 @@ class TDTestCase:
tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt))
if self.snapshot == 0: if self.snapshot == 0:
# If data writing is completed before consumer get snapshot, will consume 7500 from wal; if (totalConsumeRows != expectrowcnt):
# If data writing has not started before consumer get snapshot, will consume 10000 from wal;
minRows = int(expectrowcnt * (1 - 1/4)) # 7500
tdLog.info("consume rows should be between %d and %d, "%(minRows, expectrowcnt))
if not ((totalConsumeRows >= minRows) and (totalConsumeRows <= expectrowcnt)):
tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.exit("tmq consume rows error with snapshot = 0!")
elif self.snapshot == 1: elif self.snapshot == 1:
tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt)) tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt))
@ -494,7 +490,7 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 4 end ...... ") tdLog.printNoPrefix("======== test case 4 end ...... ")
def run(self): def run(self):
# tdSql.prepare() tdSql.prepare()
tdLog.printNoPrefix("=============================================") tdLog.printNoPrefix("=============================================")
tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
self.snapshot = 0 self.snapshot = 0
@ -520,11 +516,11 @@ class TDTestCase:
self.prepareTestEnv() self.prepareTestEnv()
self.tmqCase3() self.tmqCase3()
tdLog.printNoPrefix("=============================================") # tdLog.printNoPrefix("=============================================")
tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
self.snapshot = 0 # self.snapshot = 0
self.prepareTestEnv() # self.prepareTestEnv()
self.tmqCase4() # self.tmqCase4()
tdLog.printNoPrefix("====================================================================") tdLog.printNoPrefix("====================================================================")
tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
self.snapshot = 1 self.snapshot = 1