diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 6b4d183764..7a416c5576 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -265,8 +265,14 @@ function install_config() { [ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0 /dev/null; then csudo="sudo" fi -#ulimit -c unlimited +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||: ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||: source /etc/profile -${csudo} mkdir -p /coredump ||: -${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||: -${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||: +${csudo} mkdir -p ${corePath} ||: +${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||: diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 2532cf09a7..a323050216 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -330,7 +330,7 @@ void bnReset() { tsAccessSquence = 0; } -static int32_t bnMonitorVgroups() { +static bool bnMonitorVgroups() { void * pIter = NULL; SVgObj *pVgroup = NULL; bool hasUpdatingVgroup = false; @@ -489,6 +489,7 @@ void bnCheckStatus() { mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, pDnode->lastAccess, pDnode->status); bnSetVgroupOffline(pDnode); + bnStartTimer(3000); } } mnodeDecDnodeRef(pDnode); diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index bf046a9fae..84f8694fca 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) { } pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); + mDebug("balance thread wakes up to work"); bool updateSoon = bnStart(); + mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); + bnStartTimer(updateSoon ? 1000 : -1); pthread_mutex_unlock(&(tsBnThread.mutex)); } @@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) { tsBnThread.timer = NULL; tsAccessSquence++; - bnCheckStatus(); bnStartTimer(-1); + bnCheckStatus(); if (handle == NULL) { if (tsAccessSquence % tsBalanceInterval == 0) { @@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) { bool updateSoon = (mseconds != -1); if (updateSoon) { + mTrace("balance function will be called after %" PRId64 " ms", mseconds); taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 97b962e53a..ad7041db10 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -388,10 +388,10 @@ void tscQueueAsyncRes(SSqlObj *pSql) { return; } + assert(pSql->res.code != TSDB_CODE_SUCCESS); tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL || pSql->fetchFp == NULL){ return; } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index f313d355f1..6afc5ba223 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2597,14 +2597,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////// +static void buildHistogramInfo(SAPercentileInfo* pInfo) { + pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); + pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); +} + static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - + SAPercentileInfo* pInfo = NULL; + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (SAPercentileInfo*) pCtx->aOutputBuf; + pInfo = (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return GET_ROWCELL_INTERBUF(pResInfo); + pInfo = GET_ROWCELL_INTERBUF(pResInfo); } + + buildHistogramInfo(pInfo); + return pInfo; } static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { @@ -2616,6 +2625,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); + printf("%p, %p\n", pInfo->pHisto, pInfo->pHisto->elems); return true; } @@ -2624,6 +2634,8 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); + + assert(pInfo->pHisto->elems != NULL); for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index a99918975e..da763b9b0c 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -911,6 +911,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, } } + if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) { + pRes->numOfRows = 0; + pBeforeFillData->num = 0; + pLocalReducer->discard = true; + return; + } + pRes->numOfRowsGroup += pRes->numOfRows; // impose the limitation of output rows on the final result diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 9ad38e3360..bb0d8005c2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -284,16 +284,18 @@ void taos_close(TAOS *taos) { return; } - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode - rpcCancelRequest(pHb->rpcRid); - pHb->rpcRid = -1; - } + if (RID_VALID(pObj->hbrid)) { + SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHb != NULL) { + if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode + rpcCancelRequest(pHb->rpcRid); + pHb->rpcRid = -1; + } - tscDebug("%p HB is freed", pHb); - taosReleaseRef(tscObjRef, pHb->self); - taos_free_result(pHb); + tscDebug("%p HB is freed", pHb); + taosReleaseRef(tscObjRef, pHb->self); + taos_free_result(pHb); + } } tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index e971a9c957..bf3f8b4943 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -101,7 +101,8 @@ extern int32_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; -extern int32_t tsFlowCtrl; +extern int32_t tsEnableFlowCtrl; +extern int32_t tsEnableSlaveQuery; // restful extern int32_t tsEnableHttpModule; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index cab234c787..af483dc8ff 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -138,7 +138,8 @@ int32_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400*100; // seconds 10days int32_t tsMnodeEqualVnodeNum = 4; -int32_t tsFlowCtrl = 1; +int32_t tsEnableFlowCtrl = 1; +int32_t tsEnableSlaveQuery = 1; // restful int32_t tsEnableHttpModule = 1; @@ -221,7 +222,7 @@ int32_t uDebugFlag = 131; int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; -int32_t tsdbDebugFlag = 131; +uint32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t (*monStartSystemFp)() = NULL; @@ -542,7 +543,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsOfflineThreshold; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 5; + cfg.minValue = 3; cfg.maxValue = 7200000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; @@ -1004,7 +1005,17 @@ static void doInitGlobalConfig(void) { // module configs cfg.option = "flowctrl"; - cfg.ptr = &tsFlowCtrl; + cfg.ptr = &tsEnableFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "slaveQuery"; + cfg.ptr = &tsEnableSlaveQuery; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 32e2c97a4c..ec77d9049a 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df +Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 656741ea16..b0908d2bd1 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -349,11 +349,12 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let num_of_rows = this.libtaos.taos_fetch_block(result, pblock) - if (num_of_rows == 0) { + //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data + let pblock = this.libtaos.taos_fetch_row(result); + if (pblock == null) { return {block:null, num_of_rows:0}; } + var fieldL = this.libtaos.taos_fetch_lengths(result); let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); @@ -361,7 +362,6 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { var fieldlens = []; if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { let plen = ref.reinterpret(fieldL, 4, i*4); let len = plen.readInt32LE(0); diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index b5c4997337..517a9e9bc8 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -113,6 +113,7 @@ static void dnodeCleanupTmr() { int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; + taosIgnSIGPIPE(); taosBlockSIGPIPE(); taosResolveCRC(); taosInitGlobalCfg(); @@ -120,7 +121,6 @@ int32_t dnodeInitSystem() { taosSetCoreDump(); taosInitNotes(); dnodeInitTmr(); - signal(SIGPIPE, SIG_IGN); if (dnodeCreateDir(tsLogDir) < 0) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 3f31e49370..ea738661ce 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -54,6 +54,7 @@ void dnodeCleanupVRead() { void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { int32_t queuedMsgNum = 0; int32_t leftLen = pMsg->contLen; + int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; char * pCont = pMsg->pCont; while (leftLen > 0) { @@ -64,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { assert(pHead->contLen > 0); void *pVnode = vnodeAcquire(pHead->vgId); if (pVnode != NULL) { - int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; vnodeRelease(pVnode); } @@ -74,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; rpcSendResponse(&rpcRsp); } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index a5ae8ac830..775ced0990 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { int32_t numOfMsgs; int32_t qtype; + taosBlockSIGPIPE(); dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 3dd0c58ae2..7f6a8d65e7 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -28,7 +28,7 @@ extern "C" { default: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ break; \ - }; + } #ifdef __cplusplus } diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 588d21574b..60707f22e2 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -332,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'N': arguments->data_batch = atoi(arg); + if (arguments->data_batch >= INT16_MAX) { + arguments->data_batch = INT16_MAX - 1; + } break; case 'L': { diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 8b3b2896ff..6549d58609 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -377,6 +377,24 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { return code; } +static bool mnodeAllOnline() { + void *pIter = NULL; + bool allOnline = true; + + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) { + allOnline = false; + mnodeDecMnodeRef(pMnode); + } + } + mnodeCancelGetNextMnode(pIter); + + return allOnline; +} + void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); pMnode->mnodeId = dnodeId; @@ -389,6 +407,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { .fpRsp = mnodeCreateMnodeCb }; + if (needConfirm && !mnodeAllOnline()) { + mDebug("wait all mnode online then create new mnode"); + return; + } + int32_t code = TSDB_CODE_SUCCESS; if (needConfirm) { code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1ab6a363e7..80a9978925 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -1081,6 +1081,8 @@ static void *sdbWorkerFp(void *pWorker) { int32_t qtype; void * unUsed; + taosBlockSIGPIPE(); + while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 8e9bf97496..689e4b6eb7 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -659,7 +659,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "onlineVnodes"); + strcpy(pSchema[cols].name, "onlines"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -674,13 +674,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index baf7687dd0..13d3fa4079 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -59,6 +59,7 @@ extern "C" { // TAOS_OS_FUNC_SOCKET int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c7c9d77427..729471247f 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() { + signal(SIGPIPE, SIG_IGN); +} + void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 3b091b2699..9697c5e65f 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -46,6 +46,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index d1278e2eee..201b3b2abc 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,13 +33,6 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SGroupResInfo { - int32_t groupId; - int32_t numOfDataPages; - int32_t pageId; - int32_t rowId; -} SGroupResInfo; - typedef struct SResultRowPool { int32_t elemSize; int32_t blockSize; @@ -72,6 +65,12 @@ typedef struct SResultRow { union {STimeWindow win; char* key;}; // start key of current time window } SResultRow; +typedef struct SGroupResInfo { + int32_t rowId; + int32_t index; + SArray* pRows; // SArray +} SGroupResInfo; + /** * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. @@ -89,7 +88,6 @@ typedef struct SResultRowInfo { int32_t size:24; // number of result set int32_t capacity; // max capacity int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key } SResultRowInfo; diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 442e61750b..7742d151a0 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramPrint(SHistogramInfo* pHisto); -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val); +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); SHeapEntry* tHeapCreate(int32_t numOfEntries); void tHeapSort(SHeapEntry* pEntry, int32_t len); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index dde2e39845..974d93f89c 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -77,7 +77,6 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); bool isPointInterpoQuery(SQuery *pQuery); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0d3b068a4a..2b992a931d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -53,7 +53,7 @@ #define TIME_WINDOW_COPY(_dst, _src) do {\ (_dst).skey = (_src).skey;\ (_dst).ekey = (_src).ekey;\ -} while (0); +} while (0) enum { // when query starts to execute, this status will set @@ -178,11 +178,10 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) // todo move to utility -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo); static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, @@ -195,7 +194,6 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); @@ -291,7 +289,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getMergeResultGroupId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getMergeResultGroupId(int32_t groupIndex) { int32_t base = 50000000; return base + (groupIndex * 10000); } @@ -466,16 +464,34 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = - (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (p1 != NULL) { - pResultRowInfo->curIndex = *p1; - } else { - if (!masterscan) { // not master scan, do not add new timewindow - return NULL; + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; } + if (p1 != NULL) { + for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { + if (pResultRowInfo->pResult[i] == (*p1)) { + pResultRowInfo->curIndex = i; + existed = true; + break; + } + } + } + } else { + if (p1 != NULL) { // group by column query + return *p1; + } + } + + if (!existed) { // TODO refactor // more than the capacity, reallocate the resources if (pResultRowInfo->size >= pResultRowInfo->capacity) { @@ -499,17 +515,23 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes pResultRowInfo->capacity = (int32_t)newCapacity; } - SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); - pResultRowInfo->pResult[pResultRowInfo->size] = pResult; - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + SResultRow *pResult = NULL; + + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + } else { + pResult = *p1; } - // add a new result set for a new group + pResultRowInfo->pResult[pResultRowInfo->size] = pResult; pResultRowInfo->curIndex = pResultRowInfo->size++; - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), - (char *)&pResultRowInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -591,7 +613,6 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf if (pData->num >= numOfRowsPerPage) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -614,24 +635,20 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SDataBlockInfo* pBockInfo, - STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow** pResult, int64_t groupId) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // todo refactor - int64_t uid = getResultInfoUId(pRuntimeEnv); - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); if (pResultRow == NULL) { - *newWind = false; - return masterscan? -1:0; // no master scan, no result generated means error occurs + *pResult = NULL; + return TSDB_CODE_SUCCESS; } - *newWind = true; - // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -701,81 +718,47 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static int32_t updateResultRowCurrentIndex(SResultRowInfo* pWindowResInfo, TSKEY lastKey, bool ascQuery) { - int32_t i = 0; +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery) { int64_t skey = TSKEY_INITIAL_VAL; - - int32_t numOfClosed = 0; - for (i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { - numOfClosed += 1; - continue; - } - - TSKEY ekey = pResult->win.ekey; - if ((ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pWindowResInfo, i); - } else { - skey = pResult->win.skey; break; } - } - // all windows are closed, set the last one to be the skey - if (skey == TSKEY_INITIAL_VAL) { - assert(i == pWindowResInfo->size); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - } else { - pWindowResInfo->curIndex = i; - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; - } - - return numOfClosed; -} - -/** - * NOTE: the query status only set for the first scan of master scan. - */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->scanFlag != MASTER_SCAN || pWindowResInfo->size == 0) { - return pWindowResInfo->size; - } - - // no qualified results exist, abort check - int32_t numOfClosed = 0; - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - // query completed - if ((lastKey >= pQuery->current->win.ekey && ascQuery) || (lastKey <= pQuery->current->win.ekey && (!ascQuery))) { - closeAllResultRows(pWindowResInfo); - - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); - } else { // set the current index to be the last unclosed window - numOfClosed = updateResultRowCurrentIndex(pWindowResInfo, lastKey, ascQuery); - - // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pQuery->rec.threshold) { - qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", - GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - - setQueryStatus(pQuery, QUERY_RESBUF_FULL); + // new closed result rows + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); } else { - qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, - numOfClosed); + skey = pResult->win.skey; } } - // output has reached the limitation, set query completed - if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && - pRuntimeEnv->scanFlag == MASTER_SCAN) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } + // all result rows are closed, set the last one to be the skey + if (skey == TSKEY_INITIAL_VAL) { + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { - assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); - return numOfClosed; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } + } + + pResultRowInfo->curIndex = i + 1; // current not closed result object + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; + } +} + +static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery) { + if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { + doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey, ascQuery); + } } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -818,52 +801,47 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, - int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool hasPrev = pCtx[0].preAggVals.isSet; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; - pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; + pCtx[k].size = forwardStep; + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; - } - - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } - - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunction(&pCtx[k]); - } - - // restore it - pCtx[k].preAggVals.isSet = hasPrev; + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; } + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { + pCtx[k].preAggVals.isSet = false; + } + + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); + } + + // restore it + pCtx[k].preAggVals.isSet = hasPrev; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); } } } @@ -1124,6 +1102,45 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY return ts; } +static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + if (!pRuntimeEnv->timeWindowInterpo) { + return; + } + + assert(pDataBlock != NULL); + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + } +} + /** * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions * @param pRuntimeEnv @@ -1139,7 +1156,9 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + TSKEY *tsCols = NULL; if (pDataBlock != NULL) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); @@ -1164,80 +1183,46 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { - tfree(sasArray); - return; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + goto _end; } int32_t forwardStep = 0; int32_t startPos = pQuery->pos; - // in case of repeat scan/reverse scan, no new time window added. - if (hasTimeWindow) { - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { - SResultRow *pRes = pWindowResInfo->pResult[j]; + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { + for(int32_t j = prevIndex; j < curIndex; ++j) { + SResultRow *pRes = pWindowResInfo->pResult[j]; - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); - } - - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code + doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows); } - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = pQuery->pos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; - - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } - } - - bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + assert (ret == TSDB_CODE_SUCCESS); } - int32_t index = pWindowResInfo->curIndex; - STimeWindow nextWin = win; + // window start key interpolation + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); @@ -1246,50 +1231,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (!hasTimeWindow) { - continue; - } - - TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); + ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1)*step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } - } - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } - pWindowResInfo->curIndex = index; } else { /* * the sqlfunctionCtx parameters should be set done before all functions are invoked, @@ -1305,6 +1259,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } } + _end: if (pRuntimeEnv->timeWindowInterpo) { saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); } @@ -1325,8 +1280,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - int32_t GROUPRESULTID = 1; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; // not assign result buffer yet, add new result buffer, TODO remove it @@ -1342,11 +1295,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - uint64_t uid = groupIndex; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); - if (pResultRow == NULL) { - return -1; - } + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); int64_t v = -1; GET_TYPED_DATA(v, int64_t, type, pData); @@ -1363,7 +1313,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } @@ -1498,7 +1448,7 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1565,6 +1515,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* item = pQuery->current; + int64_t groupId = item->groupIndex; + SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; @@ -1635,15 +1587,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - - if (!hasTimeWindow) { - continue; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code + goto _end; } // window start key interpolation @@ -1654,18 +1601,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); } // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, - &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1674,8 +1618,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1692,16 +1635,13 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (hasTimeWindow) { - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); + doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); } pWindowResInfo->curIndex = index; @@ -1736,6 +1676,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } + _end: assert(offset >= 0); if (tsCols != NULL) { item->lastKey = tsCols[offset] + step; @@ -1763,28 +1704,26 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQInfo = pQuery->current; - SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + STableQueryInfo* pTableQueryInfo = pQuery->current; + SResultRowInfo* pResultRowInfo = &pRuntimeEnv->windowResInfo; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } // update the lastkey of current table TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // interval query with limit applied int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); - } else if (pRuntimeEnv->groupbyNormalCol) { - closeAllResultRows(pWindowResInfo); - numOfRes = pWindowResInfo->size; + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + numOfRes = pResultRowInfo->size; + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery)); } else { // projection query - numOfRes = (int32_t)getNumOfResult(pRuntimeEnv); + numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); // update the number of output result if (numOfRes > 0 && pQuery->checkBuffer == 1) { @@ -1799,8 +1738,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { + if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { setQueryStatus(pQuery, QUERY_COMPLETED); } } @@ -2182,8 +2121,8 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { // todo refactor with isLastRowQuery bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_INTERP) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_INTERP) { return true; } } @@ -2618,6 +2557,8 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW *status = BLK_DATA_NO_NEEDED; SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + SQueryCostInfo* pCost = &pRuntimeEnv->summary; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { @@ -2634,15 +2575,13 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool hasTimeWindow = false; SResultRow* pResult = NULL; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } } @@ -2840,13 +2779,9 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } @@ -2916,13 +2851,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_COMPLETED); } - if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - closeAllResultRows(&pRuntimeEnv->windowResInfo); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window - } else { - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - } + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + closeAllResultRows(&pRuntimeEnv->windowResInfo); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window } return 0; @@ -3027,7 +2958,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { +static UNUSED_FUNC void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -3176,19 +3107,18 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; - int32_t * position; - SQInfo * pQInfo; + int32_t *rowIndex; + int32_t order; } SCompSupporter; int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t left = *(int32_t *)pLeft; + int32_t left = *(int32_t *)pLeft; int32_t right = *(int32_t *)pRight; SCompSupporter * supporter = (SCompSupporter *)param; - SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; - int32_t leftPos = supporter->position[left]; - int32_t rightPos = supporter->position[right]; + int32_t leftPos = supporter->rowIndex[left]; + int32_t rightPos = supporter->rowIndex[right]; /* left source is exhausted */ if (leftPos == -1) { @@ -3200,53 +3130,55 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; + STableQueryInfo** pList = supporter->pTableQueryInfo; + + SResultRowInfo *pWindowResInfo1 = &(pList[left]->windowResInfo); SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); + TSKEY leftTimestamp = pWindowRes1->win.skey; - char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); - TSKEY leftTimestamp = GET_INT64_VAL(b1); - - SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; + SResultRowInfo *pWindowResInfo2 = &(pList[right]->windowResInfo); SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); - - char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); - TSKEY rightTimestamp = GET_INT64_VAL(b2); + TSKEY rightTimestamp = pWindowRes2->win.skey; if (leftTimestamp == rightTimestamp) { return 0; } - return leftTimestamp > rightTimestamp ? 1 : -1; + if (supporter->order == TSDB_ORDER_ASC) { + return (leftTimestamp > rightTimestamp)? 1:-1; + } else { + return (leftTimestamp < rightTimestamp)? 1:-1; + } } -int32_t mergeIntoGroupResult(SQInfo *pQInfo) { +int32_t mergeGroupResult(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); - int32_t ret = TSDB_CODE_SUCCESS; + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - while (pQInfo->groupIndex < numOfGroups) { SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - ret = mergeIntoGroupResultImpl(pQInfo, group); - if (ret < 0) { // not enough disk space to save the data into disk + + int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo); + if (ret < 0) { return -1; } - pQInfo->groupIndex += 1; - // this group generates at least one result, return results - if (ret > 0) { + pQInfo->groupIndex += 1; + if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { break; } - assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); + taosArrayClear(pGroupResInfo->pRows); + + pGroupResInfo->index = 0; + pGroupResInfo->rowId = 0; } - SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { + if (pQInfo->groupIndex == numOfGroups && taosArrayGetSize(pGroupResInfo->pRows) == 0) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -3258,89 +3190,28 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { return TSDB_CODE_SUCCESS; } +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t* index, int32_t orderType); + void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - // all results have been return to client, try next group - if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { - pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - + // all results in current group have been returned to client, try next group + if (pGroupResInfo->index >= taosArrayGetSize(pGroupResInfo->pRows)) { // current results of group has been sent to client, try next group - if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (taosArrayGetSize(pGroupResInfo->pRows) == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } } - SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t id = pQInfo->groupResInfo.groupId; - SIDList list = getDataBufPagesIdList(pResultBuf, id); - - int32_t offset = 0; - int32_t numOfCopiedRows = 0; - - size_t size = taosArrayGetSize(list); - assert(size == pGroupResInfo->numOfDataPages); - - bool done = false; - - //TODO add API for release none-dirty pages -// SPageInfo* prev = NULL; - - for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { - SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); - - // release previous buffer pages -// if (prev == NULL) { -// prev = pi; -// } else { -// if (prev->pageId != pi->pageId) { -// releaseResBufPageInfo(pResultBuf, prev); -// prev = pi; -// } -// } - - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); - - if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->rowId += numOfCopiedRows; - done = true; - } else { - numOfCopiedRows = (int32_t)pData->num; - - pGroupResInfo->pageId += 1; - pGroupResInfo->rowId = 0; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; - - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, - (size_t)bytes * numOfCopiedRows); - } - - offset += numOfCopiedRows; - if (done) { - break; - } - } - - assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; + int32_t size = (int32_t) taosArrayGetSize(pGroupResInfo->pRows); + pQuery->rec.rows = doCopyToSData(pQInfo, pGroupResInfo->pRows->pData, (int32_t) size, &pGroupResInfo->index, TSDB_ORDER_ASC); } int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { @@ -3368,155 +3239,99 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResu return 0; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { +int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); - size_t size = taosArrayGetSize(pGroup); - tFilePage **buffer = pQuery->sdata; + int32_t code = TSDB_CODE_SUCCESS; - int32_t *posList = calloc(size, sizeof(int32_t)); - STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); + int32_t *posList = NULL; + SLoserTreeInfo *pTree = NULL; + STableQueryInfo **pTableQueryInfoList = NULL; - if (pTableList == NULL || posList == NULL) { - tfree(posList); - tfree(pTableList); - - qError("QInfo:%p failed alloc memory", pQInfo); - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + size_t size = taosArrayGetSize(pTableList); + if (pGroupResInfo->pRows == NULL) { + pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); + } + + posList = calloc(size, sizeof(int32_t)); + pTableQueryInfoList = malloc(POINTER_BYTES * size); + + if (pTableQueryInfoList == NULL || posList == NULL) { + qError("QInfo:%p failed alloc memory", pQInfo); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; } - // todo opt for the case of one table per group int32_t numOfTables = 0; - SIDList pageList = NULL; - int32_t tid = -1; - for (int32_t i = 0; i < size; ++i) { - STableQueryInfo *item = taosArrayGetP(pGroup, i); - - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); - if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; + STableQueryInfo *item = taosArrayGetP(pTableList, i); + if (item->windowResInfo.size > 0) { + pTableQueryInfoList[numOfTables++] = item; } } // there is no data in current group + // no need to merge results since only one table in each group if (numOfTables == 0) { - tfree(posList); - tfree(pTableList); - return 0; - } else if (numOfTables == 1) { // no need to merge results since only one table in each group - tfree(posList); - tfree(pTableList); - - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); - pGroupResInfo->groupId = tid; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - - return pGroupResInfo->numOfDataPages; + goto _end; } - SCompSupporter cs = {pTableList, posList, pQInfo}; + SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; - SLoserTreeInfo *pTree = NULL; - tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + if (ret != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } - SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - - pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); - - // todo add windowRes iterator - int64_t lastTimestamp = -1; + int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; int64_t startt = taosGetTimestampMs(); while (1) { if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - - tfree(pTableList); - tfree(posList); - tfree(pTree); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + code = TSDB_CODE_TSC_QUERY_CANCELLED; + goto _end; } - int32_t pos = pTree->pNode[0].index; + int32_t tableIndex = pTree->pNode[0].index; - SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->windowResInfo; + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); - char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); - TSKEY ts = GET_INT64_VAL(b); - - assert(ts == pWindowRes->win.skey); int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { - cs.position[pos] += 1; + cs.rowIndex[tableIndex] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; - - // all input sources are exhausted - if (--numOfTables == 0) { + if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + if (--numOfTables == 0) { // all input sources are exhausted break; } } } else { - if (ts == lastTimestamp) { // merge with the last one - doMerge(pRuntimeEnv, ts, pWindowRes, true); - } else { // copy data to disk buffer - if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - return -1; - } + assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - } - - doMerge(pRuntimeEnv, ts, pWindowRes, false); - buffer[0]->num += 1; + if (pWindowRes->win.skey != lastTimestamp) { + taosArrayPush(pGroupResInfo->pRows, &pWindowRes); + pWindowRes->numOfRows = (uint32_t) num; } - lastTimestamp = ts; + lastTimestamp = pWindowRes->win.skey; - // move to the next element of current entry - int32_t currentPageId = pWindowRes->pageId; - - cs.position[pos] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; + // move to the next row of current entry + if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; // all input sources are exhausted - if (--numOfTables == 0) { + if ((--numOfTables) == 0) { break; } - } else { - // current page is not needed anymore - SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pageId != currentPageId) { - releaseResBufPage(pRuntimeEnv->pResultBuf, page); - } } } - tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); - } - - if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - - tfree(pTree); - tfree(pTableList); - tfree(posList); - return -1; - } + tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); } int64_t endt = taosGetTimestampMs(); @@ -3527,65 +3342,16 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - tfree(pTableList); + _end: + tfree(pTableQueryInfoList); tfree(posList); tfree(pTree); -// tfree(pResultInfo); -// tfree(buf); - - return pQInfo->groupResInfo.numOfDataPages; -} - -int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. - int32_t pageId = -1; - int32_t capacity = pResultBuf->numOfRowsPerPage; - - int32_t remain = (int32_t) pQuery->sdata[0]->num; - int32_t offset = 0; - - while (remain > 0) { - int32_t rows = (remain > capacity)? capacity:remain; - assert(rows > 0); - - // get the output buffer page - tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); - buf->num = rows; - - // pagewisely copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - - char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; - char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, (size_t)(buf->num * bytes)); - } - - offset += rows; - remain -= rows; - - pGroupResInfo->numOfDataPages += 1; + if (code != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, code); } - return TSDB_CODE_SUCCESS; -} - -void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; - pCtx[k].size = 1; - pCtx[k].startOffset = 0; - pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); - - pQuery->sdata[k]->num = 0; - } + return code; } static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { @@ -3706,7 +3472,7 @@ void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t tid = 0; - int64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -3838,12 +3604,8 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); - if (!pResult->closed) { - continue; - } setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pExpr1[j].base.functionId; if (functId == TSDB_FUNC_TS) { @@ -4127,9 +3889,9 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; @@ -4142,12 +3904,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - uint64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); - if (pResultRow == NULL) { - return; - } + assert (pResultRow != NULL); /* * not assign result buffer yet, add new result buffer @@ -4293,7 +4053,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { /** * In handling the both ascending and descending order super table query, we need to find the first qualified * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; @@ -4302,7 +4062,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (!QUERY_IS_ASC_QUERY(pQuery)) { @@ -4341,36 +4100,33 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t *index, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfResult = 0; - int32_t startIdx = 0; + int32_t start = 0; int32_t step = -1; qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); - int32_t totalSet = numOfClosedResultRows(pResultInfo); - SResultRow** result = pResultInfo->pResult; - if (orderType == TSDB_ORDER_ASC) { - startIdx = pQInfo->groupIndex; + start = (*index); step = 1; } else { // desc order copy all data - startIdx = totalSet - pQInfo->groupIndex - 1; + start = numOfRows - (*index) - 1; step = -1; } SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i]->numOfRows == 0) { - pQInfo->groupIndex += 1; + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + if (pRows[i]->numOfRows == 0) { + (*index) += 1; pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; + int32_t numOfRowsToCopy = pRows[i]->numOfRows - pGroupResInfo->rowId; int32_t oldOffset = pGroupResInfo->rowId; /* @@ -4382,16 +4138,16 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_ pGroupResInfo->rowId += numOfRowsToCopy; } else { pGroupResInfo->rowId = 0; - pQInfo->groupIndex += 1; + (*index) += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRows[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, pRows[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4422,10 +4178,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType); + int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo->pResult, pResultInfo->size, &pQInfo->groupIndex, orderType); pQuery->rec.rows += numOfResult; - assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4457,25 +4212,17 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - // TODO refactor - if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) { - closeAllResultRows(pWindowResInfo); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - } else { - updateResultRowCurrentIndex(pWindowResInfo, pTableQueryInfo->lastKey, ascQuery); - } + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery)); } } @@ -4809,13 +4556,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { if (QUERY_IS_ASC_QUERY(pQuery)) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } } else { getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } @@ -5766,6 +5510,7 @@ static void doRestoreContext(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); if (pRuntimeEnv->pTsBuf != NULL) { SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); @@ -5805,9 +5550,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ if (QUERY_IS_INTERVAL_QUERY(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif } else { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); } @@ -5852,7 +5594,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { + if (mergeGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW @@ -6024,33 +5766,6 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) } } -static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, start); - - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - finalizeQueryResult(pRuntimeEnv); - - // here we can ignore the records in case of no interpolation - // todo handle offset, in case of top/bottom interval query - if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) && pQuery->limit.offset > 0 && - pQuery->fillType == TSDB_FILL_NONE) { - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); - - int32_t c = (int32_t)(MIN(numOfClosed, pQuery->limit.offset)); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, c); - pQuery->limit.offset -= c; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { - break; - } - } -} - // handle time interval query on table static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); @@ -6058,7 +5773,6 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; // skip blocks without load the actual data block from file if no filter condition present @@ -6070,59 +5784,39 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } } - while (1) { - tableIntervalProcessImpl(pRuntimeEnv, newStartKey); + scanOneTableDataBlocks(pRuntimeEnv, newStartKey); + assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - pQInfo->groupIndex = 0; // always start from 0 - pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + finalizeQueryResult(pRuntimeEnv); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); - } + // skip offset result rows + pQuery->rec.rows = 0; - // no result generated, abort - if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) { - break; - } - - doSecondaryArithmeticProcess(pQuery); - - // the offset is handled at prepare stage if no interpolation involved - if (pQuery->fillType == TSDB_FILL_NONE) { - limitResults(pRuntimeEnv); - break; - } else { - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); - taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata); - numOfFilled = 0; - - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - limitResults(pRuntimeEnv); - break; - } - - // no result generated yet, continue retrieve data - pQuery->rec.rows = 0; - } - } - - // all data scanned, the group by normal column can return - if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result - // maxOutput <= 0, means current query does not generate any results + if (pQuery->fillType == TSDB_FILL_NONE) { + // all data scanned, the group by normal column can return int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); + if (pQuery->limit.offset > numOfClosed) { + return; + } - if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) { - // skip offset result rows - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (int32_t) pQuery->limit.offset); + pQInfo->groupIndex = (int32_t) pQuery->limit.offset; - pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); - doSecondaryArithmeticProcess(pQuery); + limitResults(pRuntimeEnv); + } else { + + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); + + taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); + taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); + + int32_t numOfFilled = 0; + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); + + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); } } @@ -6133,7 +5827,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; if (queryHasRemainResForTableQuery(pRuntimeEnv)) { - if (pQuery->fillType != TSDB_FILL_NONE) { /* * There are remain results that are not returned due to result interpolation @@ -6150,23 +5843,23 @@ static void tableQueryImpl(SQInfo *pQInfo) { return; } else { pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; // always start from 0 + assert(pRuntimeEnv->windowResInfo.size > 0); - if (pRuntimeEnv->windowResInfo.size > 0) { + if (pQInfo->groupIndex < pRuntimeEnv->windowResInfo.size) { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); - - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); - - // there are not data remains - if (pRuntimeEnv->windowResInfo.size <= 0) { - qDebug("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); - } - - return; - } } + + if (pQuery->rec.rows > 0) { + qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, + pQuery->rec.total); + } + + // there are not data remains + if (pQuery->rec.rows <= 0 || pRuntimeEnv->windowResInfo.size <= pQInfo->groupIndex) { + qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); + } + + return; } } @@ -7284,6 +6977,8 @@ static void freeQInfo(SQInfo *pQInfo) { tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosHashCleanup(pQInfo->arrTableIdInfo); + taosArrayDestroy(pQInfo->groupResInfo.pRows); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -7703,7 +7398,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->completed = 1; // notify no more result to client } else { *continueExec = true; - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); + qDebug("QInfo:%p has more results to retrieve", pQInfo); } return pQInfo->code; @@ -7981,9 +7676,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { SQueryMgmt* pQueryMgmt = pQMgmt; qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); pQueryMgmt->closed = true; -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } @@ -8021,9 +7716,9 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); if (pQueryMgmt->closed) { -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; @@ -8031,7 +7726,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), (getMaximumIdleDurationSec()*1000)); -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); return handle; } @@ -8068,6 +7763,4 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; -} - - +} \ No newline at end of file diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 35e5906d1f..6ae6ede972 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -158,8 +158,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #if defined(USE_ARRAYLIST) - int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries); + int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); + assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -356,7 +356,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { return 0; } -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) { +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t end = len - 1; int32_t start = 0; @@ -466,7 +466,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) { */ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { #if defined(USE_ARRAYLIST) - int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v); + int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); if (pHisto->elems[slotIdx].val != v) { slotIdx -= 1; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 65ac60e91f..d09993ae4e 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -96,8 +96,6 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo pResultRowInfo->curIndex = -1; pResultRowInfo->size = 0; - - pResultRowInfo->startTime = TSKEY_INITIAL_VAL; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } @@ -110,7 +108,7 @@ void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow assert(num >= 0 && num <= numOfClosed); int16_t type = pResultRowInfo->type; - int64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; char *key = NULL; int16_t bytes = -1; @@ -181,11 +179,12 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]->closed) { + SResultRow* pRow = pResultRowInfo->pResult[i]; + if (pRow->closed) { continue; } - pResultRowInfo->pResult[i]->closed = true; + pRow->closed = true; } } @@ -383,18 +382,4 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; -} - -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { - if (!pRuntimeEnv->stableQuery) { - return 0; // for simple table query, the uid is always set to be 0; - } - - SQuery* pQuery = pRuntimeEnv->pQuery; - if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) { - return 0; - } - - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); - return id->uid; } \ No newline at end of file diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 4a5f7fbbbe..82fd4bcf99 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) { pHisto->elems[i].val = i; } - int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); assert(idx == 1); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); assert(idx == 9); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); assert(idx == 10); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); assert(idx == 0); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); assert(idx == 4); free(pHisto); diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index d855c651f9..47090cfa0c 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -38,7 +38,7 @@ extern "C" { #define SYNC_MAX_FWDS 512 #define SYNC_FWD_TIMER 300 #define SYNC_ROLE_TIMER 15000 // ms -#define SYNC_CHECK_INTERVAL 1 // ms +#define SYNC_CHECK_INTERVAL 1000 // ms #define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms #define nodeRole pNode->peerInfo[pNode->selfIndex]->role @@ -86,9 +86,10 @@ typedef struct SsyncPeer { int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process + int32_t refCount; + int64_t rid; void * timer; void * pConn; - int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -98,6 +99,7 @@ typedef struct SSyncNode { int8_t quorum; int8_t selfIndex; uint32_t vgId; + int32_t refCount; int64_t rid; SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator SSyncPeer * pMaster; @@ -121,13 +123,13 @@ extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; extern char * syncStatus[]; -void *syncRetrieveData(void *param); -void *syncRestoreData(void *param); -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int32_t syncDecPeerRef(SSyncPeer *pPeer); +void * syncRetrieveData(void *param); +void * syncRestoreData(void *param); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +SSyncPeer *syncAcquirePeer(int64_t rid); +void syncReleasePeer(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index 7db51f2a71..d4674fee6b 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -25,14 +25,14 @@ typedef struct { uint32_t serverIp; int16_t port; int32_t bufferSize; - void (*processBrokenLink)(void *ahandle); - int32_t (*processIncomingMsg)(void *ahandle, void *buffer); + void (*processBrokenLink)(int64_t handleId); + int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, void *ahandle, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 1cb2b8f302..fed0774346 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -29,8 +29,8 @@ static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(void *param); -static int32_t arbProcessPeerMsg(void *param, void *buffer); +static void arbProcessBrokenLink(int64_t rid); +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; static void * tsArbTcpPool; @@ -138,20 +138,20 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, arbitrator request is accepted", pNode->id); pNode->nodeFd = connFd; - pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, pNode, connFd); + pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd); return; } -static void arbProcessBrokenLink(void *param) { - SNodeConn *pNode = param; +static void arbProcessBrokenLink(int64_t rid) { + SNodeConn *pNode = (SNodeConn *)rid; sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); tfree(pNode); } -static int32_t arbProcessPeerMsg(void *param, void *buffer) { - SNodeConn *pNode = param; +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { + SNodeConn *pNode = (SNodeConn *)rid; SSyncHead head; int32_t bytes = 0; char * cont = (char *)buffer; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index bd9df64b7f..380e44780f 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -35,19 +35,21 @@ char tsNodeFqdn[TSDB_FQDN_LEN] = {0}; static void * tsTcpPool = NULL; static void * tsSyncTmrCtrl = NULL; static void * tsVgIdHash = NULL; -static int32_t tsSyncRefId = -1; +static int32_t tsNodeRefId = -1; +static int32_t tsPeerRefId = -1; // local functions static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(void *param); -static int32_t syncProcessPeerMsg(void *param, void *buffer); +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); +static void syncProcessBrokenLink(int64_t rid); +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); +static void syncFreePeer(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); static void syncMonitorNodeRole(void *param, void *tmrId); @@ -55,7 +57,12 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t c static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); +static void syncStartCheckPeerConn(SSyncPeer *pPeer); +static void syncStopCheckPeerConn(SSyncPeer *pPeer); +static SSyncNode *syncAcquireNode(int64_t rid); +static void syncReleaseNode(SSyncNode *pNode); char* syncRole[] = { "offline", @@ -87,29 +94,34 @@ int32_t syncInit() { tsTcpPool = syncOpenTcpThreadPool(&info); if (tsTcpPool == NULL) { sError("failed to init tcpPool"); + syncCleanUp(); return -1; } tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); - syncCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; + syncCleanUp(); return -1; } tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { sError("failed to init vgIdHash"); - taosTmrCleanUp(tsSyncTmrCtrl); - syncCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; - tsSyncTmrCtrl = NULL; + syncCleanUp(); return -1; } - tsSyncRefId = taosOpenRef(200, syncFreeNode); - if (tsSyncRefId < 0) { + tsNodeRefId = taosOpenRef(200, syncFreeNode); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + return -1; + } + + tsPeerRefId = taosOpenRef(1000, syncFreePeer); + if (tsPeerRefId < 0) { + sError("failed to init peer ref"); syncCleanUp(); return -1; } @@ -121,12 +133,12 @@ int32_t syncInit() { } void syncCleanUp() { - if (tsTcpPool) { + if (tsTcpPool != NULL) { syncCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; } - if (tsSyncTmrCtrl) { + if (tsSyncTmrCtrl != NULL) { taosTmrCleanUp(tsSyncTmrCtrl); tsSyncTmrCtrl = NULL; } @@ -136,8 +148,15 @@ void syncCleanUp() { tsVgIdHash = NULL; } - taosCloseRef(tsSyncRefId); - tsSyncRefId = -1; + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } + + if (tsPeerRefId != -1) { + taosCloseRef(tsPeerRefId); + tsPeerRefId = -1; + } sInfo("sync module is cleaned up"); } @@ -170,7 +189,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - pNode->rid = taosAddRef(tsSyncRefId, pNode); + pNode->refCount = 1; + pNode->rid = taosAddRef(tsNodeRefId, pNode); if (pNode->rid < 0) { syncFreeNode(pNode); return -1; @@ -232,13 +252,18 @@ int64_t syncStart(const SSyncInfo *pInfo) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + return pNode->rid; } void syncStop(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; sInfo("vgId:%d, cleanup sync", pNode->vgId); @@ -259,14 +284,14 @@ void syncStop(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); - taosRemoveRef(tsSyncRefId, rid); + syncReleaseNode(pNode); + taosRemoveRef(tsNodeRefId, rid); } int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { int32_t i, j; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, @@ -274,6 +299,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_lock(&pNode->mutex); + syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStopCheckPeerConn(pNode->peerInfo[index]); + } + for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && @@ -330,28 +360,32 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + pthread_mutex_unlock(&pNode->mutex); sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); return 0; } int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); - if (pNode == NULL) return 0; + SSyncNode *pNode = syncAcquireNode(rid); + if (pNode == NULL) return 0; int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return code; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; @@ -367,14 +401,14 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { } } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #if 0 void syncRecover(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; // to do: add a few lines to check if recover is OK @@ -395,12 +429,12 @@ void syncRecover(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; @@ -409,8 +443,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { pNodesRole->role[i] = pNode->peerInfo[i]->role; } - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return 0; } @@ -446,24 +479,61 @@ static void syncAddArbitrator(SSyncNode *pNode) { static void syncFreeNode(void *param) { SSyncNode *pNode = param; + int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); + sDebug("vgId:%d, syncnode is freed, refCount:%d", pNode->vgId, refCount); + pthread_mutex_destroy(&pNode->mutex); tfree(pNode->pRecv); tfree(pNode->pSyncFwds); tfree(pNode); } -void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); } - -int32_t syncDecPeerRef(SSyncPeer *pPeer) { - if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) { - taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); - - sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer); - return 0; +SSyncNode *syncAcquireNode(int64_t rid) { + SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sDebug("failed to acquire syncnode from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, acquire syncnode refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount); } - return 1; + return pNode; +} + +void syncReleaseNode(SSyncNode *pNode) { + int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, dec syncnode refId:%" PRId64 " refCount:%d", pNode->vgId, pNode->rid, refCount); + + taosReleaseRef(tsNodeRefId, pNode->rid); +} + +static void syncFreePeer(void *param) { + SSyncPeer *pPeer = param; + + int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); + sDebug("%s, peer is freed, refCount:%d", pPeer->id, refCount); + + syncReleaseNode(pPeer->pSyncNode); + tfree(pPeer); +} + +SSyncPeer *syncAcquirePeer(int64_t rid) { + SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid); + if (pPeer == NULL) { + sDebug("failed to acquire peer from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1); + sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount); + } + + return pPeer; +} + +void syncReleasePeer(SSyncPeer *pPeer) { + int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); + sTrace("%s, dec peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount); + + taosReleaseRef(tsPeerRefId, pPeer->rid); } static void syncClosePeerConn(SSyncPeer *pPeer) { @@ -473,7 +543,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { taosClose(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; - syncFreeTcpConn(pPeer->pConn); + void *pConn = pPeer->pConn; + if (pConn != NULL) syncFreeTcpConn(pPeer->pConn); } } @@ -482,7 +553,28 @@ static void syncRemovePeer(SSyncPeer *pPeer) { pPeer->ip = 0; syncClosePeerConn(pPeer); - syncDecPeerRef(pPeer); + //taosRemoveRef(tsPeerRefId, pPeer->rid); + syncReleasePeer(pPeer); +} + +static void syncStartCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; + + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); + if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { + int32_t checkMs = 100 + (pNode->vgId * 10) % 100; + if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); + taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); + } +} + +static void syncStopCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + + taosTmrStopA(&pPeer->timer); + sDebug("%s, stop check peer connection", pPeer->id); } static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { @@ -508,17 +600,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->role = TAOS_SYNC_ROLE_OFFLINE; pPeer->pSyncNode = pNode; pPeer->refCount = 1; + pPeer->rid = taosAddRef(tsPeerRefId, pPeer); - sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { - int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; - sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); - } + sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid); - taosAcquireRef(tsSyncRefId, pNode->rid); + (void)syncAcquireNode(pNode->rid); return pPeer; } @@ -560,6 +646,9 @@ static void syncChooseMaster(SSyncNode *pNode) { index = i; } } + sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); + } else { + sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); } // add arbitrator connection @@ -580,6 +669,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } } } + + if (index >= 0) { + sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, + pNode->peerInfo[index]->id, index, onlineNum, replica); + } } if (index >= 0) { @@ -621,9 +715,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) { + sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } else { + nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } (*pNode->notifyRole)(pNode->vgId, nodeRole); - sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { for (int32_t index = 0; index < pNode->replica; ++index) { @@ -678,7 +776,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; @@ -711,10 +809,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (consistent) { - sDebug("vgId:%d, choose master", pNode->vgId); + sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica); } } @@ -743,7 +841,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { sDebug("%s, check peer connection in 1000 ms", pPeer->id); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -772,25 +870,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } // start a new thread to retrieve the data - syncAddPeerRef(pPeer); + (void)syncAcquirePeer(pPeer->rid); + pthread_attr_t thattr; pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); if (ret != 0) { sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno)); - syncDecPeerRef(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); } + + syncReleasePeer(pPeer); } static void syncNotStarted(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -799,15 +902,22 @@ static void syncNotStarted(void *param, void *tmrId) { sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { @@ -823,7 +933,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { // Ensure the sync of mnode not interrupted if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -832,7 +942,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { SSyncMsg msg; syncBuildSyncReqMsg(&msg, pNode->vgId); - taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { sError("%s, failed to send sync-req to peer", pPeer->id); @@ -920,8 +1030,10 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) { return 0; } -static int32_t syncProcessPeerMsg(void *param, void *buffer) { - SSyncPeer *pPeer = param; +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return -1; + SSyncHead *pHead = buffer; SSyncNode *pNode = pPeer->pSyncNode; @@ -942,12 +1054,17 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { } pthread_mutex_unlock(&pNode->mutex); + syncReleasePeer(pPeer); return code; } -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - if (pPeer->peerFd < 0 || pPeer->ip == 0) return; +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { + if (pPeer->peerFd < 0 || pPeer->ip == 0) { + sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); + syncRestartConnection(pPeer); + return -1; + } SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus msg; @@ -970,9 +1087,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd); + return 0; } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); + return -1; } } @@ -989,7 +1108,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -1000,17 +1119,19 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); taosClose(connFd); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } static void syncCheckPeerConnection(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -1019,6 +1140,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { syncSetupPeerConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1029,8 +1152,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - syncAddPeerRef(pPeer); - int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + (void)syncAcquirePeer(pPeer->rid); + + int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); if (ret < 0) { @@ -1038,10 +1162,11 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { nodeSStatus = TAOS_SYNC_STATUS_INIT; sError("%s, failed to create sync thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); } else { sInfo("%s, sync connection is up", pPeer->id); } + + syncReleasePeer(pPeer); } static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { @@ -1073,7 +1198,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } - sDebug("vgId:%d, sync msg is received, tranId:%u", vgId, msg.tranId); + sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId); SSyncNode *pNode = *ppNode; pthread_mutex_lock(&pNode->mutex); @@ -1101,8 +1226,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); sDebug("%s, ready to exchange data", pPeer->id); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); } @@ -1111,23 +1235,21 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pthread_mutex_unlock(&pNode->mutex); } -static void syncProcessBrokenLink(void *param) { - if (param == NULL) return; // the connection for arbitrator - SSyncPeer *pPeer = param; +static void syncProcessBrokenLink(int64_t rid) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; - if (syncDecPeerRef(pPeer) != 0) { - syncRestartConnection(pPeer); - } - + syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, pNode->rid); + + syncReleasePeer(pPeer); } static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { @@ -1198,7 +1320,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code static void syncMonitorNodeRole(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; for (int32_t index = 0; index < pNode->replica; index++) { @@ -1215,12 +1337,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncFwds *pSyncFwds = pNode->pSyncFwds; @@ -1246,7 +1368,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 8651879eb6..e815594883 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -90,15 +90,18 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { break; } + sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, + minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic); + // remove extra files on slave between the current and last index syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); pindex = minfo.index; // check the file info sinfo = minfo; - sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); - sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, - &sinfo.fversion); + sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); + sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, sinfo.name, + sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic); // if file not there or magic is not the same, file shall be synced memset(&fileAck, 0, sizeof(SFileAck)); @@ -116,6 +119,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { if (fileAck.sync == 0) { sDebug("%s, %s is the same", pPeer->id, minfo.name); continue; + } else { + sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } // if sync is required, open file, receive from master, and write to file @@ -155,7 +160,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { return code; } -static int32_t syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; uint64_t lastVer = 0; @@ -198,6 +203,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } free(pHead); + *wver = lastVer; return code; } @@ -321,12 +327,19 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeVersion = fversion; - sInfo("%s, start to restore wal", pPeer->id); - if (syncRestoreWal(pPeer) < 0) { - sError("%s, failed to restore wal", pPeer->id); + sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); + uint64_t wver = 0; + code = syncRestoreWal(pPeer, &wver); // lastwar + if (code < 0) { + sError("%s, failed to restore wal, code:%d", pPeer->id, code); return -1; } + if (wver != 0) { + nodeVersion = wver; + sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion); + } + nodeSStatus = TAOS_SYNC_STATUS_CACHE; sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { @@ -338,7 +351,10 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return NULL; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); @@ -369,7 +385,7 @@ void *syncRestoreData(void *param) { taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); - syncDecPeerRef(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 02d990313e..f3e0a6d353 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -104,7 +104,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); syncBuildFileInfo(&fileInfo, pNode->vgId); - sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); + sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, fileInfo.name, + fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo)); @@ -144,6 +145,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; sDebug("%s, %s is the same", pPeer->id, fileInfo.name); continue; + } else { + sDebug("%s, %s will be sent", pPeer->id, fileInfo.name); } // get the full path to file @@ -461,7 +464,10 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return NULL; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); @@ -490,7 +496,7 @@ void *syncRetrieveData(void *param) { pPeer->fileChanged = 0; taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 7bfdc4e440..4744666737 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -42,7 +42,7 @@ typedef struct SPoolObj { typedef struct { SThreadObj *pThread; - void * ahandle; + int64_t handleId; int32_t fd; int32_t closedByApp; } SConnObj; @@ -112,7 +112,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -130,7 +130,7 @@ void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { pConn->fd = connFd; pConn->pThread = pThread; - pConn->ahandle = pPeer; + pConn->handleId = rid; pConn->closedByApp = 0; event.events = EPOLLIN | EPOLLRDHUP; @@ -164,7 +164,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { SPoolInfo * pInfo = &pPool->info; if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->ahandle); + (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); @@ -221,7 +221,7 @@ static void *syncProcessTcpData(void *param) { } if (pConn->closedByApp == 0) { - if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) { + if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) { syncFreeTcpConn(pConn); continue; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5c978abd1d..15da19d95d 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -31,19 +31,19 @@ extern "C" { #endif -extern int tsdbDebugFlag; +extern uint32_t tsdbDebugFlag; -#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} -#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} -#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} -#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} -#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} -#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define TSDB_MAX_TABLE_SCHEMAS 16 -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 637b02cd32..6b2ffe118e 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -161,6 +161,11 @@ _err: static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + SMemTable *pIMem = pRepo->imem; + tsdbLockRepo(pRepo); + pRepo->imem = NULL; + tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); sem_post(&(pRepo->readyToCommit)); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 999e2deb41..bedefa55ed 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -17,6 +17,7 @@ #include "tsdbMain.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 +#define TSDB_MAX_INSERT_BATCH 512 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); @@ -205,7 +206,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; - SMemTable *pIMem = pRepo->imem; + ASSERT(pRepo->imem == NULL); sem_wait(&(pRepo->readyToCommit)); @@ -220,8 +221,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { tsdbScheduleCommit(pRepo); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; - return 0; } @@ -606,19 +605,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; - void ** rows = NULL; + void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; ASSERT(pBlock->tid < pMeta->maxTables); pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - rows = (void **)calloc(pBlock->numOfRows, sizeof(void *)); - if (rows == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) { @@ -632,9 +625,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * if (rows[rowCounter] != NULL) { rowCounter++; } + + if (rowCounter == TSDB_MAX_INSERT_BATCH) { + if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + goto _err; + } + + rowCounter = 0; + memset(rows, 0, sizeof(rows)); + } } - if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { goto _err; } @@ -642,11 +644,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * pRepo->stat.pointsWritten += points * schemaNCols(pSchema); pRepo->stat.totalStorage += points * schemaVLen(pSchema); - free(rows); return 0; _err: - free(rows); return -1; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d5cc566b55..6368e62ef4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -956,9 +956,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } @@ -1704,7 +1704,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO return TSDB_CODE_SUCCESS; } -static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); + +static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + SQueryFilePos* cur = &pQueryHandle->cur; + + while(1) { + int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + // all data blocks in current file has been checked already, try next file if exists + return getFirstFileDataBlock(pQueryHandle, exists); + } else { // next block of the same file + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; + pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + } + } +} + +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { pQueryHandle->numOfBlocks = 0; SQueryFilePos* cur = &pQueryHandle->cur; @@ -1789,7 +1814,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists); + return getDataBlockRv(pQueryHandle, pBlockInfo, exists); +} + +static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { + assert(cur != NULL && numOfBlocks > 0); + return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); +} + +static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + + SQueryFilePos* cur = &pQueryHandle->cur; + assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; } static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { @@ -1800,14 +1841,14 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists if (!pQueryHandle->locateStart) { pQueryHandle->locateStart = true; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - return getDataBlocksInFilesImpl(pQueryHandle, exists); + return getFirstFileDataBlock(pQueryHandle, exists); } else { // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -1815,27 +1856,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - // all data blocks in current file has been checked already, try next file if exists - return getDataBlocksInFilesImpl(pQueryHandle, exists); - } else { - // next block of the same file - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->slot += step; - - cur->mixBlock = false; - cur->blockCompleted = false; - - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); - } + // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, + pQueryHandle->qinfo); int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = pQueryHandle->realNumOfRows > 0; + *exists = (pQueryHandle->realNumOfRows > 0); - return code; + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + } + + // current block is empty, try next block in file + // all data blocks in current file has been checked already, try next file if exists + if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { + return getFirstFileDataBlock(pQueryHandle, exists); + } else { + moveToNextDataBlockInCurrentFile(pQueryHandle); + STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pQueryHandle, pNext, exists); } } } diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index cd5092f30a..085c10c551 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid); // return the number of references in system int taosListRef(); +#define RID_VALID(x) ((x) > 0) + /* sample code to iterate the refs void demoIterateRefs(int rsetId) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2571f11ba4..379221e352 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { return; } + __cache_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; - pElem->prev = NULL; + pElem->prev = NULL; + pElem->next = NULL; pNode->inTrashcan = true; pNode->pTNodeHeader = pElem; - __cache_wr_lock(pCacheObj); pElem->next = pCacheObj->pTrash; if (pCacheObj->pTrash) { pCacheObj->pTrash->prev = pElem; @@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, - pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); + uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, + pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 7c950b7999..e3798162e8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -280,10 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - if (iter->cur == pSkipList->pTail) { + // no data in the skip list + if (iter->cur == pSkipList->pTail || iter->next == NULL) { + iter->cur = pSkipList->pTail; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it @@ -295,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->step++; } else { if (iter->cur == pSkipList->pHead) { + iter->cur = pSkipList->pHead; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 46068e8992..1a5c3bd64d 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,6 +18,10 @@ #include "tsocket.h" #include "taoserror.h" +#ifndef SIGPIPE + #define SIGPIPE EPIPE +#endif + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -115,6 +119,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nwritten; ptr += nwritten; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); @@ -142,6 +150,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nread; ptr += nread; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index aff7315d28..b0cc47d663 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -106,9 +106,10 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); - goto PARSE_VCFG_ERROR; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; } - vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index ee5aa5ca90..5cae7b7606 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -89,7 +89,10 @@ static void vnodeIncRef(void *ptNode) { } void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + SVnodeObj **ppVnode = NULL; + if (tsVnodesHash != NULL) { + ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + } if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 637d470f8a..a972ffec1c 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -65,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) { return TSDB_CODE_APP_NOT_READY; } - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica, - syncRole[pVnode->role], pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; + if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) { + return TSDB_CODE_SUCCESS; + } + + vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica, + syncRole[pVnode->role], pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; } void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { @@ -303,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { - void *h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + //void *h1 = qGetResultRetrieveMsg(*qhandle); + + /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */ + //assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 80e2dc422e..7cbe73fd45 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -297,7 +297,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; - if (tsFlowCtrl == 0) { + if (tsEnableFlowCtrl == 0) { int32_t ms = pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index aae81c19f4..e7d8a0b70f 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -50,12 +50,7 @@ pipeline { agent{label 'master'} steps { pre_test() - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/tests/pytest - python3 concurrent_inquiry.py -c 1 - ''' - } + sh ''' cd ${WKC}/tests ./test-all.sh b1 @@ -82,7 +77,37 @@ pipeline { ./handle_crash_gen_val_log.sh ''' } + sh''' + systemctl start taosd + sleep 10 + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single -DskipTests >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } sh ''' + systemctl stop taosd cd ${WKC}/tests ./test-all.sh b2 date @@ -95,6 +120,15 @@ pipeline { steps { pre_test() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + nohup taosd >/dev/null & + sleep 10 + python3 concurrent_inquiry.py -c 1 + + ''' + } sh ''' cd ${WKC}/tests/pytest ./valgrind-test.sh 2>&1 > mem-error-out.log @@ -106,41 +140,7 @@ pipeline { date''' } } - stage('connector'){ - agent{label "release"} - steps{ - sh''' - cd ${WORKSPACE} - git checkout develop - ''' - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/gotest - bash batchtest.sh - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# - dotnet run - ''' - } - - } - } + stage('arm64_build'){ agent{label 'arm64'} steps{ diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 9a8c359e4e..7bdab8bc67 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -388,7 +388,9 @@ class ConcurrentInquiry: print( "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) loop -= 1 if loop == 0: break @@ -415,7 +417,9 @@ class ConcurrentInquiry: print( "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) loop -= 1 if loop == 0: break diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index e2ce4b26fa..c6b857b097 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2224,22 +2224,25 @@ class ClientManager: if svcMgr: # gConfig.auto_start_service: svcMgr.stopTaosServices() svcMgr = None + + + # Release global variables + gConfig = None + gSvcMgr = None + logger = None + + thPool = None + dbManager.cleanUp() # destructor wouldn't run in time + dbManager = None + # Print exec status, etc., AFTER showing messages from the server self.conclude() # print("TC failed (2) = {}".format(self.tc.isFailed())) # Linux return code: ref https://shapeshed.com/unix-exit-codes/ ret = 1 if self.tc.isFailed() else 0 self.tc.cleanup() - - # Release global variables - gConfig = None - gSvcMgr = None - logger = None - # Release variables here self.tc = None - thPool = None - dbManager = None gc.collect() # force garbage collection # h = hpy() diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 855e18be55..dc072d7abc 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -394,6 +394,7 @@ class DbManager(): cType == 'native') else DbConn.createRest(dbTarget) try: self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + Logging.debug("DbManager opened DB connection...") except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) if (err.msg == 'client disconnected'): # cannot open DB connection @@ -412,6 +413,10 @@ class DbManager(): # Moved to Database() # self._stateMachine = StateMechine(self._dbConn) + def __del__(self): + ''' Release the underlying DB connection upon deletion of DbManager ''' + self.cleanUp() + def getDbConn(self): return self._dbConn @@ -438,5 +443,8 @@ class DbManager(): return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + if self._dbConn: + self._dbConn.close() + self._dbConn = None + Logging.debug("DbManager closed DB connection...") diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index a374ed943b..6ea5691ce2 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -2,6 +2,7 @@ import threading import random import logging import os +import sys import taos @@ -53,7 +54,7 @@ class Logging: # global misc.logger _logger = logging.getLogger('CrashGen') # real logger _logger.addFilter(LoggingFilter()) - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file _logger.addHandler(ch) # Logging adapter, to be used as a logger diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index fd12284b9d..de2d9b0780 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -19,5 +19,5 @@ if __name__ == "__main__": mExec.init() exitCode = mExec.run() - print("Exiting with code: {}".format(exitCode)) + print("\nCrash_Gen is now exiting with status code: {}".format(exitCode)) sys.exit(exitCode) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index f00fe40c69..cb37661789 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,9 +5,10 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & +#nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & +nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 -pidof taosd|xargs kill +pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` @@ -31,4 +32,4 @@ if [ -n "$defiMemError" ]; then exit 8 fi fi -done \ No newline at end of file +done diff --git a/tests/pytest/import_merge/importCacheFileH.py b/tests/pytest/import_merge/importCacheFileH.py index 3398f7bdad..33724dfa68 100644 --- a/tests/pytest/import_merge/importCacheFileH.py +++ b/tests/pytest/import_merge/importCacheFileH.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHO.py b/tests/pytest/import_merge/importCacheFileHO.py index 19520dc3d0..9f552978f9 100644 --- a/tests/pytest/import_merge/importCacheFileHO.py +++ b/tests/pytest/import_merge/importCacheFileHO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHPO.py b/tests/pytest/import_merge/importCacheFileHPO.py index 9e0a57fb46..f2d0a3ddbf 100644 --- a/tests/pytest/import_merge/importCacheFileHPO.py +++ b/tests/pytest/import_merge/importCacheFileHPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileS.py b/tests/pytest/import_merge/importCacheFileS.py index 2f0af569e5..250952b237 100644 --- a/tests/pytest/import_merge/importCacheFileS.py +++ b/tests/pytest/import_merge/importCacheFileS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileSub.py b/tests/pytest/import_merge/importCacheFileSub.py index 300bb6e8d0..663bdcd635 100644 --- a/tests/pytest/import_merge/importCacheFileSub.py +++ b/tests/pytest/import_merge/importCacheFileSub.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileT.py b/tests/pytest/import_merge/importCacheFileT.py index ab33cf6a93..f2feeeacff 100644 --- a/tests/pytest/import_merge/importCacheFileT.py +++ b/tests/pytest/import_merge/importCacheFileT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTO.py b/tests/pytest/import_merge/importCacheFileTO.py index 00e22da976..421ca6a755 100644 --- a/tests/pytest/import_merge/importCacheFileTO.py +++ b/tests/pytest/import_merge/importCacheFileTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTPO.py b/tests/pytest/import_merge/importCacheFileTPO.py index c6089e1d68..ca582d7d5a 100644 --- a/tests/pytest/import_merge/importCacheFileTPO.py +++ b/tests/pytest/import_merge/importCacheFileTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py index a21f0c47be..4de567d943 100644 --- a/tests/pytest/import_merge/importDataH2.py +++ b/tests/pytest/import_merge/importDataH2.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py index fdcaedd83c..f40de71d6d 100644 --- a/tests/pytest/import_merge/importDataHO.py +++ b/tests/pytest/import_merge/importDataHO.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO2.py b/tests/pytest/import_merge/importDataHO2.py index b094701132..518975e0b7 100644 --- a/tests/pytest/import_merge/importDataHO2.py +++ b/tests/pytest/import_merge/importDataHO2.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHPO.py b/tests/pytest/import_merge/importDataHPO.py index 9d74c0c352..c7f0625d4b 100644 --- a/tests/pytest/import_merge/importDataHPO.py +++ b/tests/pytest/import_merge/importDataHPO.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastH.py b/tests/pytest/import_merge/importDataLastH.py index c8e5f62423..f736cb925a 100644 --- a/tests/pytest/import_merge/importDataLastH.py +++ b/tests/pytest/import_merge/importDataLastH.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHO.py b/tests/pytest/import_merge/importDataLastHO.py index 33215997a4..b1eebfdd47 100644 --- a/tests/pytest/import_merge/importDataLastHO.py +++ b/tests/pytest/import_merge/importDataLastHO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHPO.py b/tests/pytest/import_merge/importDataLastHPO.py index fa8542f35b..9aa8127f50 100644 --- a/tests/pytest/import_merge/importDataLastHPO.py +++ b/tests/pytest/import_merge/importDataLastHPO.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py index 2f595fef54..2c1559d290 100644 --- a/tests/pytest/import_merge/importDataLastS.py +++ b/tests/pytest/import_merge/importDataLastS.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index bfcad2d252..415d1fcba2 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,7 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -60,9 +60,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/import_merge/importDataLastT.py b/tests/pytest/import_merge/importDataLastT.py index 08e944eb68..839320cc22 100644 --- a/tests/pytest/import_merge/importDataLastT.py +++ b/tests/pytest/import_merge/importDataLastT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTO.py b/tests/pytest/import_merge/importDataLastTO.py index a82c054141..0c35519fcf 100644 --- a/tests/pytest/import_merge/importDataLastTO.py +++ b/tests/pytest/import_merge/importDataLastTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTPO.py b/tests/pytest/import_merge/importDataLastTPO.py index ff75a1b2ae..c7864ae14d 100644 --- a/tests/pytest/import_merge/importDataLastTPO.py +++ b/tests/pytest/import_merge/importDataLastTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataS.py b/tests/pytest/import_merge/importDataS.py index 37627e8d6b..f0d56a5016 100644 --- a/tests/pytest/import_merge/importDataS.py +++ b/tests/pytest/import_merge/importDataS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py index 17e2b141b7..1643cc8915 100644 --- a/tests/pytest/import_merge/importDataSub.py +++ b/tests/pytest/import_merge/importDataSub.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataT.py b/tests/pytest/import_merge/importDataT.py index b0b7b82b79..c23ad9a605 100644 --- a/tests/pytest/import_merge/importDataT.py +++ b/tests/pytest/import_merge/importDataT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTO.py b/tests/pytest/import_merge/importDataTO.py index c0b57136af..b4c0d1a8ad 100644 --- a/tests/pytest/import_merge/importDataTO.py +++ b/tests/pytest/import_merge/importDataTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTPO.py b/tests/pytest/import_merge/importDataTPO.py index 8a1c9264b4..38e8cfe909 100644 --- a/tests/pytest/import_merge/importDataTPO.py +++ b/tests/pytest/import_merge/importDataTPO.py @@ -57,9 +57,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 20 data later with partly overlap") diff --git a/tests/pytest/import_merge/importHORestart.py b/tests/pytest/import_merge/importHORestart.py index f74c4c76d6..05b5b42aff 100644 --- a/tests/pytest/import_merge/importHORestart.py +++ b/tests/pytest/import_merge/importHORestart.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHPORestart.py b/tests/pytest/import_merge/importHPORestart.py index e5f79fbe6c..f916770913 100644 --- a/tests/pytest/import_merge/importHPORestart.py +++ b/tests/pytest/import_merge/importHPORestart.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHRestart.py b/tests/pytest/import_merge/importHRestart.py index be67039789..c1d50378b2 100644 --- a/tests/pytest/import_merge/importHRestart.py +++ b/tests/pytest/import_merge/importHRestart.py @@ -54,7 +54,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importInsertThenImport.py b/tests/pytest/import_merge/importInsertThenImport.py index 292fae8c47..1372177de3 100644 --- a/tests/pytest/import_merge/importInsertThenImport.py +++ b/tests/pytest/import_merge/importInsertThenImport.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.stop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdLog.info("import 100 sequential data again") diff --git a/tests/pytest/import_merge/importLastH.py b/tests/pytest/import_merge/importLastH.py index 17fa233e37..2887c71e04 100644 --- a/tests/pytest/import_merge/importLastH.py +++ b/tests/pytest/import_merge/importLastH.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHO.py b/tests/pytest/import_merge/importLastHO.py index adb44fc0ea..1681212b6b 100644 --- a/tests/pytest/import_merge/importLastHO.py +++ b/tests/pytest/import_merge/importLastHO.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHPO.py b/tests/pytest/import_merge/importLastHPO.py index d8ed2d9ef1..389b3c11ed 100644 --- a/tests/pytest/import_merge/importLastHPO.py +++ b/tests/pytest/import_merge/importLastHPO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastS.py b/tests/pytest/import_merge/importLastS.py index bf222a0d5f..e6393bbfae 100644 --- a/tests/pytest/import_merge/importLastS.py +++ b/tests/pytest/import_merge/importLastS.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastSub.py b/tests/pytest/import_merge/importLastSub.py index 5a6b9f4150..4ff6cf27bd 100644 --- a/tests/pytest/import_merge/importLastSub.py +++ b/tests/pytest/import_merge/importLastSub.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastT.py b/tests/pytest/import_merge/importLastT.py index 2b1be1fe2b..145b1bd690 100644 --- a/tests/pytest/import_merge/importLastT.py +++ b/tests/pytest/import_merge/importLastT.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTO.py b/tests/pytest/import_merge/importLastTO.py index ce189f6371..a159dac4ac 100644 --- a/tests/pytest/import_merge/importLastTO.py +++ b/tests/pytest/import_merge/importLastTO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTPO.py b/tests/pytest/import_merge/importLastTPO.py index 627d090855..453b922a68 100644 --- a/tests/pytest/import_merge/importLastTPO.py +++ b/tests/pytest/import_merge/importLastTPO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importSRestart.py b/tests/pytest/import_merge/importSRestart.py index 29f5a19b54..58f0f60e0a 100644 --- a/tests/pytest/import_merge/importSRestart.py +++ b/tests/pytest/import_merge/importSRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importSubRestart.py b/tests/pytest/import_merge/importSubRestart.py index b1a6f30c43..85594e1772 100644 --- a/tests/pytest/import_merge/importSubRestart.py +++ b/tests/pytest/import_merge/importSubRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTORestart.py b/tests/pytest/import_merge/importTORestart.py index 07eb6c28cb..10ac77d759 100644 --- a/tests/pytest/import_merge/importTORestart.py +++ b/tests/pytest/import_merge/importTORestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTPORestart.py b/tests/pytest/import_merge/importTPORestart.py index 10bbf3efce..ab86a0247f 100644 --- a/tests/pytest/import_merge/importTPORestart.py +++ b/tests/pytest/import_merge/importTPORestart.py @@ -68,7 +68,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTRestart.py b/tests/pytest/import_merge/importTRestart.py index 63a9368eca..6106bdd753 100644 --- a/tests/pytest/import_merge/importTRestart.py +++ b/tests/pytest/import_merge/importTRestart.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index da0ef96d0e..87635f86f3 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -16,7 +16,7 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql - +from util.dnodes import tdDnodes class TDTestCase: def init(self, conn, logSql): @@ -44,6 +44,25 @@ class TDTestCase: tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") tdSql.checkRows(1) + ## test case for https://jira.taosdata.com:18080/browse/TD-2488 + tdSql.execute("create table m1(ts timestamp, k int) tags(a int)") + tdSql.execute("create table t1 using m1 tags(1)") + tdSql.execute("create table t2 using m1 tags(2)") + tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)") + tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)") + tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)") + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/table/alter_wal0.py b/tests/pytest/table/alter_wal0.py index 15ad69998f..807dd316f2 100644 --- a/tests/pytest/table/alter_wal0.py +++ b/tests/pytest/table/alter_wal0.py @@ -43,7 +43,8 @@ class TDTestCase: print("==============step2") tdDnodes.stopAll() - filename = '/var/lib/taos/mnode/wal/wal0' + path = tdDnodes.getDnodesRootDir() + filename = path + '/dnode1/data/mnode/wal/wal0' with open(filename, 'rb') as f1: temp = f1.read() @@ -57,12 +58,12 @@ class TDTestCase: print("==============step3") tdSql.execute("use demo;") tdSql.query('show tables;') - tdSql.checkRows(10) + tdSql.checkRows(9) for i in range(11,21): tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i)) tdSql.query('show tables;') - tdSql.checkRows(20) + tdSql.checkRows(19) print("==============check table numbers and create 10 tables") diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index 9ef39fb556..cd3121057b 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -11,11 +11,26 @@ system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start - -sleep 3000 sql connect sql create dnode $hostname2 -sleep 1000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ============================ step1 @@ -80,10 +95,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -110,7 +138,6 @@ if $rows != 40 then return -1 endi - sql insert into db.t100 values(now, 1) sql insert into db.t101 values(now, 1) sql insert into db.t102 values(now, 1) @@ -144,10 +171,24 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -198,7 +239,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -268,10 +309,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -321,7 +375,7 @@ if $rows != 80 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -351,13 +405,26 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache -sleep 1000 +sleep 100 sql show db.tables if $rows != 80 then @@ -401,10 +468,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step1xx: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1xx +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1xx +endi sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) @@ -454,7 +534,7 @@ if $rows != 100 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then diff --git a/tests/script/general/db/alter_tables_v1.sim b/tests/script/general/db/alter_tables_v1.sim index dde5eb6d9e..20c4c73363 100644 --- a/tests/script/general/db/alter_tables_v1.sim +++ b/tests/script/general/db/alter_tables_v1.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -51,9 +49,22 @@ print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step2 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t10 using db.st tags(0) sql create table db.t11 using db.st tags(1) @@ -91,9 +102,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t20 using db.st tags(0) sql create table db.t21 using db.st tags(1) @@ -124,7 +148,7 @@ sql insert into db.t29 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t0 if $rows != 1 then @@ -148,9 +172,22 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step7 +endi sql reset query cache sleep 1000 @@ -190,9 +227,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t30 using db.st tags(0) sql create table db.t31 using db.st tags(1) @@ -249,12 +299,24 @@ if $rows != 40 then return -1 endi - print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -294,9 +356,22 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi sql create table db.t40 using db.st tags(0) sql create table db.t41 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index 7c9262874d..b57b2c0320 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -70,9 +68,22 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -131,9 +142,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -184,7 +208,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -250,9 +274,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -331,9 +368,22 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -378,9 +428,24 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi + + sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) sql create table db.t402 using db.st tags(2) diff --git a/tests/script/general/db/delete_reusevnode.sim b/tests/script/general/db/delete_reusevnode.sim index 5781c384fe..4364a760a6 100644 --- a/tests/script/general/db/delete_reusevnode.sim +++ b/tests/script/general/db/delete_reusevnode.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======== step1 @@ -38,7 +37,6 @@ if $rows != 0 then endi system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes diff --git a/tests/script/general/db/delete_writing1.sim b/tests/script/general/db/delete_writing1.sim index 93d41b2fa3..8b369b4e3d 100644 --- a/tests/script/general/db/delete_writing1.sim +++ b/tests/script/general/db/delete_writing1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index d3558560df..946dbe8835 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -144,4 +144,20 @@ if $data03 != 319 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ===================> TD-2488 +sql create table m1(ts timestamp, k int) tags(a int); +sql create table t1 using m1 tags(1); +sql create table t2 using m1 tags(2); +sql insert into t1 values('2020-1-1 1:1:1', 1); +sql insert into t1 values('2020-1-1 1:10:1', 2); +sql insert into t2 values('2020-1-1 1:5:1', 99); + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql select ts from m1 where ts='2020-1-1 1:5:1' +if $rows != 1 then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index b55d55eb8a..2a7fb09104 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -21,8 +21,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index c6f7402b87..3a89523918 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -31,7 +31,6 @@ system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 print ============== deploy system sh/exec.sh -n dnode1 -s start -sleep 5001 sql connect sql create dnode $hostname2 @@ -43,8 +42,8 @@ print =============== step1 $x = 0 show1: $x = $x + 1 - sleep 2000 - if $x == 5 then + sleep 1000 + if $x == 10 then return -1 endi sql show mnodes -x show1 @@ -82,7 +81,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 -sleep 1000 +sleep 100 sql select * from table_rest; print rows: $rows if $rows != 300000 then @@ -91,29 +90,51 @@ endi print =============== step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode1 -s start -x SIGINT -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi print =============== step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode2 -s start -x SIGINT -sleep 5000 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a2 +endi print =============== step5 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 34b057e71b..2fc355c40c 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -288,6 +288,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/data1.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim ./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt index 23d1e0a17d..70a621290b 100644 --- a/tests/script/jenkins/basic_3.txt +++ b/tests/script/jenkins/basic_3.txt @@ -1,38 +1,4 @@ -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -#./test.sh -f unique/mnode/mgmt25.sim -#./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - - - -./test.sh -f general/parser/stream_on_sys.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -79,3 +45,35 @@ ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +./test.sh -f unique/stable/balance_replica1.sim +./test.sh -f unique/stable/dnode2_stop.sim +./test.sh -f unique/stable/dnode2.sim +./test.sh -f unique/stable/dnode3.sim +./test.sh -f unique/stable/replica2_dnode4.sim +./test.sh -f unique/stable/replica2_vnode3.sim +./test.sh -f unique/stable/replica3_dnode6.sim +./test.sh -f unique/stable/replica3_vnode3.sim + +./test.sh -f unique/mnode/mgmt20.sim +./test.sh -f unique/mnode/mgmt21.sim +./test.sh -f unique/mnode/mgmt22.sim +./test.sh -f unique/mnode/mgmt23.sim +./test.sh -f unique/mnode/mgmt24.sim +#./test.sh -f unique/mnode/mgmt25.sim +#./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt33.sim +./test.sh -f unique/mnode/mgmt34.sim +./test.sh -f unique/mnode/mgmtr2.sim + +./test.sh -f general/parser/stream_on_sys.sim +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_n.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_1.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_n.sim +./test.sh -f general/stream/table_replica1_vnoden.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index b271f5b726..3be8123a42 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -31,6 +31,8 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cd2f3772eb..9b61a33d45 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -132,10 +132,12 @@ echo "cqdebugFlag 143" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index c647a35a5c..c9c85cb45d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -360,12 +360,14 @@ print ============== step7: stop dnode3/dnode2, and cluster unable to provide se system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s71 +s71: print ============== step8: restart dnode2, and cluster Still unable to provide services system sh/exec.sh -n dnode2 -s start sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s81 +s81: print ============== step9: restart dnode3, and cluster Resume service delivery system sh/exec.sh -n dnode3 -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 23ace47e3c..387ae4625f 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -45,15 +45,30 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 -$sleepTimer = 10000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi $db = db sql create database $db replica 1 @@ -84,7 +99,6 @@ while $i < $tblNum endw sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 @@ -92,118 +106,65 @@ endi print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep $sleepTimer - - -$loopCnt = 0 -wait_dnode2_offline_0: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline_0 -endi - system sh/exec.sh -n dnode2 -s start -sleep $sleepTimer +$x = 0 +a0: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -$loopCnt = 0 -wait_dnode2_reready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_reready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_reready +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a0 endi sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 endi - print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 -sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode3_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode3_ready -endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode3_ready +if $data4_3 != ready then + goto step2 +endi + +sql alter database $db replica 2 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -211,44 +172,22 @@ if $data00 != $totalRows then return -1 endi - print ============== step4: stop dnode2 for checking if sync ok system sh/exec.sh -n dnode2 -s stop -sleep $sleepTimer -$loopCnt = 0 -wait_dnode2_offline: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a2 endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_offline -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline -endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode2_offline -endi - -sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3 # check using select sql select count(*) from $stb print data00 $data00 @@ -258,38 +197,20 @@ endi print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -$loopCnt = 0 -wait_dnode2_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_ready -endi - -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -300,40 +221,39 @@ endi print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 - -sql alter database $db replica 3 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode4_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode4_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -#$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready +if $data4_4 != ready then + goto step6 +endi + +sql alter database $db replica 3 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -343,44 +263,20 @@ endi print ============== step7: alter replica from 3 to 2, and waiting sync sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_2: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +$x = 0 +a5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a5 endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 $data10_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 $data10_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 $data10_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data8_1 -$thirdDnode_3 = $data8_2 -$thirdDnode_4 = $data8_3 -$thirdDnode_5 = $data8_4 - -if $thirdDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi - -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -390,85 +286,20 @@ endi print ============== step8: alter replica from 2 to 1, and waiting sync sql alter database $db replica 1 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_1: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +$x = 0 +a6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a6 endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data6_1 -$sencodDnode_3 = $data6_2 -$sencodDnode_4 = $data6_3 -$sencodDnode_5 = $data6_4 - -if $sencodDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi - -$loopCnt = 0 -all_dnodes_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi - -sql show dnodes -if $rows != 4 then - sleep 2000 - goto all_dnodes_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode1Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode2Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode3Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode4Status != ready then - sleep 2000 - goto all_dnodes_ready -endi - -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -481,39 +312,31 @@ sql drop dnode $hostname2 sql drop dnode $hostname3 sleep $sleepTimer -$loopCnt = 0 -wait_dnode23_dropped: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes if $rows != 2 then - sleep 2000 - goto wait_dnode23_dropped + goto step9 endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -if $dnode2Status != null then - sleep 2000 - goto wait_dnode23_dropped -endi -if $dnode3Status != null then - sleep 2000 - goto wait_dnode23_dropped -endi -if $dnode4Status != ready then - return -1 +$x = 0 +a7: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a7 endi sleep $sleepTimer #waiting move vnode from dnode3/dnode3 to dnode4 @@ -523,3 +346,13 @@ print data00 $data00 if $data00 != $totalRows then return -1 endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index c36e9f015e..dd868e9eed 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -140,3 +140,11 @@ if $data00 != $totalRows then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 53209b0d5a..b3bd853ffc 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -158,13 +158,15 @@ if $dnode4Vtatus != offline then sleep 2000 goto wait_dnode4_vgroup_offline endi -if $dnode3Vtatus != unsynced then +if $dnode3Vtatus != master then sleep 2000 goto wait_dnode4_vgroup_offline endi -sql_error select count(*) from $stb -sql_error insert into $tb values (now, 9988) +sql select count(*) from $stb -x s31 +s31: +#sql_error insert into $tb values (now, 9988) -x s32 +#s32: print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index ba973cbe06..d4eb360efb 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -140,11 +140,9 @@ if $rows != 1 then goto wait_dnode4_vgroup_offline endi print show vgroups: -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data5_2 -$dnode3Vtatus = $data7_2 +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +$dnode4Vtatus = $data05 +$dnode3Vtatus = $data07 if $dnode4Vtatus != offline then sleep 2000 @@ -198,7 +196,3 @@ if $data00 != $totalRows then sleep 2000 goto wait_table_altered endi - - - - diff --git a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim index 3867aa3699..46d5e34fcb 100644 --- a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim +++ b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim @@ -14,6 +14,9 @@ sql alter table $stb add column f1 float $tblNum = $totalTableNum $alterTblNum = 10 +sql reset query cache +sleep 100 + $i = 1 while $i < $alterTblNum $tb = tb . $i diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index 91a7c538d2..f20939d458 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -21,7 +21,6 @@ system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000 print =============== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect $i = 0 @@ -92,8 +91,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show1: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -108,7 +107,7 @@ if $data2_2 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -143,7 +142,6 @@ endi print ========== step2 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 10000 print ========== step3 sql drop dnode $hostname2 @@ -151,8 +149,8 @@ sql drop dnode $hostname2 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -171,10 +169,9 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -212,8 +209,8 @@ sql drop dnode $hostname3 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -228,10 +225,9 @@ if $data2_3 != null then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -267,14 +263,31 @@ print ========== step5 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step5 +endi + sql alter database db replica 2 $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -289,7 +302,7 @@ if $data2_4 != 4 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -321,16 +334,14 @@ if $data00 != $totalNum then goto show5 endi - print ========== step6 -sleep 3000 sql alter database db replica 1 $x = 0 show6: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -345,7 +356,7 @@ if $data2_4 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 38cb667ec0..3c5cf92c7f 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -15,7 +15,6 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c debugFlag -v 131 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======================== dnode1 start @@ -52,7 +51,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step3 $i sql select count(*) from $mt print ===> $data00 $data01 @@ -83,7 +82,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step5 $i sql select count(*) from $mt where tgcol < 20200 print ===> $data00 $data01 diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index ec0c58d416..c82e96db50 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -84,16 +84,14 @@ endi print ============================== step2 print ========= start dnode2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 5000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -129,13 +127,12 @@ sql insert into c_b1_t3 values(1520000024031, 31) print ============================== step4 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -164,16 +161,14 @@ print dnode4 ==> $dnode4Role print ============================== step5 print ========= add dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -212,13 +207,12 @@ sql_error create dnode $hostname1 print ============================== step8 sql drop dnode $hostname3 -sleep 15000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -253,13 +247,12 @@ endi print ============================== step9 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show9 @@ -302,8 +295,8 @@ sql insert into c_b1_t4 values(1520000024041, 41) $x = 0 show10: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show10 @@ -335,7 +328,23 @@ sql_error create table c_b1_t5 (t timestamp, i int) -x error3 print ============================== step13 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 +$x = 0 +step13: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode4 $data4_5 + +if $data4_5 != ready then + goto step13 +endi sql show mnodes $dnode1Role = $data2_1 @@ -383,7 +392,23 @@ endi print ============================== step14 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 15000 +$x = 0 +step14: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode4 $data4_5 + +if $data4_6 != ready then + goto step14 +endi sql create database c_b1_d7 sql use c_b1_d7 @@ -406,7 +431,7 @@ sql insert into c_b1_t8 values(1520000024081, 81) $x = 0 show14: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -440,7 +465,7 @@ print ============================== step17 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql use c_b1_d1 sql select * from c_b1_d1.c_b1_t1 diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index b8477965c6..ffd13445ed 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -50,12 +50,48 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 4001 sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 4001 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + sql create database c_b2_d1 replica 2 sql use c_b2_d1 @@ -112,13 +148,12 @@ endi print ============================== step2 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show2 @@ -155,13 +190,12 @@ print ============================== step3 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 10000 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show3 @@ -209,13 +243,12 @@ endi print ============================== step4 print ========= drop dnode3 sql drop dnode $hostname3 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -266,12 +299,11 @@ print ============================== step5 print ========= start dnode3 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -304,7 +336,7 @@ print dnode5 ==> $dnode5Role print ============================== step6 system sh/exec.sh -n dnode1 -s stop -x SIGINT print stop dnode1 and sleep 10000 -sleep 10000 +sleep 5000 sql drop dnode $hostname1 print drop dnode1 and sleep 9000 @@ -364,7 +396,7 @@ print ============================== step12 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql select * from c_b2_d1.c_b2_t1 order by t desc print $data01 $data11 $data21 $data31 $data41 diff --git a/tests/script/unique/cluster/balance3.sim b/tests/script/unique/cluster/balance3.sim index e3b8125d8c..c2e9a84514 100644 --- a/tests/script/unique/cluster/balance3.sim +++ b/tests/script/unique/cluster/balance3.sim @@ -7,7 +7,6 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 system sh/deploy.sh -n dnode6 -i 6 system sh/deploy.sh -n dnode7 -i 7 -system sh/deploy.sh -n dnode8 -i 8 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 @@ -16,7 +15,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode8 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 @@ -25,7 +23,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 0 -system sh/cfg.sh -n dnode8 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 1 system sh/cfg.sh -n dnode2 -c wallevel -v 1 @@ -34,19 +31,52 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1 system sh/cfg.sh -n dnode5 -c wallevel -v 1 system sh/cfg.sh -n dnode6 -c wallevel -v 1 system sh/cfg.sh -n dnode7 -c wallevel -v 1 -system sh/cfg.sh -n dnode8 -c wallevel -v 1 print ============== step1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 2001 - sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3001 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi sql create database c_b3_d1 replica 3 sql use c_b3_d1 @@ -89,8 +119,6 @@ $dnode2Vnodes = $data2_2 print dnode2 $dnode2Vnodes $dnode3Vnodes = $data2_3 print dnode3 $dnode3Vnodes -$dnode4Vnodes = $data2_4 -print dnode4 $dnode4Vnodes if $dnode1Vnodes != 3 then goto show1 @@ -101,30 +129,22 @@ endi if $dnode3Vnodes != 3 then goto show1 endi -if $dnode4Vnodes != null then - goto show1 -endi sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step2 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -145,21 +165,16 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step3 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show3 @@ -189,25 +204,21 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 system sh/exec.sh -n dnode2 -s stop -x SIGINT print ============================== step4 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 10000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi + sql show dnodes -x show4 $dnode1Vnodes = $data2_1 print dnode1 $dnode1Vnodes @@ -228,8 +239,10 @@ print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 print dnode4 ==> $data2_4 print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 + +if $data2_4 != slave then + goto show4 +endi print ============================== step5 print ========= drop dnode3 @@ -239,8 +252,8 @@ sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -277,16 +290,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_5 != slave then + goto show5 +endi + print ============================== step6 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show6 @@ -299,6 +315,15 @@ print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 print dnode6 $dnode6Vnodes +if $dnode1Vnodes != 2 then + goto show6 +endi +if $dnode4Vnodes != 2 then + goto show6 +endi +if $dnode5Vnodes != 3 then + goto show6 +endi if $dnode6Vnodes != 2 then goto show6 endi @@ -315,13 +340,12 @@ print dnode7 ==> $data2_7 print ============================== step7 print ========= drop dnode4 sql drop dnode $hostname4 -sleep 9000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show7 @@ -357,16 +381,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_6 != slave then + goto show7 +endi + print ============================== step8 sql create dnode $hostname7 system sh/exec.sh -n dnode7 -s start -sleep 9000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -379,6 +406,15 @@ print dnode6 $dnode6Vnodes $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes +if $dnode1Vnodes != 2 then + goto show8 +endi +if $dnode5Vnodes != 2 then + goto show8 +endi +if $dnode6Vnodes != 3 then + goto show8 +endi if $dnode7Vnodes != 2 then goto show8 endi @@ -393,47 +429,17 @@ print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 print ============================== step9 -print ========= drop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 10000 - -sql drop dnode $hostname1 -print drop dnode1 and sleep 9000 -sleep 9000 - -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 -print dnode1 ==> $data2_1 -print dnode2 ==> $data2_2 -print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 - -if $dnode1Role != offline then - return -1 -endi - -print ============================== step9.1 -sleep 2000 -system sh/exec.sh -n dnode1 -s start $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi +sql show mnodes -x show9 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -442,7 +448,60 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -sql show dnodes -x show9 +if $data2_1 != offline then + goto show9 +endi +if $data2_5 != master then + goto show9 +endi +if $data2_6 != slave then + goto show9 +endi + +print ============================== step10 +sql drop dnode $hostname1 +$x = 0 +show10: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show mnodes -x show10 + +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +print dnode3 ==> $data2_3 +print dnode4 ==> $data2_4 +print dnode5 ==> $data2_5 +print dnode6 ==> $data2_6 +print dnode7 ==> $data2_7 + +if $data2_1 != null then + goto show10 +endi +if $data2_5 != master then + goto show10 +endi +if $data2_6 != slave then + goto show10 +endi +if $data2_7 != slave then + goto show10 +endi + +print ============================== step11 +system sh/exec.sh -n dnode1 -s start + +$x = 0 +show11: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes -x show11 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -451,17 +510,16 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 3 then - goto show9 + goto show11 endi if $dnode6Vnodes != 3 then - goto show9 + goto show11 endi if $dnode7Vnodes != 3 then - goto show9 + goto show11 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql show mnodes print dnode1 ==> $data2_1 @@ -472,15 +530,13 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -print ============================== step11 -print ========= add db4 - +print ============================== step12 sql create database c_b3_d4 replica 3 sql use c_b3_d4 $x = 0 create4: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -491,16 +547,14 @@ sql insert into c_b3_t4 values(1520000022043, 43) sql insert into c_b3_t4 values(1520000023042, 42) sql insert into c_b3_t4 values(1520000024041, 41) -sleep 3000 - $x = 0 -show11: +show12: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi -sql show dnodes -x show11 +sql show dnodes -x show12 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -509,21 +563,18 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 4 then - goto show11 + goto show12 endi if $dnode6Vnodes != 4 then - goto show11 + goto show12 endi if $dnode7Vnodes != 4 then - goto show11 + goto show12 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 - print ============================== step13 sql reset query cache -sleep 1000 +sleep 200 print ========= check data @@ -590,4 +641,3 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim index 68a5bad6b3..656ed2ec44 100644 --- a/tests/script/unique/cluster/vgroup100.sim +++ b/tests/script/unique/cluster/vgroup100.sim @@ -17,7 +17,6 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 0 print ============================== step1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================== step2 @@ -27,8 +26,6 @@ system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 - $maxNum = 102 $maxNum = 12 @@ -92,13 +89,11 @@ print ============================== step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 print ============================== step6 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 10000 print ============================== step7 diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index fee47caadb..f84339be79 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -18,7 +18,6 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 @@ -38,8 +37,6 @@ while $i < 2000 $i = $i + 1 endw -sleep 2500 - sql show db.vgroups if $rows != 2 then return -1 @@ -73,7 +70,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -92,7 +88,6 @@ if $data2_1 != master then goto step3 endi -sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index fd92c160d9..682444e922 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,11 +31,29 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + $loop = 0 begin: @@ -53,7 +71,6 @@ begin: sql insert into t13 values(now, 1 ) sql create table t14 (ts timestamp, i int) sql insert into t14 values(now, 1 ) - sleep 1200 sql create table t21 (ts timestamp, i int) sql insert into t21 values(now, 1 ) @@ -63,7 +80,6 @@ begin: sql insert into t23 values(now, 1 ) sql create table t24 (ts timestamp, i int) sql insert into t24 values(now, 1 ) - sleep 1200 sql create table t31 (ts timestamp, i int) sql insert into t31 values(now, 1 ) @@ -73,7 +89,6 @@ begin: sql insert into t33 values(now, 1 ) sql create table t34 (ts timestamp, i int) sql insert into t34 values(now, 1 ) - sleep 1200 sql create table t41 (ts timestamp, i int) sql insert into t41 values(now, 1 ) @@ -83,7 +98,6 @@ begin: sql insert into t43 values(now, 1 ) sql create table t44 (ts timestamp, i int) sql insert into t44 values(now, 1 ) - sleep 1200 sql create table t51 (ts timestamp, i int) sql insert into t51 values(now, 1 ) @@ -93,7 +107,6 @@ begin: sql insert into t53 values(now, 1 ) sql create table t54 (ts timestamp, i int) sql insert into t54 values(now, 1 ) - sleep 1200 sql create table t61 (ts timestamp, i int) sql insert into t61 values(now, 1 ) @@ -103,7 +116,6 @@ begin: sql insert into t63 values(now, 1 ) sql create table t64 (ts timestamp, i int) sql insert into t64 values(now, 1 ) - sleep 1200 sql create table t71 (ts timestamp, i int) sql insert into t71 values(now, 1 ) @@ -113,7 +125,6 @@ begin: sql insert into t73 values(now, 1 ) sql create table t74 (ts timestamp, i int) sql insert into t74 values(now, 1 ) - sleep 1200 sql create table t81 (ts timestamp, i int) sql insert into t81 values(now, 1 ) @@ -123,7 +134,6 @@ begin: sql insert into t83 values(now, 1 ) sql create table t84 (ts timestamp, i int) sql insert into t84 values(now, 1 ) - sleep 1200 sql show dnodes print dnode1 openVnodes $data2_1 @@ -138,29 +148,36 @@ begin: print ======== step2 $loop system sh/exec.sh -n dnode2 -s stop - sleep 1000 print ==> drop database $db sql drop database $db print ======== step3 $loop sleep 2000 system sh/exec.sh -n dnode2 -s start - sleep 15000 - sql show dnodes + + $x = 0 + step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + + sql show dnodes -x step3 print dnode1 openVnodes $data2_1 $data4_1 print dnode2 openVnodes $data2_2 $data4_2 if $data2_1 != 0 then - return -1 + goto step3 endi if $data2_2 != 0 then - return -1 + goto step3 endi if $data4_1 != ready then - return -1 + goto step3 endi if $data4_2 != ready then - return -1 + goto step3 endi print ===> test times : $loop @@ -171,7 +188,7 @@ begin: $loop = $loop + 1 sql reset query cache - sleep 1000 + sleep 100 goto begin diff --git a/tests/script/unique/db/replica_add12.sim b/tests/script/unique/db/replica_add12.sim index 62d7e9daa3..d46187e445 100644 --- a/tests/script/unique/db/replica_add12.sim +++ b/tests/script/unique/db/replica_add12.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -27,7 +32,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -65,8 +106,6 @@ if $rows != 1 then return -1 endi -sleep 2000 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -89,10 +128,40 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 10000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ======== step3 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -115,7 +184,6 @@ sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -138,33 +206,54 @@ if $rows != 2 then endi sql reset query cache -sleep 2000 +sleep 200 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 - -print ===== insert data - -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) -sleep 1000 +sql select * from d1.t1 -x s51 +s51: +#sql insert into d1.t1 values(now, 3) -x s52 +s52: print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql insert into d1.t1 values(now, 3) -x step6 +sql insert into d2.t2 values(now, 3) -x step6 +sql insert into d3.t3 values(now, 3) -x step6 +sql insert into d4.t4 values(now, 3) -x step6 sql select * from d1.t1 if $rows != 3 then @@ -187,30 +276,52 @@ if $rows != 3 then endi print ========= step61 - system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +#sql insert into d1.t1 values(now, 3) -x s61 +s61: -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) -sleep 1000 - -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d2.t2 -x s62 +s62: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) -sleep 1000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql insert into d1.t1 values(now, 5) -x step7 +sql insert into d2.t2 values(now, 5) -x step7 +sql insert into d3.t3 values(now, 5) -x step7 +sql insert into d4.t4 values(now, 5) -x step7 sql select * from d1.t1 if $rows != 4 then @@ -235,8 +346,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index 5d34440e3e..13a5c97832 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,41 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -46,7 +85,6 @@ sql insert into d1.t1 values(1589529000011, 1) sql insert into d2.t2 values(1589529000021, 1) sql insert into d3.t3 values(1589529000031, 1) sql insert into d4.t4 values(1589529000041, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -73,17 +111,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi print ======== step3 -$x = 0 -show3: - $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 - endi - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -111,7 +172,6 @@ sql insert into d1.t1 values(1589529000012, 2) sql insert into d2.t2 values(1589529000022, 2) sql insert into d3.t3 values(1589529000032, 2) sql insert into d4.t4 values(1589529000042, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -135,15 +195,13 @@ endi print ========= step5 sql reset query cache -sleep 1000 +sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000013, 3) sql insert into d2.t2 values(1589529000023, 3) sql insert into d3.t3 values(1589529000033, 3) sql insert into d4.t4 values(1589529000043, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -167,15 +225,45 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000014, 4) sql insert into d2.t2 values(1589529000024, 4) sql insert into d3.t3 values(1589529000034, 4) sql insert into d4.t4 values(1589529000044, 4) -sleep 1000 sql select * from d1.t1 print select * from d1.t1 $rows @@ -203,15 +291,45 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000015, 5) sql insert into d2.t2 values(1589529000025, 5) sql insert into d3.t3 values(1589529000035, 5) sql insert into d4.t4 values(1589529000045, 5) -sleep 1000 sql select * from d1.t1 if $rows != 5 then @@ -235,15 +353,45 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000016, 6) sql insert into d2.t2 values(1589529000026, 6) sql insert into d3.t3 values(1589529000036, 6) sql insert into d4.t4 values(1589529000046, 6) -sleep 1000 sql select * from d1.t1 if $rows != 6 then @@ -268,8 +416,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add23.sim b/tests/script/unique/db/replica_add23.sim index 9285f68137..ac0bd6d9d7 100644 --- a/tests/script/unique/db/replica_add23.sim +++ b/tests/script/unique/db/replica_add23.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,42 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -46,7 +86,6 @@ sql insert into d1.t1 values(1588262400001, 1) sql insert into d2.t2 values(1588262400001, 1) sql insert into d3.t3 values(1588262400001, 1) sql insert into d4.t4 values(1588262400001, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -73,17 +112,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -111,7 +173,6 @@ sql insert into d1.t1 values(1588262400002, 2) sql insert into d2.t2 values(1588262400002, 2) sql insert into d3.t3 values(1588262400002, 2) sql insert into d4.t4 values(1588262400002, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -134,17 +195,14 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 100 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql insert into d1.t1 values(1588262400003, 3) sql insert into d2.t2 values(1588262400003, 3) sql insert into d3.t3 values(1588262400003, 3) sql insert into d4.t4 values(1588262400003, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -168,15 +226,45 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400004, 4) sql insert into d2.t2 values(1588262400004, 4) sql insert into d3.t3 values(1588262400004, 4) sql insert into d4.t4 values(1588262400004, 4) -sleep 1000 sql select * from d1.t1 if $rows != 4 then @@ -200,15 +288,45 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400005, 5) sql insert into d2.t2 values(1588262400005, 5) sql insert into d3.t3 values(1588262400005, 5) sql insert into d4.t4 values(1588262400005, 5) -sleep 1000 sql select * from d1.t1 if $rows != 5 then @@ -232,15 +350,45 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400006, 6) sql insert into d2.t2 values(1588262400006, 6) sql insert into d3.t3 values(1588262400006, 6) sql insert into d4.t4 values(1588262400006, 6) -sleep 1000 sql select * from d1.t1 if $rows != 6 then @@ -265,8 +413,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index 5c5f98a108..9880ec666c 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -14,6 +14,10 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +25,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +73,6 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -61,24 +95,85 @@ if $rows != 1 then endi print ========= step2 alter db -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========= step3 system sh/exec.sh -n dnode2 -s start -sleep 10000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi print ========= step4 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -102,44 +197,104 @@ endi print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 3) -x s1 s1: -sql insert into d2.t2 values(now, 3) -x s2 -s2: -sql insert into d3.t3 values(now, 3) -x s3 -s3: -sql insert into d4.t4 values(now, 3) -x s4 -s4: +#sql insert into d2.t2 values(now, 3) -x s2 +#s2: +#sql insert into d3.t3 values(now, 3) -x s3 +#s3: +#sql insert into d4.t4 values(now, 3) -x s4 +#s4: print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 -sql insert into d1.t1 values(now, 4) -x s5 -s5: +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +#sql insert into d1.t1 values(now, 4) -x s5 +#s5: sql insert into d2.t2 values(now, 4) -x s6 s6: -sql insert into d3.t3 values(now, 4) -x s7 -s7: -sql insert into d4.t4 values(now, 4) -x s8 -s8: +#sql insert into d3.t3 values(now, 4) -x s7 +#s7: +#sql insert into d4.t4 values(now, 4) -x s8 +#s8: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) -sleep 1000 sql select * from d1.t1 sql select * from d2.t2 @@ -148,9 +303,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce21.sim b/tests/script/unique/db/replica_reduce21.sim index 7fe7d9504a..d3a76485f8 100644 --- a/tests/script/unique/db/replica_reduce21.sim +++ b/tests/script/unique/db/replica_reduce21.sim @@ -14,12 +14,42 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -63,12 +93,25 @@ sql create database d5 replica 1 print ========= step3 alter d1 sql alter database d1 replica 1 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + print ========= step4 query d1 sql insert into d1.t1 values(now, 2) sql select * from d1.t1 -sleep 1000 if $rows != 2 then return -1 endi @@ -77,22 +120,20 @@ print ========= step5 query d5 sql create table d5.t5 (ts timestamp, i int) sql insert into d5.t5 values(now, 1); sql select * from d5.t5 -sleep 1000 if $rows != 1 then return -1 endi return print ========= step7 drop d1 sql drop database d1 -sleep 5000 - sql reset query cache +sleep 100 + print ========= step8 sql insert into d5.t5 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d5.t5 if $rows != 2 then @@ -116,13 +157,11 @@ endi print ======== step9 stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d5.t5 values(now, 3) sql insert into d2.t2 values(now, 3) sql insert into d3.t3 values(now, 3) sql insert into d4.t4 values(now, 3) -sleep 1000 sql select * from d5.t5 if $rows != 3 then @@ -145,10 +184,4 @@ if $rows != 3 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce31.sim b/tests/script/unique/db/replica_reduce31.sim index cbe0417dbd..00a0bbfcb3 100644 --- a/tests/script/unique/db/replica_reduce31.sim +++ b/tests/script/unique/db/replica_reduce31.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +74,6 @@ sql insert into d1.t1 values(now, 1) sql insert into d2.t2 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -68,23 +103,70 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 8000 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi sql alter database d1 replica 1 sql alter database d2 replica 1 sql alter database d3 replica 1 -sleep 8000 +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi print ========= step3 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -110,13 +192,37 @@ print ========= step4 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 8000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) sql insert into d3.t3 values(now, 3) sql insert into d4.t4 values(now, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -140,10 +246,8 @@ endi print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 4) -x step1 step1: @@ -156,9 +260,7 @@ step4: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(now, 5) -x step5 step5: @@ -171,13 +273,11 @@ step8: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) sql insert into d3.t3 values(now, 6) sql insert into d4.t4 values(now, 6) -sleep 1000 sql select * from d1.t1 sql select * from d2.t2 @@ -186,9 +286,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index c453f16ce6..a5eb78dacb 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +79,6 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -65,14 +105,44 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ========= step3 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -95,40 +165,111 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 200 print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep 5000 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x s41 +s41: +sql insert into d2.t2 values(now, 3) -x s42 +s42: +sql insert into d3.t3 values(now, 3) -x s43 +s43: +sql insert into d4.t4 values(now, 3) -x s44 +s44: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d1.t1 -x s45 +s45: +sql select * from d2.t2 -x s46 +s46: +sql select * from d3.t3 -x s47 +s47: +sql select * from d4.t4 -x s48 +s48: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql_error insert into d1.t1 values(now, 3) -x s51 +s51: +sql_error insert into d2.t2 values(now, 3) -x s52 +s52: +sql_error insert into d3.t3 values(now, 3) -x s53 +s53: +sql_error insert into d4.t4 values(now, 3) -x s54 +s54: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -150,9 +291,4 @@ print d4.t4 $rows system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 10d185f353..8743204a03 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -33,7 +33,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -50,15 +49,14 @@ if $data2_1 != 1 then endi print ========== step2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -124,8 +122,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -174,8 +172,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -258,7 +256,7 @@ endi system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 23897df690..7b07eb3f37 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -32,7 +32,29 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -70,8 +92,8 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -90,7 +112,6 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step3 sql create dnode $hostname4 @@ -156,8 +177,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -189,8 +210,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -217,10 +238,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index 9f7f3abb8b..f5558d504e 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -38,7 +38,32 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi sql create database d1 replica 3 sql create table d1.t1 (t timestamp, i int) @@ -81,7 +106,7 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -106,7 +131,6 @@ if $data2_4 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start @@ -114,8 +138,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show3: $x = $x + 1 - sleep 4000 - if $x == 15 then + sleep 1000 + if $x == 60 then return -1 endi @@ -154,8 +178,8 @@ sql insert into d3.t3 values(now+5s, 31) $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -189,8 +213,8 @@ system sh/exec.sh -n dnode6 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -216,8 +240,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -245,10 +269,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/offline3.sim b/tests/script/unique/dnode/offline3.sim new file mode 100644 index 0000000000..93c75e3b13 --- /dev/null +++ b/tests/script/unique/dnode/offline3.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 300 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +if $data2_1 != master then + goto step1 +endi + +print ========== step2 +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, value int) +sql insert into tb values (now, 1) +sql insert into tb values (now, 2) + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step2 +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi +if $data4_3 != ready then + goto step2 +endi +if $data4_4 != null then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/reason.sim b/tests/script/unique/dnode/reason.sim index ad61a81b97..c685b1129d 100644 --- a/tests/script/unique/dnode/reason.sim +++ b/tests/script/unique/dnode/reason.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode2 -i 2 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -18,38 +17,75 @@ endi print ========== step2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes -print dnode1 off: $data7_1 -print dnode2 off: $data7_2 +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -sleep 3000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_2 if $data7_2 != @status msg timeout@ then - return -1 + goto step3 endi print ========== step4 sql drop dnode $hostname2 -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes if $rows != 1 then - return -1 + goto step4 endi print ========== step5 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_3 if $data7_3 != @dnodeId not match@ then - return -1 + goto step5 endi print ========== step6 @@ -58,12 +94,19 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode4 off: $data7_4 if $data7_4 != @mnEqualVn not match@ then - return -1 + goto step6 endi print ========== step7 @@ -72,12 +115,19 @@ system sh/cfg.sh -n dnode5 -c statusInterval -v 3 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode5 off: $data7_5 if $data7_5 != @interval not match@ then - return -1 + goto step7 endi print ========== step8 @@ -86,12 +136,19 @@ system sh/cfg.sh -n dnode6 -c balance -v 0 system sh/exec.sh -n dnode6 -s start sql create dnode $hostname6 -sleep 3000 +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode6 off: $data7_6 if $data7_6 != @balance not match@ then - return -1 + goto step8 endi print ========== step9 @@ -100,12 +157,19 @@ system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000 system sh/exec.sh -n dnode7 -s start sql create dnode $hostname7 -sleep 3000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode7 off: $data7_7 if $data7_7 != @maxTabPerVn not match@ then - return -1 + goto step9 endi print ========== step10 @@ -114,12 +178,19 @@ system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode8 -s start sql create dnode $hostname8 -sleep 3000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode8 off: $data7_8 if $data7_8 != @maxVgPerDb not match@ then - return -1 + goto step10 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 6bde68b8b5..6f830d2cf8 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,12 +99,11 @@ endi print ========== step3 sql drop dnode $hostname2 -sleep 7001 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -101,8 +118,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -115,7 +132,6 @@ if $data2_2 != null then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step5 sql create dnode $hostname4 @@ -124,8 +140,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 903e262be7..f97e55164b 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,7 +99,6 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT sql drop dnode $hostname2 -sleep 5000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -91,14 +108,20 @@ print ========== step4 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 if $data2_3 != 0 then - return -1 + goto step4 endi print ============ step 4.1 @@ -107,8 +130,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/dnode/vnode_clean.sim b/tests/script/unique/dnode/vnode_clean.sim index f90f3d3fd0..76311a6cac 100644 --- a/tests/script/unique/dnode/vnode_clean.sim +++ b/tests/script/unique/dnode/vnode_clean.sim @@ -45,8 +45,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -86,8 +86,8 @@ sql drop dnode $hostname2 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -106,15 +106,14 @@ endi system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========== step5 -sleep 5000 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes @@ -153,8 +152,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi @@ -184,8 +183,8 @@ sql insert into d4.t4 values(now+5s, 41) $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -208,8 +207,8 @@ sql drop dnode $hostname3 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -228,7 +227,6 @@ if $data2_4 != 4 then endi system sh/exec.sh -n dnode3 -s stop -sleep 5000 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/import/replica2.sim b/tests/script/unique/import/replica2.sim index f2836fe470..54ce28543e 100644 --- a/tests/script/unique/import/replica2.sim +++ b/tests/script/unique/import/replica2.sim @@ -27,11 +27,30 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi sql create database ir2db replica 2 days 7 sql use ir2db @@ -96,9 +115,22 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -163,9 +195,22 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -193,9 +238,20 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 +endi print ================= step14 #1520000000000 @@ -214,12 +270,24 @@ endi print ================= step15 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a4 +endi + +sql select * from tb; if $rows != 52 then - return -1 + goto a4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 3a9f03a7ea..714141c412 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -27,7 +27,6 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -35,6 +34,29 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database ir3db replica 3 days 7 sql use ir3db @@ -99,9 +121,20 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -166,9 +199,21 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step10 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -217,10 +262,21 @@ endi print ================= step15 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step15: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step15 +endi +sql select * from tb; if $rows != 52 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt20.sim b/tests/script/unique/mnode/mgmt20.sim index e51d429925..6eb35b67ac 100644 --- a/tests/script/unique/mnode/mgmt20.sim +++ b/tests/script/unique/mnode/mgmt20.sim @@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ============== step1 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 sql connect print ============== step2 @@ -20,8 +19,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index d55e36d7fc..1dc419b17d 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -9,7 +9,6 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -26,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -64,8 +63,8 @@ sql connect $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -82,13 +81,12 @@ endi print ============== step7 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 5000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt23.sim b/tests/script/unique/mnode/mgmt23.sim index d1820ef8c6..19c7b4ba76 100644 --- a/tests/script/unique/mnode/mgmt23.sim +++ b/tests/script/unique/mnode/mgmt23.sim @@ -25,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -65,7 +65,14 @@ endi print ============== step4 sql drop dnode $hostname2 -sleep 10000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -76,13 +83,13 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop @@ -93,7 +100,14 @@ sql create dnode $hostname2 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode2 -s start -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -104,19 +118,17 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - sql_error show mnodes print ============== step7 @@ -126,7 +138,4 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 205e1b05d3..ce7cdce35d 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -28,7 +28,14 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -39,19 +46,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -62,18 +76,25 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi print ============== step4 sql drop dnode $hostname2 -sleep 8000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -84,25 +105,30 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step5 sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -113,20 +139,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != slave then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step6 $dnode1Role = $data2_1 $dnode2Role = $data2_4 $dnode3Role = $data2_3 @@ -135,7 +167,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != offline then - return -1 + goto step6 endi #if $dnode2Role != master then # return -1 @@ -146,9 +178,15 @@ endi print ============== step7 sql drop dnode $hostname1 -sleep 8000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -157,7 +195,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != null then - return -1 + goto step7 endi #if $dnode2Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index 96e3133d76..d8a46b0955 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -12,7 +12,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -32,7 +31,13 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -45,22 +50,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi if $dnode4Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -73,23 +85,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi if $dnode4Role != null then - return -1 + goto step3 endi print ============== step4 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 8000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -102,21 +120,27 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi if $dnode4Role != null then - return -1 + goto step4 endi print ============== step5 sql drop dnode $hostname2 -sleep 8000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -129,28 +153,32 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi if $dnode4Role != slave then - return -1 + goto step5 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step6 sql create dnode $hostname2 -sleep 8000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -163,23 +191,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step6 endi if $dnode2Role != null then - return -1 + goto step6 endi if $dnode3Role != slave then - return -1 + goto step6 endi if $dnode4Role != slave then - return -1 + goto step6 endi print ============== step7 system sh/exec.sh -n dnode1 -s stop -sleep 4000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -190,14 +224,19 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != offline then - return -1 + goto step7 endi print ============== step8 sql drop dnode $hostname1 -sleep 8000 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step8 $dnode1Role = $data2_1 $dnode2Role = $data2_5 $dnode3Role = $data2_3 @@ -208,10 +247,10 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != null then - return -1 + goto step8 endi if $dnode2Role != slave then - return -1 + goto step8 endi #if $dnode3Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 1fba912d49..0c9f961d25 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -55,6 +55,14 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start print ============== step4 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + sql show mnodes $dnode1Role = $data2_1 @@ -65,20 +73,15 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != null then - return -1 + goto step4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 8bd6372ae9..869c7f8295 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -23,7 +23,37 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db1 replica 2 diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index e862d745d4..5dbb24437d 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -23,7 +23,38 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 2 @@ -64,7 +95,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_basic.sim b/tests/script/unique/vnode/replica3_basic.sim index ab5bebb50d..0ff42b523b 100644 --- a/tests/script/unique/vnode/replica3_basic.sim +++ b/tests/script/unique/vnode/replica3_basic.sim @@ -16,14 +16,48 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi $N = 10 $table = table_r3 $db = db1 -sleep 3000 - print =================== step 1 sql create database $db replica 3 @@ -66,7 +100,21 @@ endi print =================== step 4 system sh/exec.sh -n dnode2 -s start -sleep 2000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step4 +endi + $y = $x + $N $expect = $N * 3 while $x < $y @@ -83,7 +131,6 @@ endi print =================== step 5 system sh/exec.sh -n dnode3 -s stop -sleep 2000 $y = $x + $N $expect = $N * 4 while $x < $y @@ -100,7 +147,21 @@ endi print =================== step 6 system sh/exec.sh -n dnode3 -s start -sleep 2000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + $y = $x + $N $expect = $N * 5 while $x < $y @@ -117,7 +178,6 @@ endi print =================== step 7 system sh/exec.sh -n dnode1 -s stop -sleep 2000 $y = $x + $N $expect = $N * 6 while $x < $y @@ -134,7 +194,21 @@ endi print =================== step 8 system sh/exec.sh -n dnode1 -s start -sleep 2000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + $y = $x + $N $expect = $N * 7 while $x < $y diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 4b5c852de8..d866fb05d8 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -26,7 +26,42 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 3 @@ -75,7 +110,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/test-all.sh b/tests/test-all.sh index 4448c8a2a4..30d687de98 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -31,11 +31,17 @@ function runSimCaseOneByOnefq { case=`echo $line | grep sim$ |awk '{print $NF}'` start_time=`date +%s` - ./test.sh -f $case > /dev/null 2>&1 && \ + IN_TDINTERNAL="community" + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + ./test.sh -f $case > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log - - + else + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ + ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + fi + out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then exit 8