diff --git a/.travis.yml b/.travis.yml index 7aacfd3e58..4d7a809e29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ branches: matrix: - os: linux - dist: bionic + dist: focal language: c git: @@ -28,8 +28,6 @@ matrix: - build-essential - cmake - net-tools - - python-pip - - python-setuptools - python3-pip - python3-setuptools - valgrind @@ -54,13 +52,19 @@ matrix: cd ${TRAVIS_BUILD_DIR}/debug make install > /dev/null || travis_terminate $? - pip install numpy - pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install numpy pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ cd ${TRAVIS_BUILD_DIR}/tests ./test-all.sh smoke || travis_terminate $? + sleep 1 + + cd ${TRAVIS_BUILD_DIR}/tests/pytest + pkill -TERM -x taosd + fuser -k -n tcp 6030 + sleep 1 + ./crash_gen.sh -a -p -t 4 -s 25|| travis_terminate $? + sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest ./valgrind-test.sh 2>&1 > mem-error-out.log diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 4992692109..4af929bf41 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -160,7 +160,9 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); void tscFieldInfoClear(SFieldInfo* pFieldInfo); -int32_t tscNumOfFields(SQueryInfo* pQueryInfo); + +static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } + int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index a398ad659e..be82eb64a8 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -412,7 +412,44 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); +//void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); + +static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { + SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); + assert(pInfo->pSqlExpr != NULL); + + int32_t type = pInfo->pSqlExpr->resType; + int32_t bytes = pInfo->pSqlExpr->resBytes; + + char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; + + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); + + if (isNull(pData, type)) { + pRes->tsrow[columnIndex] = NULL; + } else { + pRes->tsrow[columnIndex] = ((tstr*)pData)->data; + } + + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; + } else { + assert(bytes == tDataTypeDesc[type].nSize); + + if (isNull(pData, type)) { + pRes->tsrow[columnIndex] = NULL; + } else { + pRes->tsrow[columnIndex] = pData; + } + + pRes->length[columnIndex] = bytes; + } +} extern void * tscCacheHandle; extern void * tscTmr; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 41464aa660..ffe42de236 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -43,7 +43,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA; pSql->fp = fp; pSql->sqlstr = calloc(1, sqlLen + 1); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 909338aa4a..06e5ff73bf 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2952,10 +2952,14 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); assert(pCtx->inputBytes == pCtx->outputBytes); - - for (int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); - + + tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); + char* data = pCtx->aOutputBuf; + pCtx->aOutputBuf += pCtx->outputBytes; + + // directly copy from the first one + for (int32_t i = 1; i < pCtx->size; ++i) { + memmove(pCtx->aOutputBuf, data, pCtx->outputBytes); pCtx->aOutputBuf += pCtx->outputBytes; } } @@ -3941,7 +3945,7 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { tsBufFlush(pTSbuf); strcpy(pCtx->aOutputBuf, pTSbuf->path); - tsBufDestory(pTSbuf); + tsBufDestroy(pTSbuf); doFinalizer(pCtx); } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 13523818d1..80fc82d90b 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -274,6 +274,10 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->numOfBuffer = idx; SCompareParam *param = malloc(sizeof(SCompareParam)); + if (param == NULL) { + tfree(pReducer); + return; + } param->pLocalData = pReducer->pLocalDataSrc; param->pDesc = pReducer->pDesc; param->num = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; @@ -284,6 +288,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); if (pReducer->pLoserTree == NULL || pRes->code != 0) { + tfree(param); tfree(pReducer); return; } @@ -332,6 +337,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd tfree(pReducer->pResultBuf); tfree(pReducer->pFinalRes); tfree(pReducer->prevRowOfInput); + tfree(pReducer->pLoserTree); + tfree(param); tfree(pReducer); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 9f0d1a26ab..0cf69dfd46 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -497,7 +497,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { tsem_init(&pSql->rspSem, 0, 0); pSql->signature = pSql; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA; pStmt->pSql = pSql; return pStmt; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 65e2c976e0..ca6dfcee9e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -175,7 +175,6 @@ static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { return TSDB_CODE_SUCCESS; } -// todo handle memory leak in error handle function int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) { return TSDB_CODE_TSC_APP_ERROR; @@ -5490,9 +5489,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { } if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f9f93b3f89..73b93435ad 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -113,7 +113,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA; tsem_init(&pSql->rspSem, 0, 0); pObj->pDnodeConn = pDnodeConn; @@ -403,7 +403,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); sem_wait(&pSql->rspSem); } - + return doSetResultRowData(pSql, true); } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index f214e76cc7..6cc27a4cfe 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -255,6 +255,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf // release the metric/meter meta information reference, so data in cache can be updated taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + tscFreeSqlResult(pSql); + tfree(pSql->pSubs); + pSql->numOfSubs = 0; tfree(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index f7d03bd787..2c5035c2ef 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -107,7 +107,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* pSql->signature = pSql; pSql->param = pSql; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA; pSql->fp = asyncCallback; int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b5b659de0c..5e2ba46fbc 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -152,8 +152,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufFlush(output1); tsBufFlush(output2); - tsBufDestory(pSupporter1->pTSBuf); - tsBufDestory(pSupporter2->pTSBuf); + tsBufDestroy(pSupporter1->pTSBuf); + tsBufDestroy(pSupporter2->pTSBuf); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal, @@ -550,7 +550,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, return true; } -static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { +static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql); SJoinSupporter* p1 = pParentSql->pSubs[0]->param; @@ -568,10 +568,7 @@ static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParent *s2 = taosArrayInit(p2->num, p2->tagSize); if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) { - freeJoinSubqueryObj(pParentSql); - pParentSql->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY; - tscQueueAsyncRes(pParentSql); - return; + return TSDB_CODE_QRY_DUP_JOIN_KEY; } int32_t i = 0, j = 0; @@ -594,6 +591,8 @@ static void getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParent i++; } } + + return TSDB_CODE_SUCCESS; } static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { @@ -680,7 +679,14 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } SArray *s1 = NULL, *s2 = NULL; - getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); + int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); + if (code != TSDB_CODE_SUCCESS) { + freeJoinSubqueryObj(pParentSql); + pParentSql->res.code = code; + tscQueueAsyncRes(pParentSql); + return; + } + if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); freeJoinSubqueryObj(pParentSql); @@ -762,7 +768,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); - tsBufDestory(pBuf); + tsBufDestroy(pBuf); } // continue to retrieve ts-comp data from vnode @@ -1447,9 +1453,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tscDebug("%p start to free subquery result", pSql); - if (pSql->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSql); - } + taos_free_result(pSql); tfree(trsupport->localBuffer); @@ -2108,9 +2112,9 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - + assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - + if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { if (pRes->completed) { tfree(pRes->tsrow); @@ -2118,29 +2122,31 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { return pRes->tsrow; } - + if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker tfree(pRes->tsrow); return pRes->tsrow; } - + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - + size_t size = tscNumOfFields(pQueryInfo); for (int i = 0; i < size; ++i) { - SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); + SFieldSupInfo* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pSupportInfo, i); if (pSup->pSqlExpr != NULL) { tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } - + // primary key column cannot be null in interval query, no need to check if (i == 0 && pQueryInfo->intervalTime > 0) { continue; } - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - transferNcharData(pSql, i, pField); - + + TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pFields, i); + if (pRes->tsrow[i] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) { + transferNcharData(pSql, i, pField); + } + // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { if (pRes->pArithSup == NULL) { @@ -2150,10 +2156,10 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { sas->numOfCols = tscSqlExprNumOfExprs(pQueryInfo); sas->exprList = pQueryInfo->exprList; sas->data = calloc(sas->numOfCols, POINTER_BYTES); - + pRes->pArithSup = sas; } - + if (pRes->buffer[i] == NULL) { TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pRes->buffer[i] = malloc(field->bytes); @@ -2163,13 +2169,13 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - + tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; } } - + pRes->row++; // index increase one-step return pRes->tsrow; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 54c9cdea65..eea5d60f64 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -794,7 +794,7 @@ SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { } SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { - return taosArrayGet(pFieldInfo->pSupportInfo, index); + return TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, index); } SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { @@ -858,11 +858,9 @@ void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { } TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { - return taosArrayGet(pFieldInfo->pFields, index); + return TARRAY_GET_ELEM(pFieldInfo->pFields, index); } -int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } - int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); assert(pInfo != NULL); @@ -1546,7 +1544,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->groupbyExpr.columnInfo = NULL; } - pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); + pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); tfree(pQueryInfo->fillVal); } @@ -1651,7 +1649,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pNew->fp = fp; pNew->param = param; - pNew->maxRetry = TSDB_MAX_REPLICA_NUM; + pNew->maxRetry = TSDB_MAX_REPLICA; pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { @@ -1806,7 +1804,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->fp = fp; pNew->param = param; - pNew->maxRetry = TSDB_MAX_REPLICA_NUM; + pNew->maxRetry = TSDB_MAX_REPLICA; char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; @@ -1822,7 +1820,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pPrevInfo->vgroupList = NULL; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); } @@ -2086,42 +2083,42 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } } -void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { - SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex); - assert(pInfo->pSqlExpr != NULL); - - int32_t type = pInfo->pSqlExpr->resType; - int32_t bytes = pInfo->pSqlExpr->resBytes; - - char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; - } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } - - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; - } - - pRes->length[columnIndex] = realLen; - } else { - assert(bytes == tDataTypeDesc[type].nSize); - - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; - } else { - pRes->tsrow[columnIndex] = pData; - } - - pRes->length[columnIndex] = bytes; - } -} +//void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { +// SFieldSupInfo* pInfo = TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); +// assert(pInfo->pSqlExpr != NULL); +// +// int32_t type = pInfo->pSqlExpr->resType; +// int32_t bytes = pInfo->pSqlExpr->resBytes; +// +// char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; +// +// if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { +// int32_t realLen = varDataLen(pData); +// assert(realLen <= bytes - VARSTR_HEADER_SIZE); +// +// if (isNull(pData, type)) { +// pRes->tsrow[columnIndex] = NULL; +// } else { +// pRes->tsrow[columnIndex] = ((tstr*)pData)->data; +// } +// +// if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor +// *(pData + realLen + VARSTR_HEADER_SIZE) = 0; +// } +// +// pRes->length[columnIndex] = realLen; +// } else { +// assert(bytes == tDataTypeDesc[type].nSize); +// +// if (isNull(pData, type)) { +// pRes->tsrow[columnIndex] = NULL; +// } else { +// pRes->tsrow[columnIndex] = pData; +// } +// +// pRes->length[columnIndex] = bytes; +// } +//} void* malloc_throw(size_t size) { void* p = malloc(size); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index da42c064ec..fbdfc84f5f 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -69,6 +69,7 @@ extern int64_t tsMaxRetentWindow; extern int32_t tsCacheBlockSize; extern int32_t tsBlocksPerVnode; extern int32_t tsMaxTablePerVnode; +extern int32_t tsMaxVgroupsPerDb; extern int16_t tsDaysPerFile; extern int32_t tsDaysToKeep; extern int32_t tsMinRowsInFileBlock; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index fae771e855..a8ef3f8e8d 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -110,14 +110,9 @@ int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; -int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; - -#ifdef _TD_ARM_32_ - int32_t tsMaxTablePerVnode = 100; -#else - int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; -#endif - +int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; +int32_t tsMaxVgroupsPerDb = 0; +int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; // balance int32_t tsEnableBalance = 1; int32_t tsAlternativeRole = 0; @@ -560,7 +555,7 @@ static void doInitGlobalConfig() { cfg.ptr = &tsMinIntervalTime; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; + cfg.minValue = 1; cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; @@ -606,8 +601,18 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "maxVgroupsPerDb"; + cfg.ptr = &tsMaxVgroupsPerDb; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 8192; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // database configs - cfg.option = "maxtablesPerVnode"; + cfg.option = "maxTablesPerVnode"; cfg.ptr = &tsMaxTablePerVnode; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; @@ -701,8 +706,8 @@ static void doInitGlobalConfig() { cfg.ptr = &tsReplications; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_REPLICA_NUM; - cfg.maxValue = TSDB_MAX_REPLICA_NUM; + cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION; + cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 8b85ecfbc7..15ec68fc66 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -82,7 +82,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in } int64_t start = ((startTime - intervalTime) / slidingTime + 1) * slidingTime; - if (!(timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h')) { + if (!(timeUnit == 'u' || timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h')) { /* * here we revised the start time of day according to the local time zone, * but in case of DST, the start time of one day need to be dynamically decided. diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index eff25e8c93..44b1d9fc38 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -367,31 +367,31 @@ bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR; } -bool isNull(const char *val, int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; - case TSDB_DATA_TYPE_TINYINT: - return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; - case TSDB_DATA_TYPE_SMALLINT: - return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; - case TSDB_DATA_TYPE_INT: - return *(uint32_t *)val == TSDB_DATA_INT_NULL; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; - case TSDB_DATA_TYPE_FLOAT: - return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; - case TSDB_DATA_TYPE_DOUBLE: - return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; - case TSDB_DATA_TYPE_NCHAR: - return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; - case TSDB_DATA_TYPE_BINARY: - return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; - default: - return false; - }; -} +//bool isNull(const char *val, int32_t type) { +// switch (type) { +// case TSDB_DATA_TYPE_BOOL: +// return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; +// case TSDB_DATA_TYPE_TINYINT: +// return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; +// case TSDB_DATA_TYPE_SMALLINT: +// return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; +// case TSDB_DATA_TYPE_INT: +// return *(uint32_t *)val == TSDB_DATA_INT_NULL; +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_TIMESTAMP: +// return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; +// case TSDB_DATA_TYPE_FLOAT: +// return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; +// case TSDB_DATA_TYPE_DOUBLE: +// return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; +// case TSDB_DATA_TYPE_NCHAR: +// return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; +// case TSDB_DATA_TYPE_BINARY: +// return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; +// default: +// return false; +// }; +//} void setVardataNull(char* val, int32_t type) { if (type == TSDB_DATA_TYPE_BINARY) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java index 7145673c71..273bc6920c 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java @@ -52,7 +52,7 @@ public class TDNode { public void start() { String selfPath = System.getProperty("user.dir"); String binPath = ""; - String projDir = selfPath + "/../../../../"; + String projDir = selfPath + "/../../../"; try { ArrayList taosdPath = new ArrayList<>(); @@ -68,7 +68,7 @@ public class TDNode { return; } else { for(String p : taosdPath) { - if(!p.contains("packing")) { + if(!p.contains("packaging")) { binPath = p; } } @@ -77,7 +77,7 @@ public class TDNode { e.printStackTrace(); } - if(binPath.equals("")) { + if(binPath.isEmpty()) { System.out.println("taosd not found"); return; } else { @@ -106,14 +106,10 @@ public class TDNode { String toBeKilled = "taosd"; if (this.running != 0) { - String psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print " + toBeKilled + "}'"; - try { - Process ps = Runtime.getRuntime().exec(psCmd); - ps.waitFor(); - long pid = ps.pid(); - - String killCmd = "kill -9 " + pid; - Runtime.getRuntime().exec(killCmd).waitFor(); + String killCmd = "pkill -kill -x " + toBeKilled; + String[] killCmds = {"sh", "-c", killCmd}; + try { + Runtime.getRuntime().exec(killCmds).waitFor(); for(int port = 6030; port < 6041; port ++) { String fuserCmd = "fuser -k -n tcp " + port; @@ -124,7 +120,7 @@ public class TDNode { } this.running = 0; - System.out.println("dnode:" + this.index + "is stopped by kill -9"); + System.out.println("dnode:" + this.index + " is stopped by pkill"); } } @@ -137,7 +133,6 @@ public class TDNode { } } - public void stopIP() { try{ String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down"; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java index 948cda95cf..ea15ae9863 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java @@ -16,11 +16,9 @@ public class TDNodes { public void setPath(String path) { try { - String psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" ; - Process ps = Runtime.getRuntime().exec(psCmd); - ps.waitFor(); - String killCmd = "kill -9 " + ps.pid(); - Runtime.getRuntime().exec(killCmd).waitFor(); + String killCmd = "pkill -kill -x taosd"; + String[] killCmds = {"sh", "-c", killCmd}; + Runtime.getRuntime().exec(killCmds).waitFor(); String binPath = System.getProperty("user.dir"); binPath += "/../../../debug"; diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index b1c93d7195..d0c42c4412 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -698,10 +698,12 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->alternativeRole = (uint8_t) tsAlternativeRole; // fill cluster cfg parameters - pStatus->clusterCfg.numOfMnodes = tsNumOfMnodes; - pStatus->clusterCfg.mnodeEqualVnodeNum = tsMnodeEqualVnodeNum; - pStatus->clusterCfg.offlineThreshold = tsOfflineThreshold; - pStatus->clusterCfg.statusInterval = tsStatusInterval; + pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); + pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); + pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); + pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); + pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); + pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); strcpy(pStatus->clusterCfg.arbitrator, tsArbitrator); strcpy(pStatus->clusterCfg.timezone, tsTimezone); strcpy(pStatus->clusterCfg.locale, tsLocale); diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 01f0cf25c0..971bd0a110 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "dnodeInt.h" #include "dnodeMain.h" +#include "tfile.h" static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static sem_t exitSem; @@ -67,6 +68,18 @@ int32_t main(int32_t argc, char *argv[]) { taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); } } +#endif +#ifdef TAOS_RANDOM_FILE_FAIL + else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { + if ( (i+1) < argc ) { + int factor = atoi(argv[i+1]); + printf("The factor of random failure is %d\n", factor); + taosSetRandomFileFailFactor(factor); + } else { + printf("Please specify a number for random failure factor!"); + exit(EXIT_FAILURE); + } + } #endif } diff --git a/src/inc/query.h b/src/inc/query.h index c648270b21..d201b649f9 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -20,7 +20,6 @@ extern "C" { #endif typedef void* qinfo_t; -typedef void (*_qinfo_free_fn_t)(void*); /** * create the qinfo object according to QueryTableMsg @@ -29,13 +28,8 @@ typedef void (*_qinfo_free_fn_t)(void*); * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, _qinfo_free_fn_t fn, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo); -/** - * Destroy QInfo object - * @param qinfo qhandle - */ -void qDestroyQueryInfo(qinfo_t qinfo); /** * the main query execution function, including query on both table and multitables, @@ -84,8 +78,14 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo); */ int32_t qKillQuery(qinfo_t qinfo); +/** + * destroy query info structure + * @param qHandle + */ +void qDestroyQueryInfo(qinfo_t qHandle); + void* qOpenQueryMgmt(int32_t vgId); -void qSetQueryMgmtClosed(void* pExecutor); +void qQueryMgmtNotifyClosed(void* pExecutor); void qCleanupQueryMgmt(void* pExecutor); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index e4ee058cef..eb3c3abc84 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -160,7 +160,32 @@ extern tDataTypeDescriptor tDataTypeDesc[11]; #define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) bool isValidDataType(int32_t type); -bool isNull(const char *val, int32_t type); +//bool isNull(const char *val, int32_t type); +static inline __attribute__((always_inline)) bool isNull(const char *val, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; + case TSDB_DATA_TYPE_TINYINT: + return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; + case TSDB_DATA_TYPE_SMALLINT: + return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; + case TSDB_DATA_TYPE_INT: + return *(uint32_t *)val == TSDB_DATA_INT_NULL; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; + case TSDB_DATA_TYPE_FLOAT: + return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; + case TSDB_DATA_TYPE_DOUBLE: + return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; + case TSDB_DATA_TYPE_NCHAR: + return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + case TSDB_DATA_TYPE_BINARY: + return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + default: + return false; + }; +} void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); @@ -271,8 +296,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_DEFAULT_TOTAL_BLOCKS 4 #define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 200000 -#define TSDB_DEFAULT_TABLES 1000 +#define TSDB_MAX_TABLES 5000000 +#define TSDB_DEFAULT_TABLES 200000 +#define TSDB_TABLES_STEP 10000 #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 @@ -306,9 +332,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 -#define TSDB_MIN_REPLICA_NUM 1 -#define TSDB_MAX_REPLICA_NUM 3 -#define TSDB_DEFAULT_REPLICA_NUM 1 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index d2bef9ea57..07b55b6acb 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -200,6 +200,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb inval TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "tsdb file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, 0, 0x0611, "tsdb need to reconfigure table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, 0, 0x0612, "tsdb create table information") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index ac89d1dabb..d96e9f68d5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -569,6 +569,8 @@ typedef struct { char timezone[64]; // tsTimezone char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset + int32_t maxtablesPerVnode; + int32_t maxVgroupsPerDb; } SClusterCfg; typedef struct { @@ -644,7 +646,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; int8_t numOfIps; - SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; + SIpAddr ipAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; typedef struct { diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index a440db7301..afd2d85dae 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -136,7 +136,7 @@ static void shellGetDirectoryFileList(char *inputDir) static void shellSourceFile(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = malloc(MAX_COMMAND_SIZE); + char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; size_t line_len = 0; @@ -185,7 +185,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { int lineNo = 0; while ((read_len = getline(&line, &line_len, f)) != -1) { ++lineNo; - if (read_len >= MAX_COMMAND_SIZE) continue; + if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index c8557af166..ca6c2a86a2 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -44,10 +44,7 @@ void mnodeRemoveSuperTableFromDb(SDbObj *pDb); void mnodeAddTableIntoDb(SDbObj *pDb); void mnodeRemoveTableFromDb(SDbObj *pDb); void mnodeAddVgroupIntoDb(SVgObj *pVgroup); -void mnodeAddVgroupIntoDbTail(SVgObj *pVgroup); void mnodeRemoveVgroupFromDb(SVgObj *pVgroup); -void mnodeMoveVgroupToTail(SVgObj *pVgroup); -void mnodeMoveVgroupToHead(SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index b40080759f..90c2ff9e66 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -145,10 +145,8 @@ typedef struct SVgObj { int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; - struct SVgObj *prev, *next; struct SDbObj *pDb; void * idPool; - SChildTableObj **tableList; } SVgObj; typedef struct { @@ -183,9 +181,11 @@ typedef struct SDbObj { int32_t numOfVgroups; int32_t numOfTables; int32_t numOfSuperTables; - SVgObj *pHead; - SVgObj *pTail; + int32_t vgListSize; + int32_t vgListIndex; + SVgObj **vgList; struct SAcctObj *pAcct; + pthread_mutex_t mutex; } SDbObj; typedef struct SUserObj { @@ -246,7 +246,8 @@ typedef struct { int16_t offset[TSDB_MAX_COLUMNS]; int16_t bytes[TSDB_MAX_COLUMNS]; int32_t numOfReads; - int8_t reserved0[2]; + int8_t maxReplica; + int8_t reserved0[0]; uint16_t payloadLen; char payload[]; } SShowObj; diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index 9e21f8f56a..f371786524 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -40,6 +40,7 @@ char* mnodeGetDnodeStatusStr(int32_t dnodeStatus); void mgmtMonitorDnodeModule(); int32_t mnodeGetDnodesNum(); +int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlinDnodesNum(); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); void mnodeIncDnodeRef(SDnodeObj *pDnode); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index eec6d45e23..0c47f684f8 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -94,6 +94,7 @@ void sdbDecRef(void *thandle, void *pRow); int64_t sdbGetNumOfRows(void *handle); int32_t sdbGetId(void *handle); uint64_t sdbGetVersion(); +bool sdbCheckRowDeleted(void *thandle, void *pRow); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 29a0fe1799..6ddf8e44b9 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -30,17 +30,17 @@ void mnodeDecVgroupRef(SVgObj *pVgroup); void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); -void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +//void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeUpdateVgroup(SVgObj *pVgroup); void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload); void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes); -int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg, SDbObj *pDb); +int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg); void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); -SVgObj *mnodeGetAvailableVgroup(SDbObj *pDb); +int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid); void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index c13cd7c95c..e8183e9089 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -38,6 +38,7 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" +#define VG_LIST_SIZE 8 static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; @@ -50,8 +51,14 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); +static void mnodeDestroyDb(SDbObj *pDb) { + pthread_mutex_destroy(&pDb->mutex); + tfree(pDb->vgList); + tfree(pDb); +} + static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { - tfree(pOper->pObj); + mnodeDestroyDb(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -59,8 +66,9 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) { SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); - pDb->pHead = NULL; - pDb->pTail = NULL; + pthread_mutex_init(&pDb->mutex, NULL); + pDb->vgListSize = VG_LIST_SIZE; + pDb->vgList = calloc(pDb->vgListSize, sizeof(SVgObj *)); pDb->numOfVgroups = 0; pDb->numOfTables = 0; pDb->numOfSuperTables = 0; @@ -94,14 +102,15 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) { } static int32_t mnodeDbActionUpdate(SSdbOper *pOper) { - SDbObj *pDb = pOper->pObj; - SDbObj *pSaved = mnodeGetDb(pDb->name); - if (pDb != pSaved) { - memcpy(pSaved, pDb, pOper->rowSize); - free(pDb); + SDbObj *pNew = pOper->pObj; + SDbObj *pDb = mnodeGetDb(pNew->name); + if (pDb != NULL && pNew != pDb) { + memcpy(pDb, pNew, pOper->rowSize); + free(pNew->vgList); + free(pNew); } - mnodeUpdateAllDbVgroups(pSaved); - mnodeDecDbRef(pSaved); + //mnodeUpdateAllDbVgroups(pDb); + mnodeDecDbRef(pDb); return TSDB_CODE_SUCCESS; } @@ -179,9 +188,14 @@ void mnodeDecDbRef(SDbObj *pDb) { SDbObj *mnodeGetDbByTableId(char *tableId) { char db[TSDB_TABLE_ID_LEN], *pos; - + + // tableId format should be : acct.db.table pos = strstr(tableId, TS_PATH_DELIMITER); + assert(NULL != pos); + pos = strstr(pos + 1, TS_PATH_DELIMITER); + assert(NULL != pos); + memset(db, 0, sizeof(db)); strncpy(db, tableId, pos - tableId); @@ -273,9 +287,9 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_MND_INVALID_DB_OPTION; } - if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { - mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, - TSDB_MAX_REPLICA_NUM); + if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_DB_REPLICA_OPTION, + TSDB_MAX_DB_REPLICA_OPTION); return TSDB_CODE_MND_INVALID_DB_OPTION; } @@ -379,7 +393,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { - tfree(pDb); + mnodeDestroyDb(pDb); mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); return code; } else { @@ -416,45 +430,33 @@ void mnodePrintVgroups(SDbObj *pDb, char *oper) { void mnodeAddVgroupIntoDb(SVgObj *pVgroup) { SDbObj *pDb = pVgroup->pDb; - pVgroup->next = pDb->pHead; - pVgroup->prev = NULL; + pthread_mutex_lock(&pDb->mutex); + int32_t vgPos = pDb->numOfVgroups++; + if (vgPos >= pDb->vgListSize) { + pDb->vgList = realloc(pDb->vgList, pDb->vgListSize * 2 * sizeof(SVgObj *)); + memset(pDb->vgList + pDb->vgListSize, 0, pDb->vgListSize * sizeof(SVgObj *)); + pDb->vgListSize *= 2; + } - if (pDb->pHead) pDb->pHead->prev = pVgroup; - if (pDb->pTail == NULL) pDb->pTail = pVgroup; - - pDb->pHead = pVgroup; - pDb->numOfVgroups++; -} - -void mnodeAddVgroupIntoDbTail(SVgObj *pVgroup) { - SDbObj *pDb = pVgroup->pDb; - pVgroup->next = NULL; - pVgroup->prev = pDb->pTail; - - if (pDb->pTail) pDb->pTail->next = pVgroup; - if (pDb->pHead == NULL) pDb->pHead = pVgroup; - - pDb->pTail = pVgroup; - pDb->numOfVgroups++; + pDb->vgList[vgPos] = pVgroup; + pthread_mutex_unlock(&pDb->mutex); } void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) { SDbObj *pDb = pVgroup->pDb; - if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; - if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; - if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; - if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; - pDb->numOfVgroups--; -} -void mnodeMoveVgroupToTail(SVgObj *pVgroup) { - mnodeRemoveVgroupFromDb(pVgroup); - mnodeAddVgroupIntoDbTail(pVgroup); -} + pthread_mutex_lock(&pDb->mutex); + for (int32_t v1 = 0; v1 < pDb->numOfVgroups; ++v1) { + if (pDb->vgList[v1] == pVgroup) { + for (int32_t v2 = v1; v2 < pDb->numOfVgroups - 1; ++v2) { + pDb->vgList[v2] = pDb->vgList[v2 + 1]; + } + pDb->numOfVgroups--; + break; + } + } -void mnodeMoveVgroupToHead(SVgObj *pVgroup) { - mnodeRemoveVgroupFromDb(pVgroup); - mnodeAddVgroupIntoDb(pVgroup); + pthread_mutex_unlock(&pDb->mutex); } void mnodeCleanupDbs() { @@ -526,11 +528,6 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn #ifndef __CLOUD_VERSION__ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) { #endif - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "maxtables"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -556,12 +553,6 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "ctime(Sec.)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "wallevel"); @@ -671,10 +662,6 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void #ifndef __CLOUD_VERSION__ if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) { #endif - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1 - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; cols++; @@ -691,10 +678,6 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.commitTime; - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int8_t *)pWrite = pDb->cfg.walLevel; cols++; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 29272fbd4f..a65f54cd8b 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -69,6 +69,7 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { SDnodeObj *pDnode = pOper->pObj; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; + pDnode->lastAccess = tsAccessSquence; } return TSDB_CODE_SUCCESS; @@ -187,7 +188,27 @@ int32_t mnodeGetDnodesNum() { return sdbGetNumOfRows(tsDnodeSdb); } -int32_t mnodeGetOnlinDnodesNum(char *ep) { +int32_t mnodeGetOnlinDnodesCpuCoreNum() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + int32_t cpuCores = 0; + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (pDnode->status != TAOS_DN_STATUS_OFFLINE) { + cpuCores += pDnode->numOfCores; + } + mnodeDecDnodeRef(pDnode); + } + + sdbFreeIter(pIter); + + if (cpuCores < 2) cpuCores = 2; + return cpuCores; +} + +int32_t mnodeGetOnlinDnodesNum() { SDnodeObj *pDnode = NULL; void * pIter = NULL; int32_t onlineDnodes = 0; @@ -283,15 +304,17 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { } static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { - if (clusterCfg->numOfMnodes != tsNumOfMnodes) return false; - if (clusterCfg->mnodeEqualVnodeNum != tsMnodeEqualVnodeNum) return false; - if (clusterCfg->offlineThreshold != tsOfflineThreshold) return false; - if (clusterCfg->statusInterval != tsStatusInterval) return false; + if (clusterCfg->numOfMnodes != htonl(tsNumOfMnodes)) return false; + if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) return false; + if (clusterCfg->offlineThreshold != htonl(tsOfflineThreshold)) return false; + if (clusterCfg->statusInterval != htonl(tsStatusInterval)) return false; + if (clusterCfg->maxtablesPerVnode != htonl(tsMaxTablePerVnode)) return false; + if (clusterCfg->maxVgroupsPerDb != htonl(tsMaxVgroupsPerDb)) return false; if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) return false; if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) return false; - if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) return false; - if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) return false; + if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) return false; + if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) return false; return true; } @@ -376,7 +399,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { if (false == ret) { mnodeDecDnodeRef(pDnode); rpcFreeCont(pRsp); - mError("dnode %s cluster cfg parameters inconsistent", pStatus->dnodeEp); + mError("dnode:%d, %s cluster cfg parameters inconsistent", pDnode->dnodeId, pStatus->dnodeEp); return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; } @@ -468,18 +491,22 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - mnodeDecDnodeRef(pDnode); if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) { mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep); + mnodeDecDnodeRef(pDnode); return TSDB_CODE_MND_NO_REMOVE_MASTER; } mInfo("dnode:%d, start to drop it", pDnode->dnodeId); + #ifndef _SYNC - return mnodeDropDnode(pDnode, pMsg); + int32_t code = mnodeDropDnode(pDnode, pMsg); #else - return balanceDropDnode(pDnode); + int32_t code = balanceDropDnode(pDnode); #endif + + mnodeDecDnodeRef(pDnode); + return code; } static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index cdcb7357f1..c8e4ae956c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -264,8 +264,12 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { if (pOper->cb != NULL) { pOper->retCode = (*pOper->cb)(pMsg, pOper->retCode); } - dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode); + + // if ahandle, means this func is called by sdb write + if (ahandle == NULL) { + sdbDecRef(pOper->table, pOper->pObj); + } taosFreeQitem(pOper); } @@ -389,7 +393,7 @@ void sdbCleanUp() { } void sdbIncRef(void *handle, void *pObj) { - if (pObj == NULL) return; + if (pObj == NULL || handle == NULL) return; SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); @@ -398,7 +402,7 @@ void sdbIncRef(void *handle, void *pObj) { } void sdbDecRef(void *handle, void *pObj) { - if (pObj == NULL) return; + if (pObj == NULL || handle == NULL) return; SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); @@ -581,16 +585,26 @@ static int sdbWrite(void *param, void *data, int type) { return sdbInsertHash(pTable, &oper); } else if (action == SDB_ACTION_DELETE) { SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); + if (rowMeta == NULL || rowMeta->row == NULL) { + sdbError("table:%s, failed to get object:%s from wal while dispose delete action", pTable->tableName, + pHead->cont); + return TSDB_CODE_SUCCESS; + } SSdbOper oper = {.table = pTable, .pObj = rowMeta->row}; return sdbDeleteHash(pTable, &oper); } else if (action == SDB_ACTION_UPDATE) { SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); + if (rowMeta == NULL || rowMeta->row == NULL) { + sdbError("table:%s, failed to get object:%s from wal while dispose update action", pTable->tableName, + pHead->cont); + return TSDB_CODE_SUCCESS; + } SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; code = (*pTable->decodeFp)(&oper); return sdbUpdateHash(pTable, &oper); - } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } + } else { + return TSDB_CODE_MND_INVALID_MSG_TYPE; + } } int32_t sdbInsertRow(SSdbOper *pOper) { @@ -647,6 +661,14 @@ int32_t sdbInsertRow(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } +bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { + SSdbTable *pTable = pTableInput; + if (pTable == NULL) return false; + + int8_t *updateEnd = pRow + pTable->refCountPos - 1; + return (*updateEnd == 1); +} + int32_t sdbDeleteRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -663,14 +685,18 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { return TSDB_CODE_MND_SDB_INVAID_META_ROW; } + sdbIncRef(pTable, pOper->pObj); + int32_t code = sdbDeleteHash(pTable, pOper); if (code != TSDB_CODE_SUCCESS) { sdbError("table:%s, failed to delete from hash", pTable->tableName); + sdbDecRef(pTable, pOper->pObj); return code; } // just delete data from memory if (pOper->type != SDB_OPER_GLOBAL) { + sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -692,7 +718,6 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); } - sdbIncRef(pNewOper->table, pNewOper->pObj); taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_SUCCESS; } @@ -968,10 +993,11 @@ static void *sdbWorkerFp(void *param) { int32_t code = sdbWrite(pOper, pHead, type); if (code > 0) code = 0; - if (pOper) + if (pOper) { pOper->retCode = code; - else + } else { pHead->len = code; // hackway + } } walFsync(tsSdbObj.wal); @@ -983,7 +1009,6 @@ static void *sdbWorkerFp(void *param) { if (type == TAOS_QTYPE_RPC) { pOper = (SSdbOper *)item; - sdbDecRef(pOper->table, pOper->pObj); sdbConfirmForward(NULL, pOper, pOper->retCode); } else if (type == TAOS_QTYPE_FWD) { pHead = (SWalHead *)item; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 523d7001c2..28661dc1ad 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -72,7 +72,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg); static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg); static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg); -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg); +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn); static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg); @@ -319,15 +319,6 @@ static int32_t mnodeChildTableActionRestored() { continue; } - if (pVgroup->tableList == NULL) { - mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId); - pTable->vgId = 0; - SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; - sdbDeleteRow(&desc); - mnodeDecTableRef(pTable); - continue; - } - if (pTable->info.type == TSDB_CHILD_TABLE) { SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); if (pSuperTable == NULL) { @@ -385,7 +376,7 @@ static void mnodeCleanupChildTables() { } static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { - pStable->numOfTables++; + atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); @@ -394,18 +385,22 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt if (pStable->vgHash != NULL) { if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); + mDebug("table:%s, vgId:%d is put into stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + (int32_t)taosHashGetSize(pStable->vgHash)); } } } static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { - pStable->numOfTables--; + atomic_sub_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) return; SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); if (pVgroup == NULL) { taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); + mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + (int32_t)taosHashGetSize(pStable->vgHash)); } mnodeDecVgroupRef(pVgroup); } @@ -757,11 +752,15 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - mInfo("app:%p:%p, table:%s, start to drop stable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable; + mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", + pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { - mInfo("app:%p:%p, table:%s, start to drop ctable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); - return mnodeProcessDropChildTableMsg(pMsg); + SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; + mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + pDrop->tableId, pCTable->vgId, pCTable->sid, pCTable->uid); + return mnodeProcessDropChildTableMsg(pMsg, true); } } @@ -808,7 +807,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { assert(pTable); if (code == TSDB_CODE_SUCCESS) { - mLInfo("stable:%s, is created in sdb", pTable->info.tableId); + mLInfo("stable:%s, is created in sdb, uid:%" PRIu64, pTable->info.tableId, pTable->uid); } else { mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); @@ -896,7 +895,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; - if (pStable->numOfTables != 0) { + if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); while (taosHashIterNext(pIter)) { int32_t *pVgId = taosHashIterGet(pIter); @@ -1600,8 +1599,8 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { assert(pTable); if (code == TSDB_CODE_SUCCESS) { - mDebug("app:%p:%p, table:%s, create table in sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, - pTable->sid, pTable->uid); + mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, + pMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, tstrerror(code)); } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); @@ -1730,19 +1729,15 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return code; } - SVgObj *pVgroup = mnodeGetAvailableVgroup(pMsg->pDb); - if (pVgroup == NULL) { - mDebug("app:%p:%p, table:%s, start to create a new vgroup", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); - return mnodeCreateVgroup(pMsg, pMsg->pDb); - } - if (pMsg->retry == 0) { if (pMsg->pTable == NULL) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid <= 0) { - mDebug("app:%p:%p, table:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, - pVgroup->vgId); - return mnodeCreateVgroup(pMsg, pMsg->pDb); + SVgObj *pVgroup = NULL; + int32_t sid = 0; + code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &sid); + if (code != TSDB_CODE_SUCCESS) { + mDebug("app:%p:%p, table:%s, failed to get available vgroup, reason:%s", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId, tstrerror(code)); + return code; } if (pMsg->pVgroup == NULL) { @@ -1750,7 +1745,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { mnodeIncVgroupRef(pVgroup); } - mDebug("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, pVgroup->vgId, sid); return mnodeDoCreateChildTable(pMsg, sid); @@ -1769,7 +1764,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } } -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { @@ -1793,7 +1788,9 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); - mInfo("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); + SRpcMsg rpcMsg = { .ahandle = pMsg, .pCont = pDrop, @@ -1802,6 +1799,8 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE }; + if (!needReturn) rpcMsg.ahandle = NULL; + dnodeSendMsgToDnode(&ipSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -2125,7 +2124,7 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { int32_t numOfTables = 0; SChildTableObj *pTable = NULL; - mInfo("stable:%s, all child tables(%d) will dropped from sdb", pStable->info.tableId, numOfTables); + mInfo("stable:%s, all child tables:%d will dropped from sdb", pStable->info.tableId, pStable->numOfTables); while (1) { pIter = mnodeGetNextChildTable(pIter, &pTable); @@ -2149,6 +2148,7 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } +#if 0 static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) { SVgObj *pVgroup = mnodeGetVgroup(vnode); if (pVgroup == NULL) return NULL; @@ -2159,8 +2159,11 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) { mnodeDecVgroupRef(pVgroup); return pTable; } +#endif static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { + return TSDB_CODE_COM_OPS_NOT_SUPPORT; +#if 0 SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); @@ -2184,6 +2187,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { pMsg->rpcRsp.rsp = pCreate; pMsg->rpcRsp.len = htonl(pCreate->contLen); return TSDB_CODE_SUCCESS; +#endif } // handle drop child response @@ -2195,12 +2199,15 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); - mInfo("app:%p:%p, table:%s, drop table rsp received, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + + mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, + mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, failed to drop in dnode, reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - pTable->info.tableId, tstrerror(rpcMsg->code)); + mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, + tstrerror(rpcMsg->code)); dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); return; } @@ -2247,6 +2254,14 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); + // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode + if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) { + mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid); + mnodeProcessDropChildTableMsg(mnodeMsg, false); + rpcMsg->code = TSDB_CODE_SUCCESS; + } + if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont; if (pCreate->getMeta) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 966d4b0dd8..5658f4bcbf 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -46,6 +46,7 @@ typedef enum { static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; +static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup); static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg); @@ -53,17 +54,17 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) ; static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); -static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { - SVgObj *pVgroup = pOper->pObj; +static void mnodeDestroyVgroup(SVgObj *pVgroup) { if (pVgroup->idPool) { taosIdPoolCleanUp(pVgroup->idPool); pVgroup->idPool = NULL; } - if (pVgroup->tableList) { - tfree(pVgroup->tableList); - } - tfree(pOper->pObj); + tfree(pVgroup); +} + +static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { + mnodeDestroyVgroup(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -82,21 +83,9 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { } pVgroup->pDb = pDb; - pVgroup->prev = NULL; - pVgroup->next = NULL; pVgroup->accessState = TSDB_VN_ALL_ACCCESS; - - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; - pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *)); - if (pVgroup->tableList == NULL) { - mError("vgId:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); - return -1; - } - - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables); - if (pVgroup->idPool == NULL) { - mError("vgId:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); - tfree(pVgroup->tableList); + if (mnodeAllocVgroupIdPool(pVgroup) < 0) { + mError("vgId:%d, failed to init idpool for vgroups", pVgroup->vgId); return -1; } @@ -134,20 +123,6 @@ static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static void mnodeVgroupUpdateIdPool(SVgObj *pVgroup) { - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - SDbObj *pDb = pVgroup->pDb; - if (pDb != NULL) { - if (pDb->cfg.maxTables != oldTables) { - mInfo("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables); - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; - pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size); - memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj *)); - } - } -} - static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { SVgObj *pNew = pOper->pObj; SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); @@ -174,7 +149,6 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { free(pNew); } - mnodeVgroupUpdateIdPool(pVgroup); // reset vgid status on vgroup changed mDebug("vgId:%d, reset sync status to unsynced", pVgroup->vgId); @@ -344,8 +318,132 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl } } -SVgObj *mnodeGetAvailableVgroup(SDbObj *pDb) { - return pDb->pHead; +static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) { + SDbObj *pDb = pInputVgroup->pDb; + if (pDb == NULL) return TSDB_CODE_MND_APP_ERROR; + + int32_t minIdPoolSize = TSDB_MAX_TABLES; + int32_t maxIdPoolSize = TSDB_MIN_TABLES; + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { + SVgObj *pVgroup = pDb->vgList[v]; + if (pVgroup == NULL) continue; + + int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); + minIdPoolSize = MIN(minIdPoolSize, idPoolSize); + maxIdPoolSize = MAX(maxIdPoolSize, idPoolSize); + } + + // new vgroup + if (pInputVgroup->idPool == NULL) { + pInputVgroup->idPool = taosInitIdPool(maxIdPoolSize); + if (pInputVgroup->idPool == NULL) { + mError("vgId:%d, failed to init idPool for vgroup, size:%d", pInputVgroup->vgId, maxIdPoolSize); + return TSDB_CODE_MND_OUT_OF_MEMORY; + } else { + mDebug("vgId:%d, init idPool for vgroup, size:%d", pInputVgroup->vgId, maxIdPoolSize); + return TSDB_CODE_SUCCESS; + } + } + + // realloc all vgroups in db + int32_t newIdPoolSize; + if (minIdPoolSize * 4 < TSDB_TABLES_STEP) { + newIdPoolSize = minIdPoolSize * 4; + } else { + newIdPoolSize = ((minIdPoolSize / TSDB_TABLES_STEP) + 1) * TSDB_TABLES_STEP; + } + + if (newIdPoolSize > tsMaxTablePerVnode) { + if (minIdPoolSize >= tsMaxTablePerVnode) { + mError("db:%s, minIdPoolSize:%d newIdPoolSize:%d larger than maxTablesPerVnode:%d", pDb->name, minIdPoolSize, newIdPoolSize, + tsMaxTablePerVnode); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } else { + newIdPoolSize = tsMaxTablePerVnode; + } + } + + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { + SVgObj *pVgroup = pDb->vgList[v]; + if (pVgroup == NULL) continue; + + int32_t oldIdPoolSize = taosIdPoolMaxSize(pVgroup->idPool); + if (newIdPoolSize == oldIdPoolSize) continue; + + if (taosUpdateIdPool(pVgroup->idPool, newIdPoolSize) < 0) { + mError("vgId:%d, failed to update idPoolSize from %d to %d", pVgroup->vgId, oldIdPoolSize, newIdPoolSize); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } else { + mDebug("vgId:%d, idPoolSize update from %d to %d", pVgroup->vgId, oldIdPoolSize, newIdPoolSize); + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSid) { + SDbObj *pDb = pMsg->pDb; + pthread_mutex_lock(&pDb->mutex); + + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { + int vgIndex = (v + pDb->vgListIndex) % pDb->numOfVgroups; + SVgObj *pVgroup = pDb->vgList[vgIndex]; + if (pVgroup == NULL) { + mError("db:%s, index:%d vgroup is null", pDb->name, vgIndex); + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_APP_ERROR; + } + + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid <= 0) { + mDebug("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId); + continue; + } + + *pSid = sid; + *ppVgroup = pVgroup; + pDb->vgListIndex = vgIndex; + + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_SUCCESS; + } + + int maxVgroupsPerDb = tsMaxVgroupsPerDb; + if (maxVgroupsPerDb <= 0) { + maxVgroupsPerDb = mnodeGetOnlinDnodesCpuCoreNum(); + maxVgroupsPerDb = MAX(maxVgroupsPerDb, 2); + } + + if (pDb->numOfVgroups < maxVgroupsPerDb) { + mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, pMsg, + pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); + pthread_mutex_unlock(&pDb->mutex); + int32_t code = mnodeCreateVgroup(pMsg); + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return code; + } + + SVgObj *pVgroup = pDb->vgList[0]; + if (pVgroup == NULL) return TSDB_CODE_MND_NO_ENOUGH_DNODES; + + int32_t code = mnodeAllocVgroupIdPool(pVgroup); + if (code != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&pDb->mutex); + return code; + } + + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid <= 0) { + mError("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId); + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } + + *pSid = sid; + *ppVgroup = pVgroup; + pDb->vgListIndex = 0; + pthread_mutex_unlock(&pDb->mutex); + + return TSDB_CODE_SUCCESS; } void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { @@ -378,8 +476,9 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { +int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + SDbObj *pDb = pMsg->pDb; SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN); @@ -407,7 +506,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { pMsg->pVgroup = NULL; - tfree(pVgroup); + mnodeDestroyVgroup(pVgroup); } else { code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -461,29 +560,27 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - int32_t maxReplica = 0; - SVgObj *pVgroup = NULL; - STableObj *pTable = NULL; - if (pShow->payloadLen > 0 ) { - pTable = mnodeGetTable(pShow->payload); - if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) { - mnodeDecTableRef(pTable); - return TSDB_CODE_MND_INVALID_TABLE_NAME; - } - mnodeDecTableRef(pTable); - pVgroup = mnodeGetVgroup(((SChildTableObj*)pTable)->vgId); - if (NULL == pVgroup) return TSDB_CODE_MND_INVALID_TABLE_NAME; - mnodeDecVgroupRef(pVgroup); - maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; - } else { - SVgObj *pVgroup = pDb->pHead; - while (pVgroup != NULL) { - maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; - pVgroup = pVgroup->next; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "poolSize"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "maxTables"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->maxReplica = 1; + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { + SVgObj *pVgroup = pDb->vgList[v]; + if (pVgroup != NULL) { + pShow->maxReplica = pVgroup->numOfVnodes > pShow->maxReplica ? pVgroup->numOfVnodes : pShow->maxReplica; } } - for (int32_t i = 0; i < maxReplica; ++i) { + for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "dnode"); @@ -507,48 +604,54 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->numOfColumns = cols; pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - - if (NULL == pTable) { - pShow->numOfRows = pDb->numOfVgroups; - pShow->pIter = pDb->pHead; - } else { - pShow->numOfRows = 1; - pShow->pIter = pVgroup; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - mnodeDecDbRef(pDb); + pShow->numOfRows = pDb->numOfVgroups; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + mnodeDecDbRef(pDb); return 0; } +static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) { + if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) { + return true; + } + + SChildTableObj *pCTable = (SChildTableObj *)pTable; + if (pVgroup->vgId == pCTable->vgId) { + return true; + } else { + return false; + } +} + int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SVgObj *pVgroup = NULL; - int32_t maxReplica = 0; int32_t cols = 0; char * pWrite; SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); return 0; } - pVgroup = pDb->pHead; - while (pVgroup != NULL) { - maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; - pVgroup = pVgroup->next; + STableObj *pTable = NULL; + if (pShow->payloadLen > 0 ) { + pTable = mnodeGetTable(pShow->payload); } while (numOfRows < rows) { - pVgroup = (SVgObj *) pShow->pIter; + pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup); if (pVgroup == NULL) break; - pShow->pIter = (void *) pVgroup->next; + if (pVgroup->pDb != pDb) continue; + if (!mnodeFilterVgroups(pVgroup, pTable)) continue; cols = 0; @@ -560,7 +663,15 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC *(int32_t *) pWrite = pVgroup->numOfTables; cols++; - for (int32_t i = 0; i < maxReplica; ++i) { + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = taosIdPoolMaxSize(pVgroup->idPool); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = tsMaxTablePerVnode; + cols++; + + for (int32_t i = 0; i < pShow->maxReplica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; cols++; @@ -588,38 +699,36 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC } } + mnodeDecVgroupRef(pVgroup); numOfRows++; } pShow->numOfReads += numOfRows; + mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); return numOfRows; } void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { - if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] == NULL) { - pVgroup->tableList[pTable->sid - 1] = pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); - pVgroup->numOfTables++; - } - - if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) { - mnodeMoveVgroupToTail(pVgroup); + int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); + if (pTable->sid > idPoolSize) { + mnodeAllocVgroupIdPool(pVgroup); } - mnodeIncVgroupRef(pVgroup); + if (pTable->sid >= 1) { + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; + mnodeIncVgroupRef(pVgroup); + } } void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { - if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] != NULL) { - pVgroup->tableList[pTable->sid - 1] = NULL; + if (pTable->sid >= 1) { taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; + mnodeDecVgroupRef(pVgroup); } - - mnodeMoveVgroupToHead(pVgroup); - mnodeDecVgroupRef(pVgroup); } SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) { @@ -630,13 +739,16 @@ SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) { if (pVnode == NULL) return NULL; strcpy(pVnode->db, pVgroup->dbName); + int32_t maxTables = taosIdPoolMaxSize(pVgroup->idPool); + //TODO: dynamic alloc tables in tsdb + maxTables = MAX(10000, tsMaxTablePerVnode); SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); - pCfg->maxTables = htonl(pDb->cfg.maxTables + 1); + pCfg->maxTables = htonl(maxTables + 1); pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); @@ -857,9 +969,10 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { sdbFreeIter(pIter); - mInfo("dnode:%d, all vgroups is dropped from sdb", pDropDnode->dnodeId); + mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups); } +#if 0 void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { void * pIter = NULL; SVgObj *pVgroup = NULL; @@ -881,6 +994,7 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name); } +#endif void mnodeDropAllDbVgroups(SDbObj *pDropDb) { void * pIter = NULL; diff --git a/src/plugins/http/inc/httpCode.h b/src/plugins/http/inc/httpCode.h index 0235040139..08111260e9 100644 --- a/src/plugins/http/inc/httpCode.h +++ b/src/plugins/http/inc/httpCode.h @@ -105,7 +105,7 @@ #define HTTP_OP_VALUE_TYPE 79 //tgf -#define HTTP_TG_STABLE_NOT_EXIST 80 +#define HTTP_TG_STABLE_NOT_EXIST 80 extern char *httpMsg[]; diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 8ca1c2ff11..ffd621be7a 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -61,6 +61,9 @@ #define HTTP_CHECK_BODY_CONTINUE 0 #define HTTP_CHECK_BODY_SUCCESS 1 +#define HTTP_READ_DATA_SUCCESS 0 +#define HTTP_READ_DATA_FAILED 1 + #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 #define HTTP_EXPIRED_TIME 60000 diff --git a/src/plugins/http/inc/httpServer.h b/src/plugins/http/inc/httpServer.h index 04dadfe04c..508baa6112 100644 --- a/src/plugins/http/inc/httpServer.h +++ b/src/plugins/http/inc/httpServer.h @@ -23,6 +23,6 @@ void httpCleanUpConnect(); void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); void httpCleanUpServer(HttpServer *pServer); -bool httpReadDataImp(HttpContext *pContext); +int httpReadDataImp(HttpContext *pContext); #endif diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 2c94f61950..407d19b307 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -60,6 +60,7 @@ bool httpParseURL(HttpContext* pContext) { char* pSeek; char* pEnd = strchr(pParser->pLast, ' '); if (pEnd == NULL) { + httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); return false; } @@ -275,14 +276,14 @@ bool httpParseChunkedBody(HttpContext* pContext, HttpParser* pParser, bool test) return true; } -bool httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { +int httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { bool parsedOk = httpParseChunkedBody(pContext, pParser, true); if (parsedOk) { httpParseChunkedBody(pContext, pParser, false); return HTTP_CHECK_BODY_SUCCESS; } else { httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); - if (!httpReadDataImp(pContext)) { + if (httpReadDataImp(pContext) != HTTP_READ_DATA_SUCCESS) { httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); return HTTP_CHECK_BODY_ERROR; } else { @@ -296,7 +297,6 @@ int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) { if (dataReadLen > pParser->data.len) { httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d", pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); - httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR); return HTTP_CHECK_BODY_ERROR; } else if (dataReadLen < pParser->data.len) { httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", @@ -358,20 +358,13 @@ bool httpParseRequest(HttpContext* pContext) { } int httpCheckReadCompleted(HttpContext* pContext) { - HttpParser *pParser = &pContext->parser; - if (pContext->httpChunked == HTTP_UNCUNKED) { - int ret = httpReadUnChunkedBody(pContext, pParser); - if (ret != HTTP_CHECK_BODY_SUCCESS) { - return ret; - } - } else { - int ret = httpReadChunkedBody(pContext, pParser); - if (ret != HTTP_CHECK_BODY_SUCCESS) { - return ret; - } - } + HttpParser* pParser = &pContext->parser; - return HTTP_CHECK_BODY_SUCCESS; + if (pContext->httpChunked == HTTP_UNCUNKED) { + return httpReadUnChunkedBody(pContext, pParser); + } else { + return httpReadChunkedBody(pContext, pParser); + } } bool httpDecodeRequest(HttpContext* pContext) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index dbe299cef7..177d447f10 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -69,7 +69,7 @@ void httpCleanUpConnect() { httpDebug("http server:%s is cleaned up", pServer->label); } -bool httpReadDataImp(HttpContext *pContext) { +int httpReadDataImp(HttpContext *pContext) { HttpParser *pParser = &pContext->parser; while (pParser->bufsize <= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { @@ -85,8 +85,7 @@ bool httpReadDataImp(HttpContext *pContext) { } else { httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", pContext, pContext->fd, pContext->ipstr, errno); - httpReleaseContext(pContext); - return false; + return HTTP_READ_DATA_FAILED; } } else { pParser->bufsize += nread; @@ -95,15 +94,13 @@ bool httpReadDataImp(HttpContext *pContext) { if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d", pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE); - httpSendErrorResp(pContext, HTTP_REQUSET_TOO_BIG); - httpNotifyContextClose(pContext); - return false; + return HTTP_REQUSET_TOO_BIG; } } pParser->buffer[pParser->bufsize] = 0; - return true; + return HTTP_READ_DATA_SUCCESS; } static bool httpDecompressData(HttpContext *pContext) { @@ -141,8 +138,14 @@ static bool httpReadData(HttpContext *pContext) { httpInitContext(pContext); } - if (!httpReadDataImp(pContext)) { - httpNotifyContextClose(pContext); + int32_t code = httpReadDataImp(pContext); + if (code != HTTP_READ_DATA_SUCCESS) { + if (code == HTTP_READ_DATA_FAILED) { + httpReleaseContext(pContext); + } else { + httpSendErrorResp(pContext, code); + httpNotifyContextClose(pContext); + } return false; } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index c43d928d1b..7a515d124e 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -233,10 +233,11 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num } } -void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { +void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; + int32_t code = taos_errno(result); HttpEncodeMethod *encode = pContext->encodeMethod; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -260,8 +261,8 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { return; } - int num_fields = taos_field_count(result); - if (num_fields == 0) { + bool isUpdate = tscIsUpdateQuery(result); + if (isUpdate) { // not select or show commands int affectRows = taos_affected_rows(result); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 6cd6edd6db..1faad6b141 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -192,7 +192,6 @@ typedef struct SQInfo { int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; - T_REF_DECLARE() /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. @@ -201,8 +200,6 @@ typedef struct SQInfo { */ int32_t tableIndex; int32_t numOfGroupResultPages; - _qinfo_free_fn_t freeFn; //todo remove it - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; } SQInfo; diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index e437e1c4e5..59b224e096 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -107,7 +107,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order); STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder); -void* tsBufDestory(STSBuf* pTSBuf); +void* tsBufDestroy(STSBuf* pTSBuf); void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 63b7abb379..0c69bc02d3 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -187,7 +187,7 @@ typedef struct SQLFunctionCtx { } SQLFunctionCtx; typedef struct SQLAggFuncElem { - char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; + char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; uint8_t nAggIdx; // index of function in aAggs int8_t stableFuncId; // transfer function for super table query diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 53a32a2356..ed7a86d843 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -44,7 +44,7 @@ #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) +#define GET_QINFO_ADDR(x) ((SQInfo *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) #define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -120,6 +120,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { #define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) static void setQueryStatus(SQuery *pQuery, int8_t status); +static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0) @@ -351,27 +352,6 @@ static bool hasTagValOutput(SQuery* pQuery) { return false; } -static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, int32_t numOfCols, int32_t index) { - // for a tag column, no corresponding field info - SColIndex *pColIndex = &pQuery->pSelectExpr[index].base.colInfo; - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - return NULL; - } - - /* - * Choose the right column field info by field id, since the file block may be out of date, - * which means the newest table schema is not equalled to the schema of this block. - * TODO: speedup by using bsearch - */ - for (int32_t i = 0; i < numOfCols; ++i) { - if (pColIndex->colId == pStatis[i].colId) { - return &pStatis[i]; - } - } - - return NULL; -} - /** * @param pQuery * @param col @@ -380,19 +360,14 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, int32_t * @param pColStatis * @return */ -static bool hasNullValue(SQuery *pQuery, int32_t col, int32_t numOfCols, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - return false; - } - - // query on primary timestamp column, not null value at all - if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { +static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } if (pStatis != NULL) { - *pColStatis = getStatisInfo(pQuery, pStatis, numOfCols, col); + *pColStatis = &pStatis[pColIndex->colIndex]; + assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } @@ -842,8 +817,8 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (pDataBlock == NULL) { return NULL; } - char *dataBlock = NULL; + char *dataBlock = NULL; SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -864,6 +839,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); if (sas->data == NULL) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -887,10 +863,14 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { + if (TSDB_COL_IS_TAG(pCol->flag)) { dataBlock = NULL; } else { - dataBlock = getDataBlockImpl(pDataBlock, pCol->colId); + SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; + SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); + assert(p->info.colId == pColIndex->colId); + + dataBlock = p->pData; } } @@ -922,6 +902,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1168,6 +1149,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1365,7 +1347,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; SDataStatis *tpField = NULL; - pCtx->hasNull = hasNullValue(pQuery, colIndex, pBlockInfo->numOfCols, pStatis, &tpField); + pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; if (tpField != NULL) { @@ -1619,22 +1601,21 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); + pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); } -static bool isQueryKilled(SQInfo *pQInfo) { - return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED); -} +#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; } -static bool isFixedOutputQuery(SQuery *pQuery) { - if (pQuery->intervalTime != 0) { +static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { + SQuery* pQuery = pRuntimeEnv->pQuery; + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { return false; } // Note:top/bottom query is fixed output query - if (isTopBottomQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyNormalCol) { return true; } @@ -2054,7 +2035,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, // check if this data block is required to load for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base; - + int32_t functionId = pSqlFunc->functionId; int32_t colId = pSqlFunc->colInfo.colId; r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); @@ -2066,8 +2047,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } if (r == BLK_DATA_NO_NEEDED) { - qDebug("QInfo:%p data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("QInfo:%p data block discard, rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->rows); pRuntimeEnv->summary.discardBlocks += 1; } else if (r == BLK_DATA_STATIS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { @@ -2199,7 +2179,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pQuery)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { @@ -2249,8 +2229,10 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - return 0; + + if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); @@ -3304,8 +3286,9 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { cond.twindow.skey, cond.twindow.ekey); // check if query is killed or not - if (isQueryKilled(pQInfo)) { - return; + if (IS_QUERY_KILLED(pQInfo)) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } } @@ -3695,23 +3678,24 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } -static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { +static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; // update the number of result for each, only update the number of rows for the corresponding window result. - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + return; + } - for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i]; + for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { + SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pRuntimeEnv->pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { - continue; - } - - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pRuntimeEnv->pCtx[j].functionId; + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + continue; } + + pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); } } } @@ -3729,8 +3713,6 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo * } else { blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - - updateWindowResNumOfRes(pRuntimeEnv, pTableQueryInfo); } bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { @@ -3950,8 +3932,9 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - return; + if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); @@ -4099,7 +4082,7 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { return; } - if (isSTableQuery && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isFixedOutputQuery(pQuery))) { + if (isSTableQuery && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isFixedOutputQuery(pRuntimeEnv))) { return; } @@ -4115,7 +4098,7 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) - && (!isFixedOutputQuery(pQuery)) + && (!isFixedOutputQuery(pRuntimeEnv)) ) { SArray* pa = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); @@ -4158,7 +4141,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + pQuery->precision = tsdbGetCfg(tsdb)->precision; + pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); + pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQInfo, false); @@ -4236,10 +4222,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pQuery->fillType, pColInfo); } - // todo refactor - pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); - pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -4267,8 +4249,9 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - if (isQueryKilled(pQInfo)) { - break; + if (IS_QUERY_KILLED(pQInfo)) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); @@ -4304,6 +4287,8 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); } + updateWindowResNumOfRes(pRuntimeEnv); + int64_t et = taosGetTimestampMs(); return et - st; } @@ -4316,7 +4301,9 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SArray *group = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); - setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { + setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); + } STableId* id = TSDB_TABLEID(pCheckInfo->pTable); qDebug("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, @@ -4549,8 +4536,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) { - if (isQueryKilled(pQInfo)) { - return; + if (IS_QUERY_KILLED(pQInfo)) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pQuery->current = taosArrayGetP(group, pQInfo->tableIndex); @@ -4744,9 +4732,10 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qDebug("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el); // query error occurred or query is killed, abort current execution - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - return; + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } // close all time window results @@ -4766,9 +4755,10 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - return; + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { @@ -4806,8 +4796,9 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); - if (isQueryKilled(pQInfo)) { - return; + if (IS_QUERY_KILLED(pQInfo)) { + finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously. @@ -4839,10 +4830,6 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); finalizeQueryResult(pRuntimeEnv); - if (isQueryKilled(pQInfo)) { - return; - } - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { skipResults(pRuntimeEnv); @@ -4887,10 +4874,6 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) while (1) { scanOneTableDataBlocks(pRuntimeEnv, start); - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - return; - } - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); finalizeQueryResult(pRuntimeEnv); @@ -5024,7 +5007,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); - } else if (isFixedOutputQuery(pQuery)) { + } else if (isFixedOutputQuery(pRuntimeEnv)) { tableFixedOutputProcess(pQInfo, item); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); @@ -5044,7 +5027,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && !isFirstLastRowQuery(pQuery))) { multiTableQueryProcess(pQInfo); } else { @@ -5811,7 +5794,7 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param, _qinfo_free_fn_t fn) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5836,7 +5819,6 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ } pQInfo->param = param; - pQInfo->freeFn = fn; if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); @@ -6038,8 +6020,7 @@ typedef struct SQueryMgmt { pthread_mutex_t lock; } SQueryMgmt; -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, _qinfo_free_fn_t fn, - qinfo_t* pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) { assert(pQueryMsg != NULL && tsdb != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6129,7 +6110,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo goto _over; } - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param, fn); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param); _over: free(tagCond); @@ -6153,43 +6134,25 @@ _over: if (code != TSDB_CODE_SUCCESS) { *pQInfo = NULL; } else { - SQInfo* pq = (SQInfo*) (*pQInfo); +// SQInfo* pq = (SQInfo*) (*pQInfo); - T_REF_INC(pq); - T_REF_INC(pq); +// T_REF_INC(pq); +// T_REF_INC(pq); } // if failed to add ref for all meters in this query, abort current query return code; } -static void doDestoryQueryInfo(SQInfo* pQInfo) { - assert(pQInfo != NULL); - qDebug("QInfo:%p query completed", pQInfo); - queryCostStatis(pQInfo); // print the query cost summary - freeQInfo(pQInfo); -} - void qDestroyQueryInfo(qinfo_t qHandle) { SQInfo* pQInfo = (SQInfo*) qHandle; if (!isValidQInfo(pQInfo)) { return; } - int32_t ref = T_REF_DEC(pQInfo); - qDebug("QInfo:%p dec refCount, value:%d", pQInfo, ref); - - if (ref == 0) { - _qinfo_free_fn_t freeFp = pQInfo->freeFn; - void* param = pQInfo->param; - - doDestoryQueryInfo(pQInfo); - if (freeFp != NULL) { - assert(param != NULL); - freeFp(param); - } - - } + qDebug("QInfo:%p query completed", pQInfo); + queryCostStatis(pQInfo); // print the query cost summary + freeQInfo(pQInfo); } void qTableQuery(qinfo_t qinfo) { @@ -6200,31 +6163,24 @@ void qTableQuery(qinfo_t qinfo) { return; } - if (isQueryKilled(pQInfo)) { + if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo); return; } if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table exists for query, abort", pQInfo); - sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo); return; } - int32_t ret = setjmp(pQInfo->runtimeEnv.env); - // error occurs, record the error code and return to client + int32_t ret = setjmp(pQInfo->runtimeEnv.env); if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; - qDebug("QInfo:%p query abort due to error occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); + qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo); - return; } @@ -6241,7 +6197,7 @@ void qTableQuery(qinfo_t qinfo) { } SQuery* pQuery = pRuntimeEnv->pQuery; - if (isQueryKilled(pQInfo)) { + if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed", pQInfo); } else if (pQuery->rec.rows == 0) { qDebug("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); @@ -6251,7 +6207,6 @@ void qTableQuery(qinfo_t qinfo) { } sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { @@ -6262,7 +6217,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if (isQueryKilled(pQInfo)) { + if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); return pQInfo->code; } @@ -6295,7 +6250,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } if (ret) { - T_REF_INC(pQInfo); +// T_REF_INC(pQInfo); qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); } @@ -6337,7 +6292,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co code = pQInfo->code; } - if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client } @@ -6352,7 +6307,6 @@ int32_t qKillQuery(qinfo_t qinfo) { } setQueryKilled(pQInfo); - qDestroyQueryInfo(pQInfo); return TSDB_CODE_SUCCESS; } @@ -6497,6 +6451,7 @@ void freeqinfoFn(void *qhandle) { } qKillQuery(*handle); + qDestroyQueryInfo(*handle); } void* qOpenQueryMgmt(int32_t vgId) { @@ -6515,7 +6470,11 @@ void* qOpenQueryMgmt(int32_t vgId) { return pQueryHandle; } -void qSetQueryMgmtClosed(void* pQMgmt) { +static void queryMgmtKillQueryFn(void* handle) { + qKillQuery(handle); +} + +void qQueryMgmtNotifyClosed(void* pQMgmt) { if (pQMgmt == NULL) { return; } @@ -6527,7 +6486,7 @@ void qSetQueryMgmtClosed(void* pQMgmt) { pQueryMgmt->closed = true; pthread_mutex_unlock(&pQueryMgmt->lock); - taosCacheRefresh(pQueryMgmt->qinfoPool, freeqinfoFn); + taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } void qCleanupQueryMgmt(void* pQMgmt) { diff --git a/src/query/src/qtokenizer.c b/src/query/src/qtokenizer.c index 80d59a384e..784dd9af67 100644 --- a/src/query/src/qtokenizer.c +++ b/src/query/src/qtokenizer.c @@ -509,10 +509,11 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { for (i = 1; isdigit(z[i]); i++) { } - /* here is the 1a/2s/3m/9y */ - if ((z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' || z[i] == 'y' || - z[i] == 'w' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || - z[i] == 'Y' || z[i] == 'W') && + /* here is the 1u/1a/2s/3m/9y */ + if ((z[i] == 'u' || z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' || + z[i] == 'y' || z[i] == 'w' || + z[i] == 'U' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || + z[i] == 'Y' || z[i] == 'W') && (isIdChar[(uint8_t)z[i + 1]] == 0)) { *tokenType = TK_VARIABLE; i += 1; diff --git a/src/query/src/qtsbuf.c b/src/query/src/qtsbuf.c index fedaa315d3..b84fbded38 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qtsbuf.c @@ -79,7 +79,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } @@ -92,7 +92,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->tsOrder = header.tsOrder; if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } @@ -100,7 +100,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize); if (buf == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } @@ -120,7 +120,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { struct stat fileStat; if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } @@ -137,7 +137,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return pTSBuf; } -void* tsBufDestory(STSBuf* pTSBuf) { +void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return NULL; } @@ -920,13 +920,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE; pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx)); if (pTSBuf->pData == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } @@ -936,13 +936,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } pTSBuf->block.payload = malloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); return NULL; } diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 1d2d94f3c0..f813261957 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -47,7 +47,7 @@ void simpleTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.numOfElem, num); - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); } // one large list of ts, the ts list need to be split into several small blocks @@ -71,7 +71,7 @@ void largeTSTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.numOfElem, num); - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); } void multiTagsTest() { @@ -101,7 +101,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.numOfElem, num); - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); } void multiVnodeTagsTest() { @@ -139,7 +139,7 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.numOfElem, num); - tsBufDestory(pTSBuf); + tsBufDestroy(pTSBuf); } void loadDataTest() { @@ -386,8 +386,8 @@ void mergeDiffVnodeBufferTest() { tsBufDisplay(pTSBuf1); - tsBufDestory(pTSBuf2); - tsBufDestory(pTSBuf1); + tsBufDestroy(pTSBuf2); + tsBufDestroy(pTSBuf1); } void mergeIdenticalVnodeBufferTest() { @@ -432,8 +432,8 @@ void mergeIdenticalVnodeBufferTest() { printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); } - tsBufDestory(pTSBuf1); - tsBufDestory(pTSBuf2); + tsBufDestroy(pTSBuf1); + tsBufDestroy(pTSBuf2); } } // namespace diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 1fdd420d03..401ab1ed24 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -111,7 +111,8 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { if (pMemTable == NULL) return 0; - T_REF_INC(pMemTable); + int ref = T_REF_INC(pMemTable); + tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); return 0; } @@ -119,7 +120,9 @@ int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { if (pMemTable == NULL) return 0; - if (T_REF_DEC(pMemTable) == 0) { + int ref = T_REF_DEC(pMemTable); + tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); + if (ref == 0) { STsdbCfg * pCfg = &pRepo->config; STsdbBufPool *pBufPool = pRepo->pPool; @@ -159,6 +162,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; + tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); return 0; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 84c1c8e7d1..6d65e1e743 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -57,8 +57,30 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { STable * super = NULL; STable * table = NULL; int newSuper = 0; + int tid = pCfg->tableId.tid; + STable * pTable = NULL; - STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); + if (tid < 0 || tid >= pRepo->config.maxTables) { + tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid); + terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; + goto _err; + } + + if (pMeta->tables[tid] != NULL) { + if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { + tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), TABLE_UID(pTable)); + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } else { + tsdbError("vgId:%d table %s at tid %d uid %" PRIu64 + " exists, replace it with new table, this can be not reasonable", + REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), + TABLE_UID(pMeta->tables[tid])); + tsdbDropTable(pRepo, pMeta->tables[tid]->tableId); + } + } + + pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); if (pTable != NULL) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); @@ -72,10 +94,10 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { super = tsdbNewTable(pCfg, true); if (super == NULL) goto _err; } else { - // TODO - if (super->type != TSDB_SUPER_TABLE) return -1; - if (super->tableId.uid != pCfg->superUid) return -1; - // tsdbUpdateTable(pRepo, super, pCfg); + if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { + terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; + goto _err; + } } } @@ -705,6 +727,9 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { T_REF_INC(pTable); + tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + TABLE_UID(pTable)); + return pTable; _err: @@ -714,7 +739,9 @@ _err: static void tsdbFreeTable(STable *pTable) { if (pTable) { - if (pTable->name != NULL) tsdbDebug("table %s is destroyed", TABLE_CHAR_NAME(pTable)); + if (pTable->name != NULL) + tsdbDebug("table %s tid %d uid %" PRIu64 " is destroyed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + TABLE_UID(pTable)); tfree(TABLE_NAME(pTable)); if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { @@ -782,7 +809,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo tsdbGetTableSchemaImpl(pTable, false, false, -1)); } - tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index e935298afc..21efb1f60f 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -877,7 +877,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1); - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + tsdbTrace("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset), (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, @@ -948,7 +948,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); } - tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, + tsdbTrace("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); return 0; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a4e0151f89..c055a27c39 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -110,9 +110,10 @@ typedef struct STsdbQueryHandle { SFileGroupIter fileIter; SRWHelper rhelper; STableBlockInfo* pDataBlockInfo; + int32_t allocSize; // allocated data block size SMemTable* mem; // mem-table SMemTable* imem; // imem-table, acquired from snapshot - + SArray* defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ } STsdbQueryHandle; @@ -136,6 +137,34 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } +static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + assert(numOfCols <= TSDB_MAX_COLUMNS); + + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + taosArrayPush(pIdList, &pCol->info.colId); + } + + return pIdList; +} + +static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pQueryHandle); + + // check if the primary time stamp column needs to load + int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); + + // the primary timestamp column does not be included in the the specified load column list, add it + if (loadTS && colId != 0) { + int16_t columnId = 0; + taosArrayInsert(pLocalIdList, 0, &columnId); + } + + return pLocalIdList; +} + TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); pQueryHandle->order = pCond->order; @@ -148,7 +177,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->activeIndex = 0; // current active table index pQueryHandle->qinfo = qinfo; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; - + pQueryHandle->allocSize = 0; + tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb); tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); @@ -195,7 +225,9 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } - + + pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); @@ -546,33 +578,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo .tid = (_checkInfo)->tableId.tid, \ .uid = (_checkInfo)->tableId.uid}) -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - assert(numOfCols <= TSDB_MAX_COLUMNS); - - SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - taosArrayPush(pIdList, &pCol->info.colId); - } - - return pIdList; -} -static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); - - // check if the primary time stamp column needs to load - int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); - - // the primary timestamp column does not be included in the the specified load column list, add it - if (loadTS && colId != 0) { - int16_t columnId = 0; - taosArrayInsert(pLocalIdList, 0, &columnId); - } - - return pLocalIdList; -} static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { STsdbRepo *pRepo = pQueryHandle->pTsdb; @@ -584,8 +590,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo data->uid = pCheckInfo->pTableObj->tableId.uid; bool blockLoaded = false; - SArray* sa = getDefaultLoadColumns(pQueryHandle, true); - int64_t st = taosGetTimestampUs(); if (pCheckInfo->pDataCols == NULL) { @@ -613,7 +617,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; - taosArrayDestroy(sa); tfree(data); int64_t et = taosGetTimestampUs() - st; @@ -656,12 +659,8 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* return; } - SArray* sa = getDefaultLoadColumns(pQueryHandle, true); - doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); - taosArrayDestroy(sa); - + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); } else { /* * no data in cache, only load data from file @@ -681,14 +680,12 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* } static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { - SArray* sa = getDefaultLoadColumns(pQueryHandle, true); SQueryFilePos* cur = &pQueryHandle->cur; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { - taosArrayDestroy(sa); return false; } @@ -702,7 +699,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = 0; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); } else { // the whole block is loaded in to buffer handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } @@ -719,13 +716,12 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = pBlock->numOfRows - 1; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); } else { handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } } - taosArrayDestroy(sa); return pQueryHandle->realNumOfRows > 0; } @@ -1250,13 +1246,19 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* } static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { - char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); - if (tmp == NULL) { - return TSDB_CODE_TDB_OUT_OF_MEMORY; + size_t size = sizeof(STableBlockInfo) * numOfBlocks; + + if (pQueryHandle->allocSize < size) { + pQueryHandle->allocSize = size; + char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); + if (tmp == NULL) { + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + + pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; } - pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; - memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks); + memset(pQueryHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -1492,9 +1494,8 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return false; } - SArray* sa = getDefaultLoadColumns(pQueryHandle, true); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo); - /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, sa); + /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn); if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { // data already retrieve, discard other data rows and return @@ -1508,7 +1509,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pQueryHandle->window = pQueryHandle->cur.win; pQueryHandle->cur.rows = 1; pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; - taosArrayDestroy(sa); return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); @@ -1565,7 +1565,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { assert(ret); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); - /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, sa); + /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); @@ -2333,6 +2333,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { } taosArrayDestroy(pQueryHandle->pColumns); + taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle->statis); diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 4d44e82b1b..71838af150 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -23,14 +23,13 @@ extern "C" { #include "os.h" #define TARRAY_MIN_SIZE 8 -#define TARRAY_GET_ELEM(array, index) ((array)->pData + (index) * (array)->elemSize) +#define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) typedef struct SArray { size_t size; size_t capacity; size_t elemSize; - - void* pData; + void* pData; } SArray; /** diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 2982b8dc70..3da604d152 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -33,17 +33,20 @@ typedef struct SCacheStatis { int64_t refreshCount; } SCacheStatis; +struct STrashElem; + typedef struct SCacheDataNode { - uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t lifespan; // expiredTime expiredTime when this element should be remove from cache - uint64_t signature; - uint32_t size; // allocated size for current SCacheDataNode + uint64_t addedTime; // the added time when this element is added or updated into cache + uint64_t lifespan; // life duration when this element should be remove from cache + uint64_t expireTime; // expire time + uint64_t signature; + struct STrashElem *pTNodeHeader; // point to trash node head + uint16_t keySize: 15; // max key size: 32kb + bool inTrashCan: 1;// denote if it is in trash or not + uint32_t size; // allocated size for current SCacheDataNode T_REF_DECLARE() - uint16_t keySize: 15; // max key size: 32kb - bool inTrashCan: 1;// denote if it is in trash or not - int32_t extendFactor; // number of life span extend - char *key; - char data[]; + char *key; + char data[]; } SCacheDataNode; typedef struct STrashElem { diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 5bddc76266..04e500743c 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -18,6 +18,7 @@ #ifdef TAOS_RANDOM_FILE_FAIL +void taosSetRandomFileFailFactor(int factor); ssize_t taos_tread(int fd, void *buf, size_t count); ssize_t taos_twrite(int fd, void *buf, size_t count); off_t taos_lseek(int fd, off_t offset, int whence); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 688e49a40b..d3c622633d 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -116,11 +116,13 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo return; } - int32_t size = pNode->size; taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + pCacheObj->totalSize -= pNode->size; uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, size); + pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, + pNode->size); + if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -285,7 +287,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode->data, pNode->addedTime, (pNode->lifespan * pNode->extendFactor + pNode->addedTime), + pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, dataSize); } else { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); @@ -312,16 +314,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen int32_t ref = 0; if (ptNode != NULL) { ref = T_REF_INC(*ptNode); - - // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan - if (pCacheObj->extendLifespan) { - int64_t now = taosGetTimestampMs(); - - if ((now - (*ptNode)->addedTime) < (*ptNode)->lifespan * (*ptNode)->extendFactor) { - (*ptNode)->extendFactor += 1; - uDebug("key:%p extend life time to %"PRId64, key, (*ptNode)->lifespan * (*ptNode)->extendFactor + (*ptNode)->addedTime); - } - } } __cache_unlock(pCacheObj); @@ -347,8 +339,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); - (*ptNode)->extendFactor += 1; -// (*ptNode)->lifespan = expireTime; + (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan; } __cache_unlock(pCacheObj); @@ -380,17 +371,6 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { int32_t ref = T_REF_INC(ptNode); uDebug("cache:%s, data: %p acquired by data in cache, refcnt:%d", pCacheObj->name, ptNode->data, ref); - // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan - if (pCacheObj->extendLifespan) { - int64_t now = taosGetTimestampMs(); - - if ((now - ptNode->addedTime) < ptNode->lifespan * ptNode->extendFactor) { - ptNode->extendFactor += 1; - uDebug("cache:%s, %p extend life time to %" PRId64, pCacheObj->name, ptNode->data, - ptNode->lifespan * ptNode->extendFactor + ptNode->addedTime); - } - } - // the data if referenced by at least one object, so the reference count must be greater than the value of 2. assert(ref >= 2); return data; @@ -431,22 +411,58 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } *data = NULL; - int16_t ref = T_REF_DEC(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); - if (_remove && (!pNode->inTrashCan)) { - __cache_wr_lock(pCacheObj); + // note: extend lifespan before dec ref count + if (pCacheObj->extendLifespan) { + atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); + uDebug("cache:%s data:%p extend life time to %"PRId64 " before release", pCacheObj->name, pNode->data, pNode->expireTime); + } - if (T_REF_VAL_GET(pNode) == 0) { - // remove directly, if not referenced by other users - taosCacheReleaseNode(pCacheObj, pNode); - } else { - // pNode may be released immediately by other thread after the reference count of pNode is set to 0, - // So we need to lock it in the first place. - taosCacheMoveToTrash(pCacheObj, pNode); + bool inTrashCan = pNode->inTrashCan; + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1); + + // NOTE: once refcount is decrease, pNode may be free by other thread immediately. + int32_t ref = T_REF_DEC(pNode); + + if (inTrashCan) { + // Remove it if the ref count is 0. + // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when + // the node is in trashcan. + if (ref == 0) { + __cache_wr_lock(pCacheObj); + assert(pNode->pTNodeHeader->pData == pNode); + taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); + __cache_unlock(pCacheObj); } - __cache_unlock(pCacheObj); + } else { + assert(pNode->pTNodeHeader == NULL); + + if (_remove) { // not in trash can, but need to remove it + __cache_wr_lock(pCacheObj); + + /* + * If not referenced by other users. Otherwise move this node to trashcan wait for all users + * releasing this resources. + * + * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread + * that tries to do the same thing. + */ + if (ref == 0) { + if (T_REF_VAL_GET(pNode) == 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { + taosCacheMoveToTrash(pCacheObj, pNode); + } + } + + __cache_unlock(pCacheObj); +// } else { // extend its life time +// if (pCacheObj->extendLifespan) { +// atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); +// uDebug("cache:%s data:%p extend life time to %"PRId64 " after release", pCacheObj->name, pNode->data, pNode->expireTime); +// } + } } } @@ -486,7 +502,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { - size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1; + size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; SCacheDataNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { @@ -503,7 +519,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); pNewNode->lifespan = duration; - pNewNode->extendFactor = 1; + pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; pNewNode->signature = (uint64_t)pNewNode; pNewNode->size = (uint32_t)totalSize; @@ -512,6 +528,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->inTrashCan) { /* node is already in trash */ + assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } @@ -527,6 +544,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->pTrash = pElem; pNode->inTrashCan = true; + pNode->pTNodeHeader = pElem; pCacheObj->numOfElemsInTrash++; uDebug("key:%p, %p move to trash, numOfElem in trash:%d", pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); @@ -629,7 +647,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t __cache_wr_lock(pCacheObj); while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); - if ((pNode->addedTime + pNode->lifespan * pNode->extendFactor) <= time && T_REF_VAL_GET(pNode) <= 0) { + if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); continue; } diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index eb7a2d5a66..92eeaef126 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -26,40 +26,51 @@ #include "os.h" -#define RANDOM_FILE_FAIL_FACTOR 5 +#ifdef TAOS_RANDOM_FILE_FAIL + +static int random_file_fail_factor = 20; + +void taosSetRandomFileFailFactor(int factor) +{ + random_file_fail_factor = factor; +} +#endif ssize_t taos_tread(int fd, void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return tread(fd, buf, count); } ssize_t taos_twrite(int fd, void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return twrite(fd, buf, count); } off_t taos_lseek(int fd, off_t offset, int whence) { #ifdef TAOS_RANDOM_FILE_FAIL - if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) { - errno = EIO; - return -1; + if (random_file_fail_factor > 0) { + if (rand() % random_file_fail_factor == 0) { + errno = EIO; + return -1; + } } #endif - return lseek(fd, offset, whence); } diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index 3764c7966b..53d81bb542 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -123,7 +123,7 @@ void taosIdPoolMarkStatus(void *handle, int id) { int taosUpdateIdPool(id_pool_t *handle, int maxId) { id_pool_t *pIdPool = (id_pool_t*)handle; if (maxId <= pIdPool->maxId) { - return -1; + return 0; } bool *idList = calloc(maxId, sizeof(bool)); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 2a24a59742..ab2aa738c6 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -260,7 +260,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe } taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - uDebug("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname); + uTrace("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname); return 0; } diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 6f67c4a136..018c0d640b 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -317,29 +317,34 @@ int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) { static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) { *result = val; + int64_t factor = 1000L; + switch (unit) { case 's': - (*result) *= MILLISECOND_PER_SECOND; + (*result) *= MILLISECOND_PER_SECOND*factor; break; case 'm': - (*result) *= MILLISECOND_PER_MINUTE; + (*result) *= MILLISECOND_PER_MINUTE*factor; break; case 'h': - (*result) *= MILLISECOND_PER_HOUR; + (*result) *= MILLISECOND_PER_HOUR*factor; break; case 'd': - (*result) *= MILLISECOND_PER_DAY; + (*result) *= MILLISECOND_PER_DAY*factor; break; case 'w': - (*result) *= MILLISECOND_PER_WEEK; + (*result) *= MILLISECOND_PER_WEEK*factor; break; case 'n': - (*result) *= MILLISECOND_PER_MONTH; + (*result) *= MILLISECOND_PER_MONTH*factor; break; case 'y': - (*result) *= MILLISECOND_PER_YEAR; + (*result) *= MILLISECOND_PER_YEAR*factor; break; case 'a': + (*result) *= factor; + break; + case 'u': break; default: { ; @@ -348,7 +353,6 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu } /* get the value in microsecond */ - (*result) *= 1000L; return 0; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 18c9ebf2e1..695a55d476 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -176,16 +176,28 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { pVnode->status = TAOS_VN_STATUS_UPDATING; int32_t code = vnodeSaveCfg(pVnodeCfg); - if (code != TSDB_CODE_SUCCESS) return code; + if (code != TSDB_CODE_SUCCESS) { + pVnode->status = TAOS_VN_STATUS_READY; + return code; + } code = vnodeReadCfg(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; + if (code != TSDB_CODE_SUCCESS) { + pVnode->status = TAOS_VN_STATUS_READY; + return code; + } code = syncReconfig(pVnode->sync, &pVnode->syncCfg); - if (code != TSDB_CODE_SUCCESS) return code; + if (code != TSDB_CODE_SUCCESS) { + pVnode->status = TAOS_VN_STATUS_READY; + return code; + } code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg); - if (code != TSDB_CODE_SUCCESS) return code; + if (code != TSDB_CODE_SUCCESS) { + pVnode->status = TAOS_VN_STATUS_READY; + return code; + } pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is altered", pVnode->vgId); @@ -508,7 +520,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount); // release local resources only after cutting off outside connections - qSetQueryMgmtClosed(pVnode->qMgmt); + qQueryMgmtNotifyClosed(pVnode->qMgmt); vnodeRelease(pVnode); } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index f054ae3904..ff58e219b0 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -82,6 +82,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); } else { assert(*qhandle == (void*) killQueryMsg->qhandle); + qKillQuery(*qhandle); qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true); } @@ -93,7 +94,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (contLen != 0) { qinfo_t pQInfo = NULL; - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, NULL, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -108,9 +109,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo); if (handle == NULL) { // failed to register qhandle pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; - - qKillQuery(pQInfo); - qKillQuery(pQInfo); + qDestroyQueryInfo(pQInfo); // destroy it directly } else { assert(*handle == pQInfo); pRsp->qhandle = htobe64((uint64_t) pQInfo); @@ -120,10 +119,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - - // NOTE: there two refcount, needs to kill twice - // query has not been put into qhandle pool, kill it directly. - qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); return pRsp->code; } @@ -134,6 +129,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { dnodePutItemIntoReadQueue(pVnode, *handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } + vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo); } else { assert(pCont != NULL); @@ -183,6 +179,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pRetrieve->free == 1) { vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); + qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -209,6 +206,9 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { dnodePutItemIntoReadQueue(pVnode, *handle); pRet->qhandle = *handle; freeHandle = false; + } else { + qKillQuery(*handle); + freeHandle = true; } } } diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 872ac39f7e..9af72af471 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1,4 +1,4 @@ -#-----!/usr/bin/python3.7 +# -----!/usr/bin/python3.7 ################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. @@ -11,7 +11,31 @@ ################################################################### # -*- coding: utf-8 -*- -from __future__ import annotations # For type hinting before definition, ref: https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel +# For type hinting before definition, ref: +# https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel +from __future__ import annotations +import taos +import crash_gen +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.log import * +from queue import Queue, Empty +from typing import IO +from typing import Set +from typing import Dict +from typing import List +from requests.auth import HTTPBasicAuth +import textwrap +import datetime +import logging +import time +import random +import threading +import requests +import copy +import argparse +import getopt import sys import os @@ -22,71 +46,48 @@ import traceback if sys.version_info[0] < 3: raise Exception("Must be using Python 3") -import getopt -import argparse -import copy -import requests -import threading -import random -import time -import logging -import datetime -import textwrap -import requests -from requests.auth import HTTPBasicAuth - -from typing import List -from typing import Dict -from typing import Set -from typing import IO -from queue import Queue, Empty - -from util.log import * -from util.dnodes import * -from util.cases import * -from util.sql import * - -import crash_gen -import taos - -# Global variables, tried to keep a small number. +# Global variables, tried to keep a small number. # Command-line/Environment Configurations, will set a bit later # ConfigNameSpace = argparse.Namespace -gConfig = argparse.Namespace() # Dummy value, will be replaced later +gConfig = argparse.Namespace() # Dummy value, will be replaced later logger = None -def runThread(wt: WorkerThread): + +def runThread(wt: WorkerThread): wt.run() + class CrashGenError(Exception): def __init__(self, msg=None, errno=None): - self.msg = msg + self.msg = msg self.errno = errno - + def __str__(self): return self.msg + class WorkerThread: - def __init__(self, pool: ThreadPool, tid, - tc: ThreadCoordinator, - # te: TaskExecutor, - ): # note: main thread context! - # self._curStep = -1 + def __init__(self, pool: ThreadPool, tid, + tc: ThreadCoordinator, + # te: TaskExecutor, + ): # note: main thread context! + # self._curStep = -1 self._pool = pool - self._tid = tid - self._tc = tc # type: ThreadCoordinator + self._tid = tid + self._tc = tc # type: ThreadCoordinator # self.threadIdent = threading.get_ident() self._thread = threading.Thread(target=runThread, args=(self,)) self._stepGate = threading.Event() # Let us have a DB connection of our own - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore # print("connector_type = {}".format(gConfig.connector_type)) - self._dbConn = DbConn.createNative() if (gConfig.connector_type == 'native') else DbConn.createRest() + self._dbConn = DbConn.createNative() if ( + gConfig.connector_type == 'native') else DbConn.createRest() - self._dbInUse = False # if "use db" was executed already + self._dbInUse = False # if "use db" was executed already def logDebug(self, msg): logger.debug(" TRD[{}] {}".format(self._tid, msg)) @@ -98,137 +99,153 @@ class WorkerThread: return self._dbInUse def useDb(self): - if ( not self._dbInUse ): + if (not self._dbInUse): self.execSql("use db") self._dbInUse = True def getTaskExecutor(self): - return self._tc.getTaskExecutor() + return self._tc.getTaskExecutor() def start(self): self._thread.start() # AFTER the thread is recorded - def run(self): + def run(self): # initialization after thread starts, in the thread context # self.isSleeping = False logger.info("Starting to run thread: {}".format(self._tid)) - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore logger.debug("Worker thread openning database connection") self._dbConn.open() - self._doTaskLoop() - + self._doTaskLoop() + # clean up - if ( gConfig.per_thread_db_connection ): # type: ignore + if (gConfig.per_thread_db_connection): # type: ignore self._dbConn.close() - def _doTaskLoop(self) : + def _doTaskLoop(self): # while self._curStep < self._pool.maxSteps: # tc = ThreadCoordinator(None) - while True: - tc = self._tc # Thread Coordinator, the overall master + while True: + tc = self._tc # Thread Coordinator, the overall master tc.crossStepBarrier() # shared barrier first, INCLUDING the last one - logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] exited barrier...".format( + self._tid)) self.crossStepGate() # then per-thread gate, after being tapped - logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] exited step gate...".format( + self._tid)) if not self._tc.isRunning(): - logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") + logger.debug( + "[TRD] Thread Coordinator not running any more, worker thread now stopping...") break # Fetch a task from the Thread Coordinator - logger.debug("[TRD] Worker thread [{}] about to fetch task".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] about to fetch task".format( + self._tid)) task = tc.fetchTask() # Execute such a task - logger.debug("[TRD] Worker thread [{}] about to execute task: {}".format(self._tid, task.__class__.__name__)) + logger.debug( + "[TRD] Worker thread [{}] about to execute task: {}".format( + self._tid, task.__class__.__name__)) task.execute(self) tc.saveExecutedTask(task) - logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid)) + logger.debug( + "[TRD] Worker thread [{}] finished executing task".format( + self._tid)) - self._dbInUse = False # there may be changes between steps - - def verifyThreadSelf(self): # ensure we are called by this own thread - if ( threading.get_ident() != self._thread.ident ): + self._dbInUse = False # there may be changes between steps + + def verifyThreadSelf(self): # ensure we are called by this own thread + if (threading.get_ident() != self._thread.ident): raise RuntimeError("Unexpectly called from other threads") - def verifyThreadMain(self): # ensure we are called by the main thread - if ( threading.get_ident() != threading.main_thread().ident ): + def verifyThreadMain(self): # ensure we are called by the main thread + if (threading.get_ident() != threading.main_thread().ident): raise RuntimeError("Unexpectly called from other threads") def verifyThreadAlive(self): - if ( not self._thread.is_alive() ): + if (not self._thread.is_alive()): raise RuntimeError("Unexpected dead thread") # A gate is different from a barrier in that a thread needs to be "tapped" def crossStepGate(self): self.verifyThreadAlive() - self.verifyThreadSelf() # only allowed by ourselves - + self.verifyThreadSelf() # only allowed by ourselves + # Wait again at the "gate", waiting to be "tapped" - logger.debug("[TRD] Worker thread {} about to cross the step gate".format(self._tid)) - self._stepGate.wait() + logger.debug( + "[TRD] Worker thread {} about to cross the step gate".format( + self._tid)) + self._stepGate.wait() self._stepGate.clear() - + # self._curStep += 1 # off to a new step... - def tapStepGate(self): # give it a tap, release the thread waiting there + def tapStepGate(self): # give it a tap, release the thread waiting there self.verifyThreadAlive() - self.verifyThreadMain() # only allowed for main thread - - logger.debug("[TRD] Tapping worker thread {}".format(self._tid)) - self._stepGate.set() # wake up! - time.sleep(0) # let the released thread run a bit + self.verifyThreadMain() # only allowed for main thread - def execSql(self, sql): # TODO: expose DbConn directly - if ( gConfig.per_thread_db_connection ): - return self._dbConn.execute(sql) + logger.debug("[TRD] Tapping worker thread {}".format(self._tid)) + self._stepGate.set() # wake up! + time.sleep(0) # let the released thread run a bit + + def execSql(self, sql): # TODO: expose DbConn directly + if (gConfig.per_thread_db_connection): + return self._dbConn.execute(sql) else: return self._tc.getDbManager().getDbConn().execute(sql) - def querySql(self, sql): # TODO: expose DbConn directly - if ( gConfig.per_thread_db_connection ): - return self._dbConn.query(sql) + def querySql(self, sql): # TODO: expose DbConn directly + if (gConfig.per_thread_db_connection): + return self._dbConn.query(sql) else: return self._tc.getDbManager().getDbConn().query(sql) def getQueryResult(self): - if ( gConfig.per_thread_db_connection ): - return self._dbConn.getQueryResult() + if (gConfig.per_thread_db_connection): + return self._dbConn.getQueryResult() else: return self._tc.getDbManager().getDbConn().getQueryResult() def getDbConn(self): - if ( gConfig.per_thread_db_connection ): - return self._dbConn + if (gConfig.per_thread_db_connection): + return self._dbConn else: return self._tc.getDbManager().getDbConn() # def querySql(self, sql): # not "execute", since we are out side the DB context # if ( gConfig.per_thread_db_connection ): - # return self._dbConn.query(sql) + # return self._dbConn.query(sql) # else: # return self._tc.getDbState().getDbConn().query(sql) # The coordinator of all worker threads, mostly running in main thread + + class ThreadCoordinator: def __init__(self, pool: ThreadPool, dbManager): - self._curStep = -1 # first step is 0 + self._curStep = -1 # first step is 0 self._pool = pool # self._wd = wd - self._te = None # prepare for every new step + self._te = None # prepare for every new step self._dbManager = dbManager - self._executedTasks: List[Task] = [] # in a given step - self._lock = threading.RLock() # sync access for a few things + self._executedTasks: List[Task] = [] # in a given step + self._lock = threading.RLock() # sync access for a few things - self._stepBarrier = threading.Barrier(self._pool.numThreads + 1) # one barrier for all threads + self._stepBarrier = threading.Barrier( + self._pool.numThreads + 1) # one barrier for all threads self._execStats = ExecutionStats() self._runStatus = MainExec.STATUS_RUNNING def getTaskExecutor(self): return self._te - def getDbManager(self) -> DbManager : + def getDbManager(self) -> DbManager: return self._dbManager def crossStepBarrier(self): @@ -238,89 +255,103 @@ class ThreadCoordinator: self._runStatus = MainExec.STATUS_STOPPING self._execStats.registerFailure("User Interruption") - def run(self): + def run(self): self._pool.createAndStartThreads(self) # Coordinate all threads step by step - self._curStep = -1 # not started yet - maxSteps = gConfig.max_steps # type: ignore - self._execStats.startExec() # start the stop watch + self._curStep = -1 # not started yet + maxSteps = gConfig.max_steps # type: ignore + self._execStats.startExec() # start the stop watch transitionFailed = False hasAbortedTask = False - while(self._curStep < maxSteps-1 and - (not transitionFailed) and - (self._runStatus==MainExec.STATUS_RUNNING) and - (not hasAbortedTask)): # maxStep==10, last curStep should be 9 + while(self._curStep < maxSteps - 1 and + (not transitionFailed) and + (self._runStatus == MainExec.STATUS_RUNNING) and + (not hasAbortedTask)): # maxStep==10, last curStep should be 9 - if not gConfig.debug: - print(".", end="", flush=True) # print this only if we are not in debug mode + if not gConfig.debug: + # print this only if we are not in debug mode + print(".", end="", flush=True) logger.debug("[TRD] Main thread going to sleep") # Now main thread (that's us) is ready to enter a step - self.crossStepBarrier() # let other threads go past the pool barrier, but wait at the thread gate - self._stepBarrier.reset() # Other worker threads should now be at the "gate" + # let other threads go past the pool barrier, but wait at the + # thread gate + self.crossStepBarrier() + self._stepBarrier.reset() # Other worker threads should now be at the "gate" # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate" - # We use this period to do house keeping work, when all worker threads are QUIET. + # We use this period to do house keeping work, when all worker + # threads are QUIET. hasAbortedTask = False - for task in self._executedTasks : - if task.isAborted() : + for task in self._executedTasks: + if task.isAborted(): print("Task aborted: {}".format(task)) hasAbortedTask = True break - if hasAbortedTask : # do transition only if tasks are error free + if hasAbortedTask: # do transition only if tasks are error free self._execStats.registerFailure("Aborted Task Encountered") - else: + else: try: sm = self._dbManager.getStateMachine() logger.debug("[STT] starting transitions") - sm.transition(self._executedTasks) # at end of step, transiton the DB state + # at end of step, transiton the DB state + sm.transition(self._executedTasks) logger.debug("[STT] transition ended") - # Due to limitation (or maybe not) of the Python library, we cannot share connections across threads - if sm.hasDatabase() : + # Due to limitation (or maybe not) of the Python library, + # we cannot share connections across threads + if sm.hasDatabase(): for t in self._pool.threadList: logger.debug("[DB] use db for all worker threads") t.useDb() - # t.execSql("use db") # main thread executing "use db" on behalf of every worker thread + # t.execSql("use db") # main thread executing "use + # db" on behalf of every worker thread except taos.error.ProgrammingError as err: - if ( err.msg == 'network unavailable' ): # broken DB connection + if (err.msg == 'network unavailable'): # broken DB connection logger.info("DB connection broken, execution failed") traceback.print_stack() transitionFailed = True - self._te = None # Not running any more + self._te = None # Not running any more self._execStats.registerFailure("Broken DB Connection") - # continue # don't do that, need to tap all threads at end, and maybe signal them to stop + # continue # don't do that, need to tap all threads at + # end, and maybe signal them to stop else: - raise + raise # finally: # pass - - self.resetExecutedTasks() # clear the tasks after we are done + + self.resetExecutedTasks() # clear the tasks after we are done # Get ready for next step logger.debug("<-- Step {} finished".format(self._curStep)) - self._curStep += 1 # we are about to get into next step. TODO: race condition here! - logger.debug("\r\n\n--> Step {} starts with main thread waking up".format(self._curStep)) # Now not all threads had time to go to sleep + self._curStep += 1 # we are about to get into next step. TODO: race condition here! + # Now not all threads had time to go to sleep + logger.debug( + "\r\n\n--> Step {} starts with main thread waking up".format(self._curStep)) # A new TE for the new step - if not transitionFailed: # only if not failed + if not transitionFailed: # only if not failed self._te = TaskExecutor(self._curStep) - logger.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format(self._curStep)) # Now not all threads had time to go to sleep - self.tapAllThreads() # Worker threads will wake up at this point, and each execute it's own task + logger.debug( + "[TRD] Main thread waking up at step {}, tapping worker threads".format( + self._curStep)) # Now not all threads had time to go to sleep + # Worker threads will wake up at this point, and each execute it's + # own task + self.tapAllThreads() logger.debug("Main thread ready to finish up...") - if not transitionFailed: # only in regular situations - self.crossStepBarrier() # Cross it one last time, after all threads finish + if not transitionFailed: # only in regular situations + self.crossStepBarrier() # Cross it one last time, after all threads finish self._stepBarrier.reset() logger.debug("Main thread in exclusive zone...") - self._te = None # No more executor, time to end + self._te = None # No more executor, time to end logger.debug("Main thread tapping all threads one last time...") - self.tapAllThreads() # Let the threads run one last time + self.tapAllThreads() # Let the threads run one last time logger.debug("Main thread joining all threads") - self._pool.joinAll() # Get all threads to finish + self._pool.joinAll() # Get all threads to finish logger.info("\nAll worker threads finished") self._execStats.endExec() @@ -333,24 +364,27 @@ class ThreadCoordinator: def getExecStats(self): return self._execStats - def tapAllThreads(self): # in a deterministic manner + def tapAllThreads(self): # in a deterministic manner wakeSeq = [] - for i in range(self._pool.numThreads): # generate a random sequence - if Dice.throw(2) == 1 : + for i in range(self._pool.numThreads): # generate a random sequence + if Dice.throw(2) == 1: wakeSeq.append(i) else: wakeSeq.insert(0, i) - logger.debug("[TRD] Main thread waking up worker threads: {}".format(str(wakeSeq))) + logger.debug( + "[TRD] Main thread waking up worker threads: {}".format( + str(wakeSeq))) # TODO: set dice seed to a deterministic value for i in wakeSeq: - self._pool.threadList[i].tapStepGate() # TODO: maybe a bit too deep?! - time.sleep(0) # yield + # TODO: maybe a bit too deep?! + self._pool.threadList[i].tapStepGate() + time.sleep(0) # yield def isRunning(self): - return self._te != None + return self._te is not None - def fetchTask(self) -> Task : - if ( not self.isRunning() ): # no task + def fetchTask(self) -> Task: + if (not self.isRunning()): # no task raise RuntimeError("Cannot fetch task when not running") # return self._wd.pickTask() # Alternatively, let's ask the DbState for the appropriate task @@ -361,31 +395,36 @@ class ThreadCoordinator: # logger.debug(" (dice:{}/{}) ".format(i, nTasks)) # # return copy.copy(tasks[i]) # Needs a fresh copy, to save execution results, etc. # return tasks[i].clone() # TODO: still necessary? - taskType = self.getDbManager().getStateMachine().pickTaskType() # pick a task type for current state - return taskType(self.getDbManager(), self._execStats) # create a task from it + # pick a task type for current state + taskType = self.getDbManager().getStateMachine().pickTaskType() + return taskType( + self.getDbManager(), + self._execStats) # create a task from it def resetExecutedTasks(self): - self._executedTasks = [] # should be under single thread + self._executedTasks = [] # should be under single thread def saveExecutedTask(self, task): with self._lock: self._executedTasks.append(task) # We define a class to run a number of threads in locking steps. + + class ThreadPool: def __init__(self, numThreads, maxSteps): self.numThreads = numThreads self.maxSteps = maxSteps # Internal class variables self.curStep = 0 - self.threadList = [] # type: List[WorkerThread] - + self.threadList = [] # type: List[WorkerThread] + # starting to run all the threads, in locking steps def createAndStartThreads(self, tc: ThreadCoordinator): - for tid in range(0, self.numThreads): # Create the threads - workerThread = WorkerThread(self, tid, tc) + for tid in range(0, self.numThreads): # Create the threads + workerThread = WorkerThread(self, tid, tc) self.threadList.append(workerThread) - workerThread.start() # start, but should block immediately before step 0 + workerThread.start() # start, but should block immediately before step 0 def joinAll(self): for workerThread in self.threadList: @@ -394,21 +433,24 @@ class ThreadPool: # A queue of continguous POSITIVE integers, used by DbManager to generate continuous numbers # for new table names + + class LinearQueue(): def __init__(self): self.firstIndex = 1 # 1st ever element self.lastIndex = 0 - self._lock = threading.RLock() # our functions may call each other - self.inUse = set() # the indexes that are in use right now + self._lock = threading.RLock() # our functions may call each other + self.inUse = set() # the indexes that are in use right now def toText(self): - return "[{}..{}], in use: {}".format(self.firstIndex, self.lastIndex, self.inUse) + return "[{}..{}], in use: {}".format( + self.firstIndex, self.lastIndex, self.inUse) # Push (add new element, largest) to the tail, and mark it in use - def push(self): + def push(self): with self._lock: - # if ( self.isEmpty() ): - # self.lastIndex = self.firstIndex + # if ( self.isEmpty() ): + # self.lastIndex = self.firstIndex # return self.firstIndex # Otherwise we have something self.lastIndex += 1 @@ -418,12 +460,12 @@ class LinearQueue(): def pop(self): with self._lock: - if ( self.isEmpty() ): - # raise RuntimeError("Cannot pop an empty queue") - return False # TODO: None? - + if (self.isEmpty()): + # raise RuntimeError("Cannot pop an empty queue") + return False # TODO: None? + index = self.firstIndex - if ( index in self.inUse ): + if (index in self.inUse): return False self.firstIndex += 1 @@ -441,33 +483,35 @@ class LinearQueue(): def allocate(self, i): with self._lock: # logger.debug("LQ allocating item {}".format(i)) - if ( i in self.inUse ): - raise RuntimeError("Cannot re-use same index in queue: {}".format(i)) + if (i in self.inUse): + raise RuntimeError( + "Cannot re-use same index in queue: {}".format(i)) self.inUse.add(i) def release(self, i): with self._lock: # logger.debug("LQ releasing item {}".format(i)) - self.inUse.remove(i) # KeyError possible, TODO: why? + self.inUse.remove(i) # KeyError possible, TODO: why? def size(self): return self.lastIndex + 1 - self.firstIndex def pickAndAllocate(self): - if ( self.isEmpty() ): + if (self.isEmpty()): return None with self._lock: - cnt = 0 # counting the interations + cnt = 0 # counting the interations while True: cnt += 1 - if ( cnt > self.size()*10 ): # 10x iteration already + if (cnt > self.size() * 10): # 10x iteration already # raise RuntimeError("Failed to allocate LinearQueue element") return None - ret = Dice.throwRange(self.firstIndex, self.lastIndex+1) - if ( not ret in self.inUse ): + ret = Dice.throwRange(self.firstIndex, self.lastIndex + 1) + if (ret not in self.inUse): self.allocate(ret) return ret + class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" @@ -480,7 +524,8 @@ class DbConn: elif connType == cls.TYPE_REST: return DbConnRest() else: - raise RuntimeError("Unexpected connection type: {}".format(connType)) + raise RuntimeError( + "Unexpected connection type: {}".format(connType)) @classmethod def createNative(cls): @@ -495,18 +540,21 @@ class DbConn: self._type = self.TYPE_INVALID def open(self): - if ( self.isOpen ): + if (self.isOpen): raise RuntimeError("Cannot re-open an existing DB connection") # below implemented by child classes self.openByType() - logger.debug("[DB] data connection opened, type = {}".format(self._type)) + logger.debug( + "[DB] data connection opened, type = {}".format( + self._type)) self.isOpen = True - def resetDb(self): # reset the whole database, etc. - if ( not self.isOpen ): - raise RuntimeError("Cannot reset database until connection is open") + def resetDb(self): # reset the whole database, etc. + if (not self.isOpen): + raise RuntimeError( + "Cannot reset database until connection is open") # self._tdSql.prepare() # Recreate database, etc. self.execute('drop database if exists db') @@ -515,83 +563,99 @@ class DbConn: # self._cursor.execute('use db') # tdSql.execute('show databases') - def queryScalar(self, sql) -> int : + def queryScalar(self, sql) -> int: return self._queryAny(sql) - def queryString(self, sql) -> str : + def queryString(self, sql) -> str: return self._queryAny(sql) - def _queryAny(self, sql) : # actual query result as an int - if ( not self.isOpen ): - raise RuntimeError("Cannot query database until connection is open") + def _queryAny(self, sql): # actual query result as an int + if (not self.isOpen): + raise RuntimeError( + "Cannot query database until connection is open") nRows = self.query(sql) - if nRows != 1 : - raise RuntimeError("Unexpected result for query: {}, rows = {}".format(sql, nRows)) + if nRows != 1: + raise RuntimeError( + "Unexpected result for query: {}, rows = {}".format( + sql, nRows)) if self.getResultRows() != 1 or self.getResultCols() != 1: - raise RuntimeError("Unexpected result set for query: {}".format(sql)) + raise RuntimeError( + "Unexpected result set for query: {}".format(sql)) return self.getQueryResult()[0][0] def execute(self, sql): raise RuntimeError("Unexpected execution, should be overriden") + def openByType(self): raise RuntimeError("Unexpected execution, should be overriden") + def getQueryResult(self): raise RuntimeError("Unexpected execution, should be overriden") + def getResultRows(self): raise RuntimeError("Unexpected execution, should be overriden") + def getResultCols(self): raise RuntimeError("Unexpected execution, should be overriden") # Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql + + class DbConnRest(DbConn): def __init__(self): super().__init__() self._type = self.TYPE_REST - self._url = "http://localhost:6020/rest/sql" # fixed for now + self._url = "http://localhost:6020/rest/sql" # fixed for now self._result = None - def openByType(self): # Open connection - pass # do nothing, always open - + def openByType(self): # Open connection + pass # do nothing, always open + def close(self): - if ( not self.isOpen ): - raise RuntimeError("Cannot clean up database until connection is open") + if (not self.isOpen): + raise RuntimeError( + "Cannot clean up database until connection is open") # Do nothing for REST logger.debug("[DB] REST Database connection closed") self.isOpen = False def _doSql(self, sql): - r = requests.post(self._url, - data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) + r = requests.post(self._url, + data=sql, + auth=HTTPBasicAuth('root', 'taosdata')) rj = r.json() # Sanity check for the "Json Result" - if (not 'status' in rj): + if ('status' not in rj): raise RuntimeError("No status in REST response") - if rj['status'] == 'error': # clearly reported error - if (not 'code' in rj): # error without code - raise RuntimeError("REST error return without code") - errno = rj['code'] # May need to massage this in the future + if rj['status'] == 'error': # clearly reported error + if ('code' not in rj): # error without code + raise RuntimeError("REST error return without code") + errno = rj['code'] # May need to massage this in the future # print("Raising programming error with REST return: {}".format(rj)) - raise taos.error.ProgrammingError(rj['desc'], errno) # todo: check existance of 'desc' + raise taos.error.ProgrammingError( + rj['desc'], errno) # todo: check existance of 'desc' - if rj['status'] != 'succ': # better be this - raise RuntimeError("Unexpected REST return status: {}".format(rj['status'])) + if rj['status'] != 'succ': # better be this + raise RuntimeError( + "Unexpected REST return status: {}".format( + rj['status'])) nRows = rj['rows'] if ('rows' in rj) else 0 - self._result = rj + self._result = rj return nRows - def execute(self, sql): - if ( not self.isOpen ): - raise RuntimeError("Cannot execute database commands until connection is open") + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError( + "Cannot execute database commands until connection is open") logger.debug("[SQL-REST] Executing SQL: {}".format(sql)) nRows = self._doSql(sql) - logger.debug("[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) return nRows - def query(self, sql) : # return rows affected + def query(self, sql): # return rows affected return self.execute(sql) def getQueryResult(self): @@ -605,8 +669,10 @@ class DbConnRest(DbConn): def getResultCols(self): print(self._result) raise RuntimeError("TBD") - + # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative + + class MyTDSql: def __init__(self): self.queryRows = 0 @@ -635,7 +701,7 @@ class MyTDSql: # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) raise return self.queryRows - + def execute(self, sql): self.sql = sql try: @@ -647,47 +713,73 @@ class MyTDSql: raise return self.affectedRows + class DbConnNative(DbConn): def __init__(self): super().__init__() self._type = self.TYPE_REST - self._conn = None + self._conn = None self._cursor = None - - def openByType(self): # Open connection - cfgPath = "../../build/test/cfg" - self._conn = taos.connect(host="127.0.0.1", config=cfgPath) # TODO: make configurable + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("communit")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def openByType(self): # Open connection + cfgPath = self.getBuildPath() + "/test/cfg" + self._conn = taos.connect( + host="127.0.0.1", + config=cfgPath) # TODO: make configurable self._cursor = self._conn.cursor() # Get the connection/cursor ready self._cursor.execute('reset query cache') - # self._cursor.execute('use db') # do this at the beginning of every step + # self._cursor.execute('use db') # do this at the beginning of every + # step # Open connection self._tdSql = MyTDSql() self._tdSql.init(self._cursor) - + def close(self): - if ( not self.isOpen ): - raise RuntimeError("Cannot clean up database until connection is open") + if (not self.isOpen): + raise RuntimeError( + "Cannot clean up database until connection is open") self._tdSql.close() logger.debug("[DB] Database connection closed") self.isOpen = False - def execute(self, sql): - if ( not self.isOpen ): - raise RuntimeError("Cannot execute database commands until connection is open") + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError( + "Cannot execute database commands until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) nRows = self._tdSql.execute(sql) - logger.debug("[SQL] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL] Execution Result, nRows = {}, SQL = {}".format( + nRows, sql)) return nRows - def query(self, sql) : # return rows affected - if ( not self.isOpen ): - raise RuntimeError("Cannot query database until connection is open") + def query(self, sql): # return rows affected + if (not self.isOpen): + raise RuntimeError( + "Cannot query database until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) nRows = self._tdSql.query(sql) - logger.debug("[SQL] Query Result, nRows = {}, SQL = {}".format(nRows, sql)) + logger.debug( + "[SQL] Query Result, nRows = {}, SQL = {}".format( + nRows, sql)) return nRows # results are in: return self._tdSql.queryResult @@ -700,13 +792,13 @@ class DbConnNative(DbConn): def getResultCols(self): return self._tdSql.queryCols - + class AnyState: - STATE_INVALID = -1 - STATE_EMPTY = 0 # nothing there, no even a DB - STATE_DB_ONLY = 1 # we have a DB, but nothing else + STATE_INVALID = -1 + STATE_EMPTY = 0 # nothing there, no even a DB + STATE_DB_ONLY = 1 # we have a DB, but nothing else STATE_TABLE_ONLY = 2 # we have a table, but totally empty - STATE_HAS_DATA = 3 # we have some data in the table + STATE_HAS_DATA = 3 # we have some data in the table _stateNames = ["Invalid", "Empty", "DB_Only", "Table_Only", "Has_Data"] STATE_VAL_IDX = 0 @@ -721,7 +813,8 @@ class AnyState: self._info = self.getInfo() def __str__(self): - return self._stateNames[self._info[self.STATE_VAL_IDX] + 1] # -1 hack to accomodate the STATE_INVALID case + # -1 hack to accomodate the STATE_INVALID case + return self._stateNames[self._info[self.STATE_VAL_IDX] + 1] def getInfo(self): raise RuntimeError("Must be overriden by child classes") @@ -732,7 +825,9 @@ class AnyState: elif isinstance(other, AnyState): return self.getValIndex() == other.getValIndex() else: - raise RuntimeError("Unexpected comparison, type = {}".format(type(other))) + raise RuntimeError( + "Unexpected comparison, type = {}".format( + type(other))) def verifyTasksToState(self, tasks, newState): raise RuntimeError("Must be overriden by child classes") @@ -742,55 +837,65 @@ class AnyState: def getValue(self): return self._info[self.STATE_VAL_IDX] + def canCreateDb(self): return self._info[self.CAN_CREATE_DB] + def canDropDb(self): return self._info[self.CAN_DROP_DB] + def canCreateFixedSuperTable(self): return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE] + def canDropFixedSuperTable(self): return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] + def canAddData(self): return self._info[self.CAN_ADD_DATA] + def canReadData(self): return self._info[self.CAN_READ_DATA] def assertAtMostOneSuccess(self, tasks, cls): sCnt = 0 - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue if task.isSuccess(): # task.logDebug("Task success found") sCnt += 1 - if ( sCnt >= 2 ): - raise RuntimeError("Unexpected more than 1 success with task: {}".format(cls)) + if (sCnt >= 2): + raise RuntimeError( + "Unexpected more than 1 success with task: {}".format(cls)) def assertIfExistThenSuccess(self, tasks, cls): sCnt = 0 exists = False - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue - exists = True # we have a valid instance + exists = True # we have a valid instance if task.isSuccess(): sCnt += 1 - if ( exists and sCnt <= 0 ): - raise RuntimeError("Unexpected zero success for task: {}".format(cls)) + if (exists and sCnt <= 0): + raise RuntimeError( + "Unexpected zero success for task: {}".format(cls)) def assertNoTask(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): - raise CrashGenError("This task: {}, is not expected to be present, given the success/failure of others".format(cls.__name__)) + raise CrashGenError( + "This task: {}, is not expected to be present, given the success/failure of others".format(cls.__name__)) def assertNoSuccess(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): if task.isSuccess(): - raise RuntimeError("Unexpected successful task: {}".format(cls)) + raise RuntimeError( + "Unexpected successful task: {}".format(cls)) def hasSuccess(self, tasks, cls): - for task in tasks : + for task in tasks: if not isinstance(task, cls): continue if task.isSuccess(): @@ -798,35 +903,40 @@ class AnyState: return False def hasTask(self, tasks, cls): - for task in tasks : + for task in tasks: if isinstance(task, cls): return True return False + class StateInvalid(AnyState): def getInfo(self): return [ self.STATE_INVALID, - False, False, # can create/drop Db - False, False, # can create/drop fixed table - False, False, # can insert/read data with fixed table + False, False, # can create/drop Db + False, False, # can create/drop fixed table + False, False, # can insert/read data with fixed table ] # def verifyTasksToState(self, tasks, newState): + class StateEmpty(AnyState): def getInfo(self): return [ self.STATE_EMPTY, - True, False, # can create/drop Db - False, False, # can create/drop fixed table - False, False, # can insert/read data with fixed table + True, False, # can create/drop Db + False, False, # can create/drop fixed table + False, False, # can insert/read data with fixed table ] - def verifyTasksToState(self, tasks, newState): - if ( self.hasSuccess(tasks, TaskCreateDb) ): # at EMPTY, if there's succes in creating DB - if ( not self.hasTask(tasks, TaskDropDb) ) : # and no drop_db tasks - self.assertAtMostOneSuccess(tasks, TaskCreateDb) # we must have at most one. TODO: compare numbers + def verifyTasksToState(self, tasks, newState): + if (self.hasSuccess(tasks, TaskCreateDb) + ): # at EMPTY, if there's succes in creating DB + if (not self.hasTask(tasks, TaskDropDb)): # and no drop_db tasks + # we must have at most one. TODO: compare numbers + self.assertAtMostOneSuccess(tasks, TaskCreateDb) + class StateDbOnly(AnyState): def getInfo(self): @@ -838,32 +948,34 @@ class StateDbOnly(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( not self.hasTask(tasks, TaskCreateDb) ): - self.assertAtMostOneSuccess(tasks, TaskDropDb) # only if we don't create any more + if (not self.hasTask(tasks, TaskCreateDb)): + # only if we don't create any more + self.assertAtMostOneSuccess(tasks, TaskDropDb) self.assertIfExistThenSuccess(tasks, TaskDropDb) # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not true in massively parrallel cases # Nothing to be said about adding data task # if ( self.hasSuccess(tasks, DropDbTask) ): # dropped the DB - # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess - # self.assertAtMostOneSuccess(tasks, DropDbTask) - # self._state = self.STATE_EMPTY + # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess + # self.assertAtMostOneSuccess(tasks, DropDbTask) + # self._state = self.STATE_EMPTY # if ( self.hasSuccess(tasks, TaskCreateSuperTable) ): # did not drop db, create table success # # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table - # if ( not self.hasTask(tasks, TaskDropSuperTable) ): + # if ( not self.hasTask(tasks, TaskDropSuperTable) ): # self.assertAtMostOneSuccess(tasks, TaskCreateSuperTable) # at most 1 attempt is successful, if we don't drop anything - # self.assertNoTask(tasks, DropDbTask) # should have have tried - # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet - # # can't say there's add-data attempts, since they may all fail - # self._state = self.STATE_TABLE_ONLY - # else: - # self._state = self.STATE_HAS_DATA + # self.assertNoTask(tasks, DropDbTask) # should have have tried + # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet + # # can't say there's add-data attempts, since they may all fail + # self._state = self.STATE_TABLE_ONLY + # else: + # self._state = self.STATE_HAS_DATA # What about AddFixedData? # elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # self._state = self.STATE_HAS_DATA # else: # no success in dropping db tasks, no success in create fixed table? read data should also fail - # # raise RuntimeError("Unexpected no-success scenario") # We might just landed all failure tasks, + # # raise RuntimeError("Unexpected no-success scenario") # We might just landed all failure tasks, # self._state = self.STATE_DB_ONLY # no change + class StateSuperTableOnly(AnyState): def getInfo(self): return [ @@ -874,9 +986,11 @@ class StateSuperTableOnly(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( self.hasSuccess(tasks, TaskDropSuperTable) ): # we are able to drop the table + if (self.hasSuccess(tasks, TaskDropSuperTable) + ): # we are able to drop the table #self.assertAtMostOneSuccess(tasks, TaskDropSuperTable) - self.hasSuccess(tasks, TaskCreateSuperTable) # we must have had recreted it + # we must have had recreted it + self.hasSuccess(tasks, TaskCreateSuperTable) # self._state = self.STATE_DB_ONLY # elif ( self.hasSuccess(tasks, AddFixedDataTask) ): # no success dropping the table, but added data @@ -890,6 +1004,7 @@ class StateSuperTableOnly(AnyState): # raise RuntimeError("Unexpected no-success scenarios") # TODO: need to revamp!! + class StateHasData(AnyState): def getInfo(self): return [ @@ -900,13 +1015,15 @@ class StateHasData(AnyState): ] def verifyTasksToState(self, tasks, newState): - if ( newState.equals(AnyState.STATE_EMPTY) ): + if (newState.equals(AnyState.STATE_EMPTY)): self.hasSuccess(tasks, TaskDropDb) - if ( not self.hasTask(tasks, TaskCreateDb) ) : - self.assertAtMostOneSuccess(tasks, TaskDropDb) # TODO: dicy - elif ( newState.equals(AnyState.STATE_DB_ONLY) ): # in DB only - if ( not self.hasTask(tasks, TaskCreateDb)): # without a create_db task - self.assertNoTask(tasks, TaskDropDb) # we must have drop_db task + if (not self.hasTask(tasks, TaskCreateDb)): + self.assertAtMostOneSuccess(tasks, TaskDropDb) # TODO: dicy + elif (newState.equals(AnyState.STATE_DB_ONLY)): # in DB only + if (not self.hasTask(tasks, TaskCreateDb) + ): # without a create_db task + # we must have drop_db task + self.assertNoTask(tasks, TaskDropDb) self.hasSuccess(tasks, TaskDropSuperTable) # self.assertAtMostOneSuccess(tasks, DropFixedSuperTableTask) # TODO: dicy # elif ( newState.equals(AnyState.STATE_TABLE_ONLY) ): # data deleted @@ -914,19 +1031,26 @@ class StateHasData(AnyState): # self.assertNoTask(tasks, TaskDropSuperTable) # self.assertNoTask(tasks, TaskAddData) # self.hasSuccess(tasks, DeleteDataTasks) - else: # should be STATE_HAS_DATA - if (not self.hasTask(tasks, TaskCreateDb) ): # only if we didn't create one - self.assertNoTask(tasks, TaskDropDb) # we shouldn't have dropped it - if (not self.hasTask(tasks, TaskCreateSuperTable)) : # if we didn't create the table - self.assertNoTask(tasks, TaskDropSuperTable) # we should not have a task that drops it + else: # should be STATE_HAS_DATA + if (not self.hasTask(tasks, TaskCreateDb) + ): # only if we didn't create one + # we shouldn't have dropped it + self.assertNoTask(tasks, TaskDropDb) + if (not self.hasTask(tasks, TaskCreateSuperTable) + ): # if we didn't create the table + # we should not have a task that drops it + self.assertNoTask(tasks, TaskDropSuperTable) # self.assertIfExistThenSuccess(tasks, ReadFixedDataTask) + class StateMechine: def __init__(self, dbConn): self._dbConn = dbConn - self._curState = self._findCurrentState() # starting state - self._stateWeights = [1,3,5,15] # transitition target probabilities, indexed with value of STATE_EMPTY, STATE_DB_ONLY, etc. - + self._curState = self._findCurrentState() # starting state + # transitition target probabilities, indexed with value of STATE_EMPTY, + # STATE_DB_ONLY, etc. + self._stateWeights = [1, 3, 5, 15] + def getCurrentState(self): return self._curState @@ -934,142 +1058,178 @@ class StateMechine: return self._curState.canDropDb() # ha, can drop DB means it has one # May be slow, use cautionsly... - def getTaskTypes(self): # those that can run (directly/indirectly) from the current state + def getTaskTypes(self): # those that can run (directly/indirectly) from the current state def typesToStrings(types): ss = [] for t in types: ss.append(t.__name__) return ss - allTaskClasses = StateTransitionTask.__subclasses__() # all state transition tasks + allTaskClasses = StateTransitionTask.__subclasses__() # all state transition tasks firstTaskTypes = [] for tc in allTaskClasses: - # t = tc(self) # create task object + # t = tc(self) # create task object if tc.canBeginFrom(self._curState): firstTaskTypes.append(tc) - # now we have all the tasks that can begin directly from the current state, let's figure out the INDIRECT ones - taskTypes = firstTaskTypes.copy() # have to have these - for task1 in firstTaskTypes: # each task type gathered so far - endState = task1.getEndState() # figure the end state - if endState == None: # does not change end state - continue # no use, do nothing - for tc in allTaskClasses: # what task can further begin from there? + # now we have all the tasks that can begin directly from the current + # state, let's figure out the INDIRECT ones + taskTypes = firstTaskTypes.copy() # have to have these + for task1 in firstTaskTypes: # each task type gathered so far + endState = task1.getEndState() # figure the end state + if endState is None: # does not change end state + continue # no use, do nothing + for tc in allTaskClasses: # what task can further begin from there? if tc.canBeginFrom(endState) and (tc not in firstTaskTypes): - taskTypes.append(tc) # gather it + taskTypes.append(tc) # gather it if len(taskTypes) <= 0: - raise RuntimeError("No suitable task types found for state: {}".format(self._curState)) - logger.debug("[OPS] Tasks found for state {}: {}".format(self._curState, typesToStrings(taskTypes))) + raise RuntimeError( + "No suitable task types found for state: {}".format( + self._curState)) + logger.debug( + "[OPS] Tasks found for state {}: {}".format( + self._curState, + typesToStrings(taskTypes))) return taskTypes def _findCurrentState(self): dbc = self._dbConn - ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state - if dbc.query("show databases") == 0 : # no database?! + ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state + if dbc.query("show databases") == 0: # no database?! # logger.debug("Found EMPTY state") - logger.debug("[STT] empty database found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] empty database found, between {} and {}".format( + ts, time.time())) return StateEmpty() - dbc.execute("use db") # did not do this when openning connection, and this is NOT the worker thread, which does this on their own - if dbc.query("show tables") == 0 : # no tables + # did not do this when openning connection, and this is NOT the worker + # thread, which does this on their own + dbc.execute("use db") + if dbc.query("show tables") == 0: # no tables # logger.debug("Found DB ONLY state") - logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] DB_ONLY found, between {} and {}".format( + ts, time.time())) return StateDbOnly() - if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName()) ) == 0 : # no regular tables + if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName()) + ) == 0: # no regular tables # logger.debug("Found TABLE_ONLY state") - logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] SUPER_TABLE_ONLY found, between {} and {}".format( + ts, time.time())) return StateSuperTableOnly() - else: # has actual tables + else: # has actual tables # logger.debug("Found HAS_DATA state") - logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) + logger.debug( + "[STT] HAS_DATA found, between {} and {}".format( + ts, time.time())) return StateHasData() def transition(self, tasks): - if ( len(tasks) == 0 ): # before 1st step, or otherwise empty + if (len(tasks) == 0): # before 1st step, or otherwise empty logger.debug("[STT] Starting State: {}".format(self._curState)) - return # do nothing + return # do nothing - self._dbConn.execute("show dnodes") # this should show up in the server log, separating steps + # this should show up in the server log, separating steps + self._dbConn.execute("show dnodes") # Generic Checks, first based on the start state if self._curState.canCreateDb(): self._curState.assertIfExistThenSuccess(tasks, TaskCreateDb) - # self.assertAtMostOneSuccess(tasks, CreateDbTask) # not really, in case of multiple creation and drops + # self.assertAtMostOneSuccess(tasks, CreateDbTask) # not really, in + # case of multiple creation and drops if self._curState.canDropDb(): self._curState.assertIfExistThenSuccess(tasks, TaskDropDb) - # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in case of drop-create-drop + # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in + # case of drop-create-drop # if self._state.canCreateFixedTable(): # self.assertIfExistThenSuccess(tasks, CreateFixedTableTask) # Not true, DB may be dropped - # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not really, in case of create-drop-create + # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not + # really, in case of create-drop-create # if self._state.canDropFixedTable(): # self.assertIfExistThenSuccess(tasks, DropFixedTableTask) # Not True, the whole DB may be dropped - # self.assertAtMostOneSuccess(tasks, DropFixedTableTask) # not really in case of drop-create-drop + # self.assertAtMostOneSuccess(tasks, DropFixedTableTask) # not + # really in case of drop-create-drop # if self._state.canAddData(): - # self.assertIfExistThenSuccess(tasks, AddFixedDataTask) # not true actually + # self.assertIfExistThenSuccess(tasks, AddFixedDataTask) # not true + # actually # if self._state.canReadData(): # Nothing for sure newState = self._findCurrentState() logger.debug("[STT] New DB state determined: {}".format(newState)) - self._curState.verifyTasksToState(tasks, newState) # can old state move to new state through the tasks? + # can old state move to new state through the tasks? + self._curState.verifyTasksToState(tasks, newState) self._curState = newState def pickTaskType(self): - taskTypes = self.getTaskTypes() # all the task types we can choose from at curent state + # all the task types we can choose from at curent state + taskTypes = self.getTaskTypes() weights = [] for tt in taskTypes: endState = tt.getEndState() - if endState != None : - weights.append(self._stateWeights[endState.getValIndex()]) # TODO: change to a method + if endState is not None: + # TODO: change to a method + weights.append(self._stateWeights[endState.getValIndex()]) else: - weights.append(10) # read data task, default to 10: TODO: change to a constant + # read data task, default to 10: TODO: change to a constant + weights.append(10) i = self._weighted_choice_sub(weights) - # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) + # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] - def _weighted_choice_sub(self, weights): # ref: https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ - rnd = random.random() * sum(weights) # TODO: use our dice to ensure it being determinstic? + # ref: + # https://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python/ + def _weighted_choice_sub(self, weights): + # TODO: use our dice to ensure it being determinstic? + rnd = random.random() * sum(weights) for i, w in enumerate(weights): rnd -= w if rnd < 0: return i # Manager of the Database Data/Connection -class DbManager(): - def __init__(self, resetDb = True): + + +class DbManager(): + def __init__(self, resetDb=True): self.tableNumQueue = LinearQueue() - self._lastTick = self.setupLastTick() # datetime.datetime(2019, 1, 1) # initial date time tick - self._lastInt = 0 # next one is initial integer + # datetime.datetime(2019, 1, 1) # initial date time tick + self._lastTick = self.setupLastTick() + self._lastInt = 0 # next one is initial integer self._lock = threading.RLock() - + # self.openDbServerConnection() - self._dbConn = DbConn.createNative() if (gConfig.connector_type=='native') else DbConn.createRest() + self._dbConn = DbConn.createNative() if ( + gConfig.connector_type == 'native') else DbConn.createRest() try: - self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) - if ( err.msg == 'client disconnected' ): # cannot open DB connection - print("Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") + if (err.msg == 'client disconnected'): # cannot open DB connection + print( + "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") sys.exit(2) else: - raise - except: + raise + except BaseException: print("[=] Unexpected exception") - raise + raise - if resetDb : - self._dbConn.resetDb() # drop and recreate DB + if resetDb: + self._dbConn.resetDb() # drop and recreate DB + + # Do this after dbConn is in proper shape + self._stateMachine = StateMechine(self._dbConn) - self._stateMachine = StateMechine(self._dbConn) # Do this after dbConn is in proper shape - def getDbConn(self): return self._dbConn - def getStateMachine(self) -> StateMechine : + def getStateMachine(self) -> StateMechine: return self._stateMachine # def getState(self): @@ -1084,15 +1244,18 @@ class DbManager(): def setupLastTick(self): t1 = datetime.datetime(2020, 6, 1) t2 = datetime.datetime.now() - elSec = int(t2.timestamp() - t1.timestamp()) # maybe a very large number, takes 69 years to exceed Python int range - elSec2 = ( elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500 ) ) * 500 # a number representing seconds within 10 years + # maybe a very large number, takes 69 years to exceed Python int range + elSec = int(t2.timestamp() - t1.timestamp()) + elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ + 500 # a number representing seconds within 10 years # print("elSec = {}".format(elSec)) - t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years - t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above + t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + t4 = datetime.datetime.fromtimestamp( + t3.timestamp() + elSec2) # see explanation above logger.info("Setting up TICKS to start from: {}".format(t4)) return t4 - def pickAndAllocateTable(self): # pick any table, and "use" it + def pickAndAllocateTable(self): # pick any table, and "use" it return self.tableNumQueue.pickAndAllocate() def addTable(self): @@ -1104,15 +1267,16 @@ class DbManager(): def getFixedSuperTableName(cls): return "fs_table" - def releaseTable(self, i): # return the table back, so others can use it + def releaseTable(self, i): # return the table back, so others can use it self.tableNumQueue.release(i) def getNextTick(self): - with self._lock: # prevent duplicate tick - if Dice.throw(10) == 0 : # 1 in 10 chance + with self._lock: # prevent duplicate tick + if Dice.throw(10) == 0: # 1 in 10 chance return self._lastTick + datetime.timedelta(0, -100) - else: # regular - self._lastTick += datetime.timedelta(0, 1) # add one second to it + else: # regular + # add one second to it + self._lastTick += datetime.timedelta(0, 1) return self._lastTick def getNextInt(self): @@ -1121,29 +1285,31 @@ class DbManager(): return self._lastInt def getNextBinary(self): - return "Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_{}".format(self.getNextInt()) + return "Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_Beijing_Shanghai_Los_Angeles_New_York_San_Francisco_Chicago_{}".format( + self.getNextInt()) def getNextFloat(self): return 0.9 + self.getNextInt() - + def getTableNameToDelete(self): - tblNum = self.tableNumQueue.pop() # TODO: race condition! - if ( not tblNum ): # maybe false + tblNum = self.tableNumQueue.pop() # TODO: race condition! + if (not tblNum): # maybe false return False - + return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + self._dbConn.close() + class TaskExecutor(): class BoundedList: - def __init__(self, size = 10): + def __init__(self, size=10): self._size = size self._list = [] - def add(self, n: int) : - if not self._list: # empty + def add(self, n: int): + if not self._list: # empty self._list.append(n) return # now we should insert @@ -1151,22 +1317,22 @@ class TaskExecutor(): insPos = 0 for i in range(nItems): insPos = i - if n <= self._list[i] : # smaller than this item, time to insert - break # found the insertion point - insPos += 1 # insert to the right + if n <= self._list[i]: # smaller than this item, time to insert + break # found the insertion point + insPos += 1 # insert to the right - if insPos == 0 : # except for the 1st item, # TODO: elimiate first item as gating item - return # do nothing + if insPos == 0: # except for the 1st item, # TODO: elimiate first item as gating item + return # do nothing # print("Inserting at postion {}, value: {}".format(insPos, n)) - self._list.insert(insPos, n) # insert - + self._list.insert(insPos, n) # insert + newLen = len(self._list) - if newLen <= self._size : - return # do nothing - elif newLen == (self._size + 1) : - del self._list[0] # remove the first item - else : + if newLen <= self._size: + return # do nothing + elif newLen == (self._size + 1): + del self._list[0] # remove the first item + else: raise RuntimeError("Corrupt Bounded List") def __str__(self): @@ -1184,7 +1350,7 @@ class TaskExecutor(): def getCurStep(self): return self._curStep - def execute(self, task: Task, wt: WorkerThread): # execute a task on a thread + def execute(self, task: Task, wt: WorkerThread): # execute a task on a thread task.execute(wt) def recordDataMark(self, n: int): @@ -1197,143 +1363,164 @@ class TaskExecutor(): # def logDebug(self, msg): # logger.debug(" T[{}.x]: ".format(self._curStep) + msg) + class Task(): taskSn = 100 @classmethod def allocTaskNum(cls): - Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy + Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy # logger.debug("Allocating taskSN: {}".format(Task.taskSn)) return Task.taskSn - def __init__(self, dbManager: DbManager, execStats: ExecutionStats): + def __init__(self, dbManager: DbManager, execStats: ExecutionStats): self._dbManager = dbManager - self._workerThread = None + self._workerThread = None self._err = None self._aborted = False self._curStep = None - self._numRows = None # Number of rows affected + self._numRows = None # Number of rows affected - # Assign an incremental task serial number + # Assign an incremental task serial number self._taskNum = self.allocTaskNum() # logger.debug("Creating new task {}...".format(self._taskNum)) self._execStats = execStats - self._lastSql = "" # last SQL executed/attempted + self._lastSql = "" # last SQL executed/attempted def isSuccess(self): - return self._err == None + return self._err is None def isAborted(self): return self._aborted - def clone(self): # TODO: why do we need this again? + def clone(self): # TODO: why do we need this again? newTask = self.__class__(self._dbManager, self._execStats) return newTask def logDebug(self, msg): - self._workerThread.logDebug("Step[{}.{}] {}".format(self._curStep, self._taskNum, msg)) + self._workerThread.logDebug( + "Step[{}.{}] {}".format( + self._curStep, self._taskNum, msg)) def logInfo(self, msg): - self._workerThread.logInfo("Step[{}.{}] {}".format(self._curStep, self._taskNum, msg)) + self._workerThread.logInfo( + "Step[{}.{}] {}".format( + self._curStep, self._taskNum, msg)) def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - raise RuntimeError("To be implemeted by child classes, class name: {}".format(self.__class__.__name__)) + raise RuntimeError( + "To be implemeted by child classes, class name: {}".format( + self.__class__.__name__)) def execute(self, wt: WorkerThread): wt.verifyThreadSelf() - self._workerThread = wt # type: ignore + self._workerThread = wt # type: ignore te = wt.getTaskExecutor() self._curStep = te.getCurStep() - self.logDebug("[-] executing task {}...".format(self.__class__.__name__)) + self.logDebug( + "[-] executing task {}...".format(self.__class__.__name__)) self._err = None - self._execStats.beginTaskType(self.__class__.__name__) # mark beginning + self._execStats.beginTaskType( + self.__class__.__name__) # mark beginning try: - self._executeInternal(te, wt) # TODO: no return value? + self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme - if ( gConfig.continue_on_exception ): # user choose to continue - self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql)) + errno2 = err.errno if ( + err.errno > 0) else 0x80000000 + err.errno # correct error scheme + if (gConfig.continue_on_exception): # user choose to continue + self.logDebug( + "[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql)) self._err = err - elif ( errno2 in [ - 0x05, # TSDB_CODE_RPC_NOT_READY - 0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, - 0x381, 0x380, 0x383, - 0x386, # DB is being dropped?! - 0x503, - 0x510, # vnode not in ready state + elif (errno2 in [ + 0x05, # TSDB_CODE_RPC_NOT_READY + 0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D, + 0x381, 0x380, 0x383, + 0x386, # DB is being dropped?! + 0x503, + 0x510, # vnode not in ready state 0x600, - 1000 # REST catch-all error - ]) : # allowed errors - self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql)) + 1000 # REST catch-all error + ]): # allowed errors + self.logDebug( + "[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql)) print("_", end="", flush=True) - self._err = err + self._err = err else: - errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, self._lastSql) + errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format( + errno2, err, self._lastSql) self.logDebug(errMsg) - if gConfig.debug : + if gConfig.debug: # raise # so that we see full stack traceback.print_exc() - print("\n\n----------------------------\nProgram ABORTED Due to Unexpected TAOS Error: \n\n{}\n".format(errMsg) + + print( + "\n\n----------------------------\nProgram ABORTED Due to Unexpected TAOS Error: \n\n{}\n".format(errMsg) + "----------------------------\n") # sys.exit(-1) self._err = err self._aborted = True - except Exception as e : + except Exception as e: self.logInfo("Non-TAOS exception encountered") - self._err = e + self._err = e self._aborted = True traceback.print_exc() - except BaseException as e : + except BaseException as e: self.logInfo("Python base exception encountered") - self._err = e + self._err = e self._aborted = True traceback.print_exc() - except : - self.logDebug("[=] Unexpected exception, SQL: {}".format(self._lastSql)) + except BaseException: + self.logDebug( + "[=] Unexpected exception, SQL: {}".format( + self._lastSql)) raise self._execStats.endTaskType(self.__class__.__name__, self.isSuccess()) - - self.logDebug("[X] task execution completed, {}, status: {}".format(self.__class__.__name__, "Success" if self.isSuccess() else "Failure")) - self._execStats.incExecCount(self.__class__.__name__, self.isSuccess()) # TODO: merge with above. + + self.logDebug("[X] task execution completed, {}, status: {}".format( + self.__class__.__name__, "Success" if self.isSuccess() else "Failure")) + # TODO: merge with above. + self._execStats.incExecCount(self.__class__.__name__, self.isSuccess()) def execSql(self, sql): self._lastSql = sql return self._dbManager.execute(sql) - def execWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread + def execWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread self._lastSql = sql return wt.execSql(sql) - def queryWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread + def queryWtSql(self, wt: WorkerThread, sql): # execute an SQL on the worker thread self._lastSql = sql return wt.querySql(sql) - def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread + def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread return wt.getQueryResult() - class ExecutionStats: def __init__(self): - self._execTimes: Dict[str, [int, int]] = {} # total/success times for a task + # total/success times for a task + self._execTimes: Dict[str, [int, int]] = {} self._tasksInProgress = 0 self._lock = threading.Lock() self._firstTaskStartTime = None self._execStartTime = None - self._elapsedTime = 0.0 # total elapsed time - self._accRunTime = 0.0 # accumulated run time + self._elapsedTime = 0.0 # total elapsed time + self._accRunTime = 0.0 # accumulated run time self._failed = False self._failureReason = None def __str__(self): - return "[ExecStats: _failed={}, _failureReason={}".format(self._failed, self._failureReason) + return "[ExecStats: _failed={}, _failureReason={}".format( + self._failed, self._failureReason) def isFailed(self): - return self._failed == True + return self._failed def startExec(self): self._execStartTime = time.time() @@ -1341,24 +1528,24 @@ class ExecutionStats: def endExec(self): self._elapsedTime = time.time() - self._execStartTime - def incExecCount(self, klassName, isSuccess): # TODO: add a lock here + def incExecCount(self, klassName, isSuccess): # TODO: add a lock here if klassName not in self._execTimes: self._execTimes[klassName] = [0, 0] - t = self._execTimes[klassName] # tuple for the data - t[0] += 1 # index 0 has the "total" execution times + t = self._execTimes[klassName] # tuple for the data + t[0] += 1 # index 0 has the "total" execution times if isSuccess: - t[1] += 1 # index 1 has the "success" execution times + t[1] += 1 # index 1 has the "success" execution times def beginTaskType(self, klassName): with self._lock: - if self._tasksInProgress == 0 : # starting a new round - self._firstTaskStartTime = time.time() # I am now the first task + if self._tasksInProgress == 0: # starting a new round + self._firstTaskStartTime = time.time() # I am now the first task self._tasksInProgress += 1 def endTaskType(self, klassName, isSuccess): with self._lock: self._tasksInProgress -= 1 - if self._tasksInProgress == 0 : # all tasks have stopped + if self._tasksInProgress == 0: # all tasks have stopped self._accRunTime += (time.time() - self._firstTaskStartTime) self._firstTaskStartTime = None @@ -1367,23 +1554,36 @@ class ExecutionStats: self._failureReason = reason def printStats(self): - logger.info("----------------------------------------------------------------------") - logger.info("| Crash_Gen test {}, with the following stats:". - format("FAILED (reason: {})".format(self._failureReason) if self._failed else "SUCCEEDED")) + logger.info( + "----------------------------------------------------------------------") + logger.info( + "| Crash_Gen test {}, with the following stats:". format( + "FAILED (reason: {})".format( + self._failureReason) if self._failed else "SUCCEEDED")) logger.info("| Task Execution Times (success/total):") execTimesAny = 0 - for k, n in self._execTimes.items(): + for k, n in self._execTimes.items(): execTimesAny += n[0] - logger.info("| {0:<24}: {1}/{2}".format(k,n[1],n[0])) - - logger.info("| Total Tasks Executed (success or not): {} ".format(execTimesAny)) - logger.info("| Total Tasks In Progress at End: {}".format(self._tasksInProgress)) - logger.info("| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format(self._accRunTime)) - logger.info("| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime/execTimesAny)) - logger.info("| Total Elapsed Time (from wall clock): {:.3f} seconds".format(self._elapsedTime)) - logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) - logger.info("----------------------------------------------------------------------") - + logger.info("| {0:<24}: {1}/{2}".format(k, n[1], n[0])) + + logger.info( + "| Total Tasks Executed (success or not): {} ".format(execTimesAny)) + logger.info( + "| Total Tasks In Progress at End: {}".format( + self._tasksInProgress)) + logger.info( + "| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format( + self._accRunTime)) + logger.info( + "| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime / execTimesAny)) + logger.info( + "| Total Elapsed Time (from wall clock): {:.3f} seconds".format( + self._elapsedTime)) + logger.info( + "| Top numbers written: {}".format( + TaskExecutor.getBoundedList())) + logger.info( + "----------------------------------------------------------------------") class StateTransitionTask(Task): @@ -1393,12 +1593,12 @@ class StateTransitionTask(Task): SMALL_NUMBER_OF_RECORDS = 3 @classmethod - def getInfo(cls): # each sub class should supply their own information + def getInfo(cls): # each sub class should supply their own information raise RuntimeError("Overriding method expected") - _endState = None + _endState = None @classmethod - def getEndState(cls): # TODO: optimize by calling it fewer times + def getEndState(cls): # TODO: optimize by calling it fewer times raise RuntimeError("Overriding method expected") # @classmethod @@ -1420,18 +1620,20 @@ class StateTransitionTask(Task): def execute(self, wt: WorkerThread): super().execute(wt) - + + class TaskCreateDb(StateTransitionTask): @classmethod def getEndState(cls): - return StateDbOnly() + return StateDbOnly() @classmethod def canBeginFrom(cls, state: AnyState): return state.canCreateDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - self.execWtSql(wt, "create database db") + self.execWtSql(wt, "create database db") + class TaskDropDb(StateTransitionTask): @classmethod @@ -1446,6 +1648,7 @@ class TaskDropDb(StateTransitionTask): self.execWtSql(wt, "drop database db") logger.debug("[OPS] database dropped at {}".format(time.time())) + class TaskCreateSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1456,115 +1659,135 @@ class TaskCreateSuperTable(StateTransitionTask): return state.canCreateFixedSuperTable() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - if not wt.dbInUse(): # no DB yet, to the best of our knowledge + if not wt.dbInUse(): # no DB yet, to the best of our knowledge logger.debug("Skipping task, no DB yet") return - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() # wt.execSql("use db") # should always be in place - self.execWtSql(wt, "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) - # No need to create the regular tables, INSERT will do that automatically + self.execWtSql( + wt, + "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) + # No need to create the regular tables, INSERT will do that + # automatically class TaskReadData(StateTransitionTask): @classmethod def getEndState(cls): - return None # meaning doesn't affect state + return None # meaning doesn't affect state @classmethod def canBeginFrom(cls, state: AnyState): return state.canReadData() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - sTbName = self._dbManager.getFixedSuperTableName() - self.queryWtSql(wt, "select TBNAME from db.{}".format(sTbName)) # TODO: analyze result set later + sTbName = self._dbManager.getFixedSuperTableName() + self.queryWtSql(wt, "select TBNAME from db.{}".format( + sTbName)) # TODO: analyze result set later - if random.randrange(5) == 0 : # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations + if random.randrange( + 5) == 0: # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations wt.getDbConn().close() wt.getDbConn().open() else: - rTables = self.getQueryResult(wt) # wt.getDbConn().getQueryResult() + # wt.getDbConn().getQueryResult() + rTables = self.getQueryResult(wt) # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) - for rTbName in rTables : # regular tables + for rTbName in rTables: # regular tables self.execWtSql(wt, "select * from db.{}".format(rTbName[0])) # tdSql.query(" cars where tbname in ('carzero', 'carone')") + class TaskDropSuperTable(StateTransitionTask): @classmethod def getEndState(cls): - return StateDbOnly() + return StateDbOnly() @classmethod def canBeginFrom(cls, state: AnyState): return state.canDropFixedSuperTable() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - # 1/2 chance, we'll drop the regular tables one by one, in a randomized sequence - if Dice.throw(2) == 0 : - tblSeq = list(range(2 + (self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))) - random.shuffle(tblSeq) - tickOutput = False # if we have spitted out a "d" character for "drop regular table" + # 1/2 chance, we'll drop the regular tables one by one, in a randomized + # sequence + if Dice.throw(2) == 0: + tblSeq = list(range( + 2 + (self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))) + random.shuffle(tblSeq) + tickOutput = False # if we have spitted out a "d" character for "drop regular table" isSuccess = True - for i in tblSeq: - regTableName = self.getRegTableName(i); # "db.reg_table_{}".format(i) + for i in tblSeq: + regTableName = self.getRegTableName( + i) # "db.reg_table_{}".format(i) try: - self.execWtSql(wt, "drop table {}".format(regTableName)) # nRows always 0, like MySQL - except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correcting for strange error number scheme - if ( errno2 in [0x362]) : # mnode invalid table name + self.execWtSql(wt, "drop table {}".format( + regTableName)) # nRows always 0, like MySQL + except taos.error.ProgrammingError as err: + # correcting for strange error number scheme + errno2 = err.errno if ( + err.errno > 0) else 0x80000000 + err.errno + if (errno2 in [0x362]): # mnode invalid table name isSuccess = False - logger.debug("[DB] Acceptable error when dropping a table") - continue # try to delete next regular table + logger.debug( + "[DB] Acceptable error when dropping a table") + continue # try to delete next regular table if (not tickOutput): - tickOutput = True # Print only one time - if isSuccess : + tickOutput = True # Print only one time + if isSuccess: print("d", end="", flush=True) else: - print("f", end="", flush=True) + print("f", end="", flush=True) # Drop the super table itself - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() self.execWtSql(wt, "drop table db.{}".format(tblName)) + class TaskAlterTags(StateTransitionTask): @classmethod def getEndState(cls): - return None # meaning doesn't affect state + return None # meaning doesn't affect state @classmethod def canBeginFrom(cls, state: AnyState): - return state.canDropFixedSuperTable() # if we can drop it, we can alter tags + return state.canDropFixedSuperTable() # if we can drop it, we can alter tags def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - tblName = self._dbManager.getFixedSuperTableName() + tblName = self._dbManager.getFixedSuperTableName() dice = Dice.throw(4) - if dice == 0 : + if dice == 0: sql = "alter table db.{} add tag extraTag int".format(tblName) - elif dice == 1 : + elif dice == 1: sql = "alter table db.{} drop tag extraTag".format(tblName) - elif dice == 2 : + elif dice == 2: sql = "alter table db.{} drop tag newTag".format(tblName) - else: # dice == 3 - sql = "alter table db.{} change tag extraTag newTag".format(tblName) + else: # dice == 3 + sql = "alter table db.{} change tag extraTag newTag".format( + tblName) self.execWtSql(wt, sql) -class TaskAddData(StateTransitionTask): - activeTable : Set[int] = set() # Track which table is being actively worked on - # We use these two files to record operations to DB, useful for power-off tests +class TaskAddData(StateTransitionTask): + # Track which table is being actively worked on + activeTable: Set[int] = set() + + # We use these two files to record operations to DB, useful for power-off + # tests fAddLogReady = None fAddLogDone = None @classmethod def prepToRecordOps(cls): - if gConfig.record_ops : - if ( cls.fAddLogReady == None ): - logger.info("Recording in a file operations to be performed...") + if gConfig.record_ops: + if (cls.fAddLogReady is None): + logger.info( + "Recording in a file operations to be performed...") cls.fAddLogReady = open("add_log_ready.txt", "w") - if ( cls.fAddLogDone == None ): + if (cls.fAddLogDone is None): logger.info("Recording in a file operations completed...") cls.fAddLogDone = open("add_log_done.txt", "w") @@ -1575,78 +1798,92 @@ class TaskAddData(StateTransitionTask): @classmethod def canBeginFrom(cls, state: AnyState): return state.canAddData() - + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): ds = self._dbManager - # wt.execSql("use db") # TODO: seems to be an INSERT bug to require this - tblSeq = list(range(self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) - random.shuffle(tblSeq) - for i in tblSeq: - if ( i in self.activeTable ): # wow already active - # logger.info("Concurrent data insertion into table: {}".format(i)) - # print("ct({})".format(i), end="", flush=True) # Concurrent insertion into table + # wt.execSql("use db") # TODO: seems to be an INSERT bug to require + # this + tblSeq = list( + range( + self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) + random.shuffle(tblSeq) + for i in tblSeq: + if (i in self.activeTable): # wow already active + # logger.info("Concurrent data insertion into table: {}".format(i)) + # print("ct({})".format(i), end="", flush=True) # Concurrent + # insertion into table print("x", end="", flush=True) else: - self.activeTable.add(i) # marking it active - # No need to shuffle data sequence, unless later we decide to do non-increment insertion - regTableName = self.getRegTableName(i); # "db.reg_table_{}".format(i) - for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS) : # number of records per table - nextInt = ds.getNextInt() + self.activeTable.add(i) # marking it active + # No need to shuffle data sequence, unless later we decide to do + # non-increment insertion + regTableName = self.getRegTableName( + i) # "db.reg_table_{}".format(i) + for j in range( + self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table + nextInt = ds.getNextInt() if gConfig.record_ops: self.prepToRecordOps() - self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) + self.fAddLogReady.write( + "Ready to write {} to {}\n".format( + nextInt, regTableName)) self.fAddLogReady.flush() os.fsync(self.fAddLogReady) sql = "insert into {} using {} tags ('{}', {}) values ('{}', {});".format( - regTableName, - ds.getFixedSuperTableName(), + regTableName, + ds.getFixedSuperTableName(), ds.getNextBinary(), ds.getNextFloat(), ds.getNextTick(), nextInt) - self.execWtSql(wt, sql) - # Successfully wrote the data into the DB, let's record it somehow + self.execWtSql(wt, sql) + # Successfully wrote the data into the DB, let's record it + # somehow te.recordDataMark(nextInt) if gConfig.record_ops: - self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.write( + "Wrote {} to {}\n".format( + nextInt, regTableName)) self.fAddLogDone.flush() os.fsync(self.fAddLogDone) - self.activeTable.discard(i) # not raising an error, unlike remove + self.activeTable.discard(i) # not raising an error, unlike remove # Deterministic random number generator class Dice(): - seeded = False # static, uninitialized + seeded = False # static, uninitialized @classmethod - def seed(cls, s): # static + def seed(cls, s): # static if (cls.seeded): - raise RuntimeError("Cannot seed the random generator more than once") + raise RuntimeError( + "Cannot seed the random generator more than once") cls.verifyRNG() random.seed(s) cls.seeded = True # TODO: protect against multi-threading @classmethod - def verifyRNG(cls): # Verify that the RNG is determinstic + def verifyRNG(cls): # Verify that the RNG is determinstic random.seed(0) x1 = random.randrange(0, 1000) x2 = random.randrange(0, 1000) x3 = random.randrange(0, 1000) - if ( x1 != 864 or x2!=394 or x3!=776 ): + if (x1 != 864 or x2 != 394 or x3 != 776): raise RuntimeError("System RNG is not deterministic") @classmethod - def throw(cls, stop): # get 0 to stop-1 + def throw(cls, stop): # get 0 to stop-1 return cls.throwRange(0, stop) @classmethod - def throwRange(cls, start, stop): # up to stop-1 - if ( not cls.seeded ): + def throwRange(cls, start, stop): # up to stop-1 + if (not cls.seeded): raise RuntimeError("Cannot throw dice before seeding it") return random.randrange(start, stop) + class LoggingFilter(logging.Filter): def filter(self, record: logging.LogRecord): - if ( record.levelno >= logging.INFO ) : - return True # info or above always log + if (record.levelno >= logging.INFO): + return True # info or above always log # Commenting out below to adjust... @@ -1654,20 +1891,23 @@ class LoggingFilter(logging.Filter): # return False return True -class MyLoggingAdapter(logging.LoggerAdapter): + +class MyLoggingAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): return "[{}]{}".format(threading.get_ident() % 10000, msg), kwargs # return '[%s] %s' % (self.extra['connid'], msg), kwargs -class SvcManager: + +class SvcManager: def __init__(self): print("Starting TDengine Service Manager") signal.signal(signal.SIGTERM, self.sigIntHandler) - signal.signal(signal.SIGINT, self.sigIntHandler) - signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! - + signal.signal(signal.SIGINT, self.sigIntHandler) + signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! + self.inSigHandler = False - # self._status = MainExec.STATUS_RUNNING # set inside _startTaosService() + # self._status = MainExec.STATUS_RUNNING # set inside + # _startTaosService() self.svcMgrThread = None def _doMenu(self): @@ -1678,30 +1918,32 @@ class SvcManager: print("2: Terminate") print("3: Restart") # Remember to update the if range below - # print("Enter Choice: ", end="", flush=True) + # print("Enter Choice: ", end="", flush=True) while choice == "": choice = input("Enter Choice: ") if choice != "": - break # done with reading repeated input - if choice in ["1", "2", "3"]: - break # we are done with whole method + break # done with reading repeated input + if choice in ["1", "2", "3"]: + break # we are done with whole method print("Invalid choice, please try again.") - choice = "" # reset + choice = "" # reset return choice - def sigUsrHandler(self, signalNumber, frame) : + def sigUsrHandler(self, signalNumber, frame): print("Interrupting main thread execution upon SIGUSR1") - if self.inSigHandler : # already + if self.inSigHandler: # already print("Ignoring repeated SIG...") - return # do nothing if it's already not running + return # do nothing if it's already not running self.inSigHandler = True choice = self._doMenu() - if choice == "1" : - self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue? - elif choice == "2" : + if choice == "1": + # TODO: can the sub-process be blocked due to us not reading from + # queue? + self.sigHandlerResume() + elif choice == "2": self.stopTaosService() - elif choice == "3" : + elif choice == "3": self.stopTaosService() self.startTaosService() else: @@ -1711,48 +1953,52 @@ class SvcManager: def sigIntHandler(self, signalNumber, frame): print("Sig INT Handler starting...") - if self.inSigHandler : + if self.inSigHandler: print("Ignoring repeated SIG_INT...") return self.inSigHandler = True - - self.stopTaosService() - print("INT signal handler returning...") + + self.stopTaosService() + print("INT signal handler returning...") self.inSigHandler = False - def sigHandlerResume(self) : + def sigHandlerResume(self): print("Resuming TDengine service manager thread (main thread)...\n\n") - + def _checkServiceManagerThread(self): - if self.svcMgrThread: # valid svc mgr thread - if self.svcMgrThread.isStopped(): # done? - self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? - self.svcMgrThread = None # no more + if self.svcMgrThread: # valid svc mgr thread + if self.svcMgrThread.isStopped(): # done? + self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? + self.svcMgrThread = None # no more def _procIpcAll(self): - while self.svcMgrThread : # for as long as the svc mgr thread is still here - self.svcMgrThread.procIpcBatch() # regular processing, - time.sleep(0.5) # pause, before next round - self._checkServiceManagerThread() - print("Service Manager Thread (with subprocess) has ended, main thread now exiting...") + while self.svcMgrThread: # for as long as the svc mgr thread is still here + self.svcMgrThread.procIpcBatch() # regular processing, + time.sleep(0.5) # pause, before next round + self._checkServiceManagerThread() + print( + "Service Manager Thread (with subprocess) has ended, main thread now exiting...") - def startTaosService(self): + def startTaosService(self): if self.svcMgrThread: - raise RuntimeError("Cannot start TAOS service when one may already be running") - self.svcMgrThread = ServiceManagerThread() # create the object + raise RuntimeError( + "Cannot start TAOS service when one may already be running") + self.svcMgrThread = ServiceManagerThread() # create the object self.svcMgrThread.start() print("TAOS service started, printing out output...") - self.svcMgrThread.procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines + self.svcMgrThread.procIpcBatch( + trimToTarget=10, + forceOutput=True) # for printing 10 lines print("TAOS service started") - - def stopTaosService(self, outputLines = 20): + + def stopTaosService(self, outputLines=20): print("Terminating Service Manager Thread (SMT) execution...") if not self.svcMgrThread: raise RuntimeError("Unexpected empty svc mgr thread") self.svcMgrThread.stop() if self.svcMgrThread.isStopped(): - self.svcMgrThread.procIpcBatch(outputLines) # one last time - self.svcMgrThread = None + self.svcMgrThread.procIpcBatch(outputLines) # one last time + self.svcMgrThread = None print("----- End of TDengine Service Output -----\n") print("SMT execution terminated") else: @@ -1760,16 +2006,17 @@ class SvcManager: def run(self): self.startTaosService() - self._procIpcAll() # pump/process all the messages - if self.svcMgrThread: # if sig handler hasn't destroyed it by now - self.stopTaosService() # should have started already - + self._procIpcAll() # pump/process all the messages + if self.svcMgrThread: # if sig handler hasn't destroyed it by now + self.stopTaosService() # should have started already + + class ServiceManagerThread: MAX_QUEUE_SIZE = 10000 def __init__(self): self._tdeSubProcess = None - self._thread = None + self._thread = None self._status = None def getStatus(self): @@ -1787,98 +2034,107 @@ class ServiceManagerThread: # Start the thread (with sub process), and wait for the sub service # to become fully operational - def start(self): - if self._thread : + def start(self): + if self._thread: raise RuntimeError("Unexpected _thread") - if self._tdeSubProcess : + if self._tdeSubProcess: raise RuntimeError("TDengine sub process already created/running") self._status = MainExec.STATUS_STARTING - self._tdeSubProcess = TdeSubProcess() + self._tdeSubProcess = TdeSubProcess() self._tdeSubProcess.start() self._ipcQueue = Queue() self._thread = threading.Thread( - target=self.svcOutputReader, + target=self.svcOutputReader, args=(self._tdeSubProcess.getStdOut(), self._ipcQueue)) - self._thread.daemon = True # thread dies with the program + self._thread.daemon = True # thread dies with the program self._thread.start() # wait for service to start - for i in range(0, 10) : + for i in range(0, 10): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up print("_zz_", end="", flush=True) - if self._status == MainExec.STATUS_RUNNING : + if self._status == MainExec.STATUS_RUNNING: logger.info("[] TDengine service READY to process requests") - return # now we've started - raise RuntimeError("TDengine service did not start successfully") # TODO: handle this better? + return # now we've started + # TODO: handle this better? + raise RuntimeError("TDengine service did not start successfully") def stop(self): # can be called from both main thread or signal handler print("Terminating TDengine service running as the sub process...") if self.isStopped(): print("Service already stopped") - return + return if self.isStopping(): print("Service is already being stopped") return - # Linux will send Control-C generated SIGINT to the TDengine process already, ref: https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes - if not self._tdeSubProcess : + # Linux will send Control-C generated SIGINT to the TDengine process + # already, ref: + # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes + if not self._tdeSubProcess: raise RuntimeError("sub process object missing") self._status = MainExec.STATUS_STOPPING self._tdeSubProcess.stop() - if self._tdeSubProcess.isRunning(): # still running - print("FAILED to stop sub process, it is still running... pid = {}".format(self.subProcess.pid)) + if self._tdeSubProcess.isRunning(): # still running + print( + "FAILED to stop sub process, it is still running... pid = {}".format( + self.subProcess.pid)) else: - self._tdeSubProcess = None # not running any more - self.join() # stop the thread, change the status, etc. - + self._tdeSubProcess = None # not running any more + self.join() # stop the thread, change the status, etc. + def join(self): # TODO: sanity check if not self.isStopping(): - raise RuntimeError("Unexpected status when ending svc mgr thread: {}".format(self._status)) + raise RuntimeError( + "Unexpected status when ending svc mgr thread: {}".format( + self._status)) - if self._thread : + if self._thread: self._thread.join() - self._thread = None + self._thread = None self._status = MainExec.STATUS_STOPPED - else : + else: print("Joining empty thread, doing nothing") def _trimQueue(self, targetSize): if targetSize <= 0: - return # do nothing + return # do nothing q = self._ipcQueue - if (q.qsize() <= targetSize ) : # no need to trim + if (q.qsize() <= targetSize): # no need to trim return logger.debug("Triming IPC queue to target size: {}".format(targetSize)) itemsToTrim = q.qsize() - targetSize - for i in range(0, itemsToTrim) : + for i in range(0, itemsToTrim): try: q.get_nowait() except Empty: - break # break out of for loop, no more trimming - + break # break out of for loop, no more trimming + TD_READY_MSG = "TDengine is initialized successfully" - def procIpcBatch(self, trimToTarget = 0, forceOutput = False): - self._trimQueue(trimToTarget) # trim if necessary - # Process all the output generated by the underlying sub process, managed by IO thread + + def procIpcBatch(self, trimToTarget=0, forceOutput=False): + self._trimQueue(trimToTarget) # trim if necessary + # Process all the output generated by the underlying sub process, + # managed by IO thread print("<", end="", flush=True) - while True : - try: - line = self._ipcQueue.get_nowait() # getting output at fast speed + while True: + try: + line = self._ipcQueue.get_nowait() # getting output at fast speed self._printProgress("_o") except Empty: # time.sleep(2.3) # wait only if there's no output # no more output print(".>", end="", flush=True) - return # we are done with THIS BATCH - else: # got line, printing out + return # we are done with THIS BATCH + else: # got line, printing out if forceOutput: logger.info(line) else: @@ -1886,8 +2142,9 @@ class ServiceManagerThread: print(">", end="", flush=True) _ProgressBars = ["--", "//", "||", "\\\\"] - def _printProgress(self, msg): # TODO: assuming 2 chars - print(msg, end="", flush=True) + + def _printProgress(self, msg): # TODO: assuming 2 chars + print(msg, end="", flush=True) pBar = self._ProgressBars[Dice.throw(4)] print(pBar, end="", flush=True) print('\b\b\b\b', end="", flush=True) @@ -1895,29 +2152,33 @@ class ServiceManagerThread: def svcOutputReader(self, out: IO, queue): # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # print("This is the svcOutput Reader...") - # for line in out : + # for line in out : for line in iter(out.readline, b''): # print("Finished reading a line: {}".format(line)) # print("Adding item to queue...") line = line.decode("utf-8").rstrip() - queue.put(line) # This might block, and then causing "out" buffer to block + # This might block, and then causing "out" buffer to block + queue.put(line) self._printProgress("_i") - if self._status == MainExec.STATUS_STARTING : # we are starting, let's see if we have started - if line.find(self.TD_READY_MSG) != -1 : # found - self._status = MainExec.STATUS_RUNNING + if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started + if line.find(self.TD_READY_MSG) != -1: # found + self._status = MainExec.STATUS_RUNNING # Trim the queue if necessary: TODO: try this 1 out of 10 times - self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size + self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size - if self.isStopping() : # TODO: use thread status instead - print("_w", end="", flush=True) # WAITING for stopping sub process to finish its outptu + if self.isStopping(): # TODO: use thread status instead + # WAITING for stopping sub process to finish its outptu + print("_w", end="", flush=True) # queue.put(line) - print("\nNo more output from IO thread managing TDengine service") # meaning sub process must have died + # meaning sub process must have died + print("\nNo more output from IO thread managing TDengine service") out.close() -class TdeSubProcess: + +class TdeSubProcess: def __init__(self): self.subProcess = None @@ -1925,20 +2186,39 @@ class TdeSubProcess: return self.subProcess.stdout def isRunning(self): - return self.subProcess != None + return self.subProcess is not None + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("communit")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath def start(self): ON_POSIX = 'posix' in sys.builtin_module_names - svcCmd = ['../../build/build/bin/taosd', '-c', '../../build/test/cfg'] + + taosdPath = self.getBuildPath() + "/build/bin/taosd" + cfgPath = self.getBuildPath() + "/test/cfg" + + svcCmd = [taosdPath, '-c', cfgPath] # svcCmd = ['vmstat', '1'] - if self.subProcess : # already there + if self.subProcess: # already there raise RuntimeError("Corrupt process state") self.subProcess = subprocess.Popen( - svcCmd, - stdout=subprocess.PIPE, + svcCmd, + stdout=subprocess.PIPE, # bufsize=1, # not supported in binary mode - close_fds=ON_POSIX) # had text=True, which interferred with reading EOF + close_fds=ON_POSIX) # had text=True, which interferred with reading EOF def stop(self): if not self.subProcess: @@ -1946,12 +2226,15 @@ class TdeSubProcess: return retCode = self.subProcess.poll() - if retCode : # valid return code, process ended + if retCode: # valid return code, process ended self.subProcess = None - else: # process still alive, let's interrupt it - print("Sub process is running, sending SIG_INT and waiting for it to terminate...") - self.subProcess.send_signal(signal.SIGINT) # sub process should end, then IPC queue should end, causing IO thread to end - try : + else: # process still alive, let's interrupt it + print( + "Sub process is running, sending SIG_INT and waiting for it to terminate...") + # sub process should end, then IPC queue should end, causing IO + # thread to end + self.subProcess.send_signal(signal.SIGINT) + try: self.subProcess.wait(10) except subprocess.TimeoutExpired as err: print("Time out waiting for TDengine service process to exit") @@ -1959,6 +2242,7 @@ class TdeSubProcess: print("TDengine service process terminated successfully from SIG_INT") self.subProcess = None + class ClientManager: def __init__(self): print("Starting service manager") @@ -1969,76 +2253,78 @@ class ClientManager: self.tc = None def sigIntHandler(self, signalNumber, frame): - if self._status != MainExec.STATUS_RUNNING : + if self._status != MainExec.STATUS_RUNNING: print("Ignoring repeated SIGINT...") - return # do nothing if it's already not running - self._status = MainExec.STATUS_STOPPING # immediately set our status + return # do nothing if it's already not running + self._status = MainExec.STATUS_STOPPING # immediately set our status print("Terminating program...") self.tc.requestToStop() - def _printLastNumbers(self): # to verify data durability + def _printLastNumbers(self): # to verify data durability dbManager = DbManager(resetDb=False) dbc = dbManager.getDbConn() - if dbc.query("show databases") == 0 : # no databae + if dbc.query("show databases") == 0: # no databae return - if dbc.query("show tables") == 0 : # no tables + if dbc.query("show tables") == 0: # no tables return dbc.execute("use db") - sTbName = dbManager.getFixedSuperTableName() + sTbName = dbManager.getFixedSuperTableName() # get all regular tables - dbc.query("select TBNAME from db.{}".format(sTbName)) # TODO: analyze result set later + # TODO: analyze result set later + dbc.query("select TBNAME from db.{}".format(sTbName)) rTables = dbc.getQueryResult() bList = TaskExecutor.BoundedList() - for rTbName in rTables : # regular tables + for rTbName in rTables: # regular tables dbc.query("select speed from db.{}".format(rTbName[0])) numbers = dbc.getQueryResult() - for row in numbers : + for row in numbers: # print("<{}>".format(n), end="", flush=True) bList.add(row[0]) print("Top numbers in DB right now: {}".format(bList)) print("TDengine client execution is about to start in 2 seconds...") time.sleep(2.0) - dbManager = None # release? + dbManager = None # release? def prepare(self): self._printLastNumbers() def run(self): - if gConfig.auto_start_service : + if gConfig.auto_start_service: svcMgr = SvcManager() svcMgr.startTaosService() self._printLastNumbers() - dbManager = DbManager() # Regular function + dbManager = DbManager() # Regular function thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) - + self.tc.run() # print("exec stats: {}".format(self.tc.getExecStats())) - # print("TC failed = {}".format(self.tc.isFailed())) - if gConfig.auto_start_service : + # print("TC failed = {}".format(self.tc.isFailed())) + if gConfig.auto_start_service: svcMgr.stopTaosService() # Print exec status, etc., AFTER showing messages from the server self.conclude() # print("TC failed (2) = {}".format(self.tc.isFailed())) - return 1 if self.tc.isFailed() else 0 # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + return 1 if self.tc.isFailed() else 0 def conclude(self): self.tc.printStats() - self.tc.getDbManager().cleanUp() + self.tc.getDbManager().cleanUp() class MainExec: STATUS_STARTING = 1 - STATUS_RUNNING = 2 + STATUS_RUNNING = 2 STATUS_STOPPING = 3 - STATUS_STOPPED = 4 + STATUS_STOPPED = 4 @classmethod def runClient(cls): @@ -2051,13 +2337,13 @@ class MainExec: svcManager.run() @classmethod - def runTemp(cls): # for debugging purposes + def runTemp(cls): # for debugging purposes # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix # dbc = dbState.getDbConn() - # sTbName = dbState.getFixedSuperTableName() + # sTbName = dbState.getFixedSuperTableName() # dbc.execute("create database if not exists db") # if not dbState.getState().equals(StateEmpty()): - # dbc.execute("use db") + # dbc.execute("use db") # rTables = None # try: # the super table may not exist @@ -2069,7 +2355,7 @@ class MainExec: # logger.info("Result: {}".format(rTables)) # except taos.error.ProgrammingError as err: # logger.info("Initial Super table OPS error: {}".format(err)) - + # # sys.exit() # if ( not rTables == None): # # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) @@ -2078,24 +2364,26 @@ class MainExec: # ds = dbState # logger.info("Inserting into table: {}".format(rTbName[0])) # sql = "insert into db.{} values ('{}', {});".format( - # rTbName[0], + # rTbName[0], # ds.getNextTick(), ds.getNextInt()) # dbc.execute(sql) - # for rTbName in rTables : # regular tables + # for rTbName in rTables : # regular tables # dbc.query("select * from db.{}".format(rTbName[0])) # TODO: check success failure - # logger.info("Initial READING operation is successful") + # logger.info("Initial READING operation is successful") # except taos.error.ProgrammingError as err: - # logger.info("Initial WRITE/READ error: {}".format(err)) - + # logger.info("Initial WRITE/READ error: {}".format(err)) + # Sandbox testing code # dbc = dbState.getDbConn() # while True: - # rows = dbc.query("show databases") + # rows = dbc.query("show databases") # print("Rows: {}, time={}".format(rows, time.time())) - return + return + def main(): - # Super cool Python argument library: https://docs.python.org/3/library/argparse.html + # Super cool Python argument library: + # https://docs.python.org/3/library/argparse.html parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\ @@ -2106,52 +2394,90 @@ def main(): ''')) - parser.add_argument('-a', '--auto-start-service', action='store_true', - help='Automatically start/stop the TDengine service (default: false)') - parser.add_argument('-c', '--connector-type', action='store', default='native', type=str, - help='Connector type to use: native, rest, or mixed (default: 10)') - parser.add_argument('-d', '--debug', action='store_true', - help='Turn on DEBUG mode for more logging (default: false)') - parser.add_argument('-e', '--run-tdengine', action='store_true', - help='Run TDengine service in foreground (default: false)') - parser.add_argument('-l', '--larger-data', action='store_true', - help='Write larger amount of data during write operations (default: false)') - parser.add_argument('-p', '--per-thread-db-connection', action='store_true', - help='Use a single shared db connection (default: false)') - parser.add_argument('-r', '--record-ops', action='store_true', - help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') - parser.add_argument('-s', '--max-steps', action='store', default=1000, type=int, - help='Maximum number of steps to run (default: 100)') - parser.add_argument('-t', '--num-threads', action='store', default=5, type=int, - help='Number of threads to run (default: 10)') - parser.add_argument('-x', '--continue-on-exception', action='store_true', - help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') + parser.add_argument( + '-a', + '--auto-start-service', + action='store_true', + help='Automatically start/stop the TDengine service (default: false)') + parser.add_argument( + '-c', + '--connector-type', + action='store', + default='native', + type=str, + help='Connector type to use: native, rest, or mixed (default: 10)') + parser.add_argument( + '-d', + '--debug', + action='store_true', + help='Turn on DEBUG mode for more logging (default: false)') + parser.add_argument( + '-e', + '--run-tdengine', + action='store_true', + help='Run TDengine service in foreground (default: false)') + parser.add_argument( + '-l', + '--larger-data', + action='store_true', + help='Write larger amount of data during write operations (default: false)') + parser.add_argument( + '-p', + '--per-thread-db-connection', + action='store_true', + help='Use a single shared db connection (default: false)') + parser.add_argument( + '-r', + '--record-ops', + action='store_true', + help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') + parser.add_argument( + '-s', + '--max-steps', + action='store', + default=1000, + type=int, + help='Maximum number of steps to run (default: 100)') + parser.add_argument( + '-t', + '--num-threads', + action='store', + default=5, + type=int, + help='Number of threads to run (default: 10)') + parser.add_argument( + '-x', + '--continue-on-exception', + action='store_true', + help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') global gConfig gConfig = parser.parse_args() - + # Logging Stuff global logger - _logger = logging.getLogger('CrashGen') # real logger - _logger.addFilter(LoggingFilter()) + _logger = logging.getLogger('CrashGen') # real logger + _logger.addFilter(LoggingFilter()) ch = logging.StreamHandler() _logger.addHandler(ch) - logger = MyLoggingAdapter(_logger, []) # Logging adapter, to be used as a logger + # Logging adapter, to be used as a logger + logger = MyLoggingAdapter(_logger, []) - if ( gConfig.debug ): - logger.setLevel(logging.DEBUG) # default seems to be INFO + if (gConfig.debug): + logger.setLevel(logging.DEBUG) # default seems to be INFO else: logger.setLevel(logging.INFO) - - Dice.seed(0) # initial seeding of dice - + + Dice.seed(0) # initial seeding of dice + # Run server or client - if gConfig.run_tdengine : # run server + if gConfig.run_tdengine: # run server MainExec.runService() - else : + else: return MainExec.runClient() + if __name__ == "__main__": exitCode = main() # print("Exiting with code: {}".format(exitCode)) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index de80361aa3..f6be6aae49 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -31,11 +31,22 @@ then exit -1 fi +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. +else + TAOS_DIR=$CURR_DIR/../.. +fi +TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + +LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib + # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 # Then let us set up the library path so that our compiled SO file can be loaded by Python -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params python3 ./crash_gen.py $@ diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7a0f2cb825..5ee33c421e 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -156,3 +156,7 @@ python3 ./test.py -f alter/alter_table_crash.py # client python3 ./test.py -f client/client.py + +# Misc +python3 testCompress.py +python3 testNoCompress.py diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 46098f4040..ccc6635ced 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -150,3 +150,7 @@ python3 ./test.py -f alter/alter_table_crash.py # client python3 ./test.py -f client/client.py + +# Misc +python3 testCompress.py +python3 testNoCompress.py diff --git a/tests/pytest/table/del_stable.py b/tests/pytest/table/del_stable.py index e458b6679f..1b078ac519 100644 --- a/tests/pytest/table/del_stable.py +++ b/tests/pytest/table/del_stable.py @@ -37,17 +37,8 @@ class TDTestCase: except Exception as e: tdLog.exit(e) - try: - tdSql.execute("select * from db.st") - except Exception as e: - if e.args[0] != 'mnode invalid table name': - tdLog.exit(e) - - try: - tdSql.execute("select * from db.tb") - except Exception as e: - if e.args[0] != 'mnode invalid table name': - tdLog.exit(e) + tdSql.error("select * from db.st") + tdSql.error("select * from db.tb") def stop(self): tdSql.close() diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py new file mode 100644 index 0000000000..db3a5fea93 --- /dev/null +++ b/tests/pytest/util/dnodes-random-fail.py @@ -0,0 +1,500 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import os.path +import subprocess +from util.log import * + + +class TDSimClient: + def __init__(self): + self.testCluster = False + + self.cfgDict = { + "numOfLogLines": "100000000", + "numOfThreadsPerCore": "2.0", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "anyIp": "0", + "sdbDebugFlag": "135", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + } + + def init(self, path): + self.__init__() + self.path = path + + def getLogDir(self): + self.logDir = "%s/sim/psim/log" % (self.path) + return self.logDir + + def getCfgDir(self): + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + return self.cfgDir + + def setTestCluster(self, value): + self.testCluster = value + + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def deploy(self): + self.logDir = "%s/sim/psim/log" % (self.path) + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("logDir", self.logDir) + + for key, value in self.cfgDict.items(): + self.cfg(key, value) + + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) + + +class TDDnode: + def __init__(self, index): + self.index = index + self.running = 0 + self.deployed = 0 + self.testCluster = False + self.valgrind = 0 + + def init(self, path): + self.path = path + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def getDataSize(self): + totalSize = 0 + + if (self.deployed == 1): + for dirpath, dirnames, filenames in os.walk(self.dataDir): + for f in filenames: + fp = os.path.join(dirpath, f) + + if not os.path.islink(fp): + totalSize = totalSize + os.path.getsize(fp) + + return totalSize + + def deploy(self): + self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( + self.path, self.index) + + cmd = "rm -rf " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.startIP() + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) + self.cfg("dataDir", self.dataDir) + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("mnodeEqualVnodeNum", "0") + self.cfg("walLevel", "1") + self.cfg("statusInterval", "1") + self.cfg("numOfTotalVnodes", "64") + self.cfg("numOfMnodes", "3") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("monitor", "0") + self.cfg("maxVnodeConnections", "30000") + self.cfg("maxMgmtConnections", "30000") + self.cfg("maxMeterConnections", "30000") + self.cfg("maxShellConns", "30000") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "UTF-8") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("dDebugFlag", "135") + self.cfg("mDebugFlag", "135") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("httpDebugFlag", "135") + self.cfg("monitorDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + self.deployed = 1 + tdLog.debug( + "dnode:%d is deployed and configured by %s" % + (self.index, self.cfgPath)) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def start(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + if self.valgrind == 0: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + else: + valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + + cmd = "nohup %s %s -c %s --random-file-fail-factor 5 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) + + print(cmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + + tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) + time.sleep(5) + + def stop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) + + def forcestop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) + + def startIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def stopIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( + self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def getDnodeRootDir(self, index): + dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + return dnodeRootDir + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim/psim" % (self.path) + return dnodesRootDir + + +class TDDnodes: + def __init__(self): + self.dnodes = [] + self.dnodes.append(TDDnode(1)) + self.dnodes.append(TDDnode(2)) + self.dnodes.append(TDDnode(3)) + self.dnodes.append(TDDnode(4)) + self.dnodes.append(TDDnode(5)) + self.dnodes.append(TDDnode(6)) + self.dnodes.append(TDDnode(7)) + self.dnodes.append(TDDnode(8)) + self.dnodes.append(TDDnode(9)) + self.dnodes.append(TDDnode(10)) + self.simDeployed = False + + def init(self, path): + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug("binPath %s" % (binPath)) + binPath = os.path.realpath(binPath) + tdLog.debug("binPath real path %s" % (binPath)) + + # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) + # tdLog.debug(cmd) + # os.system(cmd) + + # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + if path == "": + # self.path = os.path.expanduser('~') + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + + for i in range(len(self.dnodes)): + self.dnodes[i].init(self.path) + + self.sim = TDSimClient() + self.sim.init(self.path) + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def deploy(self, index): + self.sim.setTestCluster(self.testCluster) + + if (self.simDeployed == False): + self.sim.deploy() + self.simDeployed = True + + self.check(index) + self.dnodes[index - 1].setTestCluster(self.testCluster) + self.dnodes[index - 1].setValgrind(self.valgrind) + self.dnodes[index - 1].deploy() + + def cfg(self, index, option, value): + self.check(index) + self.dnodes[index - 1].cfg(option, value) + + def start(self, index): + self.check(index) + self.dnodes[index - 1].start() + + def stop(self, index): + self.check(index) + self.dnodes[index - 1].stop() + + def getDataSize(self, index): + self.check(index) + return self.dnodes[index - 1].getDataSize() + + def forcestop(self, index): + self.check(index) + self.dnodes[index - 1].forcestop() + + def startIP(self, index): + self.check(index) + + if self.testCluster: + self.dnodes[index - 1].startIP() + + def stopIP(self, index): + self.check(index) + + if self.dnodes[index - 1].testCluster: + self.dnodes[index - 1].stopIP() + + def check(self, index): + if index < 1 or index > 10: + tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + + def stopAll(self): + tdLog.info("stop all dnodes") + for i in range(len(self.dnodes)): + self.dnodes[i].stop() + + psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + if processID: + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim" % (self.path) + return dnodesRootDir + + def getSimCfgPath(self): + return self.sim.getCfgDir() + + def getSimLogPath(self): + return self.sim.getLogDir() + + def addSimExtraCfg(self, option, value): + self.sim.addExtraCfg(option, value) + + +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 226682ff92..0341b3f136 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -28,8 +28,9 @@ class TDSimClient: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", + "maxTablesPerVnode": "4", + "maxVgroupsPerDb": "1000", + "sdbDebugFlag": "143", "rpcDebugFlag": "135", "tmrDebugFlag": "131", "cDebugFlag": "135", @@ -37,7 +38,6 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", } - def init(self, path): self.__init__() self.path = path diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index f871ead11d..076f7364ee 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start @@ -10,7 +10,7 @@ sleep 3000 sql connect print ============================ dnode1 start -sql create database db maxTables 500 cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 +sql create database db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $data00 != db then @@ -31,13 +31,10 @@ endi if $data06 != 20,20,20 then return -1 endi -if $data07 != 500 then +if $data07 != 2 then return -1 endi -if $data08 != 2 then - return -1 -endi -if $data09 != 4 then +if $data08 != 4 then return -1 endi @@ -46,7 +43,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT return sql_error alter database db cache 256 sql_error alter database db blocks 1 -sql_error alter database db maxTables 10 sql_error alter database db days 10 sql_error alter database db keep 10 sql_error alter database db minRows 350 @@ -59,7 +55,6 @@ sql_error alter database db replica 2 print ============== step3 -sql alter database db maxTables 1000 sql alter database db comp 1 sql alter database db blocks 40 sql alter database db keep 30 diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index bf950a5e49..9ef39fb556 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -1,11 +1,13 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 2 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c wallevel -v 2 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 2 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start @@ -17,7 +19,7 @@ sleep 1000 print ============================ step1 -sql create database db maxTables 5 +sql create database db sql create table db.st (ts timestamp, i int) tags(t int) sql create table db.t000 using db.st tags(0) sql create table db.t001 using db.st tags(1) @@ -74,9 +76,14 @@ if $rows != 20 then endi print ============================ step3 - -sql alter database db maxTables 10 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sleep 5000 sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -133,9 +140,14 @@ if $rows != 40 then endi print ============================ step5 - -sql alter database db maxTables 15 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sleep 5000 sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -252,9 +264,14 @@ if $rows != 60 then endi print ============================ step9 - -sql alter database db maxTables 20 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sleep 5000 sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -380,9 +397,14 @@ if $rows != 80 then endi print ============================ step9 - -sql alter database db maxTables 25 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sleep 5000 sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v1.sim b/tests/script/general/db/alter_tables_v1.sim index ccddb8f959..dde5eb6d9e 100644 --- a/tests/script/general/db/alter_tables_v1.sim +++ b/tests/script/general/db/alter_tables_v1.sim @@ -1,8 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/exec.sh -n dnode1 -s start @@ -11,7 +11,7 @@ sql connect print ============================ step1 -sql create database db maxTables 10 +sql create database db sql create table db.st (ts timestamp, i int) tags(t int) sql create table db.t0 using db.st tags(0) sql create table db.t1 using db.st tags(1) @@ -49,8 +49,11 @@ endi print ============================ step3 -sql alter database db maxTables 20 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t10 using db.st tags(0) sql create table db.t11 using db.st tags(1) @@ -86,9 +89,11 @@ if $rows != 20 then endi print ============================ step5 - -sql alter database db maxTables 30 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t20 using db.st tags(0) sql create table db.t21 using db.st tags(1) @@ -183,9 +188,11 @@ if $rows != 30 then endi print ============================ step9 - -sql alter database db maxTables 40 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t30 using db.st tags(0) sql create table db.t31 using db.st tags(1) @@ -285,9 +292,11 @@ if $rows != 40 then endi print ============================ step12 - -sql alter database db maxTables 50 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t40 using db.st tags(0) sql create table db.t41 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index 75687b417e..7c9262874d 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -1,8 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start @@ -11,7 +11,7 @@ sql connect print ============================ step1 -sql create database db maxTables 5 +sql create database db sql create table db.st (ts timestamp, i int) tags(t int) sql create table db.t000 using db.st tags(0) sql create table db.t001 using db.st tags(1) @@ -68,9 +68,11 @@ if $rows != 20 then endi print ============================ step3 - -sql alter database db maxTables 10 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -127,9 +129,11 @@ if $rows != 40 then endi print ============================ step5 - -sql alter database db maxTables 15 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -244,9 +248,11 @@ if $rows != 60 then endi print ============================ step9 - -sql alter database db maxTables 20 -sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -370,10 +376,11 @@ if $rows != 80 then endi print ============================ step12 - -sql alter database db maxTables 25 -sleep 1000 - +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +sleep 5000 sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) sql create table db.t402 using db.st tags(2) diff --git a/tests/script/general/db/alter_vgroups.sim b/tests/script/general/db/alter_vgroups.sim index 93f3b1c0e5..13928cf033 100644 --- a/tests/script/general/db/alter_vgroups.sim +++ b/tests/script/general/db/alter_vgroups.sim @@ -1,8 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/exec.sh -n dnode1 -s start @@ -11,7 +11,7 @@ sql connect print ============================ step1 -sql create database db maxTables 20 +sql create database db sql create table db.st (ts timestamp, i int) tags(t int) sql create table db.t000 using db.st tags(0) sql create table db.t001 using db.st tags(1) @@ -69,7 +69,7 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 2 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2 sleep 5000 system sh/exec.sh -n dnode1 -s start sleep 5000 @@ -131,7 +131,7 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 3 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 sleep 5000 system sh/exec.sh -n dnode1 -s start sleep 5000 diff --git a/tests/script/general/db/basic.sim b/tests/script/general/db/basic.sim index 43b18abd32..9983df378e 100644 --- a/tests/script/general/db/basic.sim +++ b/tests/script/general/db/basic.sim @@ -1,8 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 +sql create database $db replica 1 days 20 keep 2000 cache 16 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then @@ -35,7 +35,7 @@ endi if $data05 != 20 then return -1 endi -if $data07 != 1000 then +if $data07 != 16 then return -1 endi @@ -76,13 +76,6 @@ if $data05 != 15 then return -1 endi -#if $data06 != 1500,15000,1500 then -# return -1 -#endi -if $data07 != 1000 then - return -1 -endi - print =============== step6 sql use $db sql create table $tb (ts timestamp, speed int) diff --git a/tests/script/general/db/delete.sim b/tests/script/general/db/delete.sim index 6d0090a78d..477962a32d 100644 --- a/tests/script/general/db/delete.sim +++ b/tests/script/general/db/delete.sim @@ -2,6 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 10 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start @@ -9,7 +11,7 @@ sleep 3000 sql connect print ======== step1 -sql create database db blocks 2 maxtables 1000 +sql create database db blocks 2 sql create table db.mt (ts timestamp, tbcol int) TAGS(tgcol int) $tbPrefix = db.t @@ -21,7 +23,7 @@ while $i < 2000 endw sql show db.vgroups -if $rows != 2 then +if $rows != 10 then return -1 endi diff --git a/tests/script/general/db/tables.sim b/tests/script/general/db/tables.sim index bf714dbf56..d700bf8068 100644 --- a/tests/script/general/db/tables.sim +++ b/tests/script/general/db/tables.sim @@ -2,15 +2,15 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect print =============== step2 -sql create database db maxtables 4 +sql create database db sql show databases print $rows $data07 @@ -18,10 +18,6 @@ if $rows != 1 then return -1 endi -if $data07 != 4 then - return -1 -endi - print =============== step3 sql use db sql create table t1 (ts timestamp, i int) @@ -78,7 +74,7 @@ sql reset query cache sleep 4000 print =============== step7 -sql create database db maxtables 4 +sql create database db sql show databases print $rows $data07 @@ -86,10 +82,6 @@ if $rows != 1 then return -1 endi -if $data07 != 4 then - return -1 -endi - print =============== step8 sql use db sql create table t1 (ts timestamp, i int) diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim index 54d8aa77c3..769eec3241 100644 --- a/tests/script/general/db/vnodes.sim +++ b/tests/script/general/db/vnodes.sim @@ -6,7 +6,8 @@ $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $maxTables +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v $maxTables +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000 @@ -17,7 +18,7 @@ print ========== prepare data system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect -sql create database db blocks 2 cache 1 maxTables $maxTables +sql create database db blocks 2 cache 1 sql use db print ========== step1 diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 60ffa4cb28..7edf394845 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","maxtables","cache(MB)","blocks","minrows","maxrows","ctime(Sec.)","wallevel","comp","precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","comp","precision","status"],"data":[],"rows":0}@ then return -1 endi diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 2fbe03c093..be8587f2ae 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -128,12 +128,12 @@ endi if $data06 != 365,365,365 then return -1 endi -print data08 = $data08 -if $data08 != $cache then - print expect $cache, actual:$data08 +print data07 = $data07 +if $data07 != $cache then + print expect $cache, actual:$data07 return -1 endi -if $data09 != 4 then +if $data08 != 4 then return -1 endi diff --git a/tests/script/general/parser/fill_us.sim b/tests/script/general/parser/fill_us.sim index a66629c90b..b597d378a2 100644 --- a/tests/script/general/parser/fill_us.sim +++ b/tests/script/general/parser/fill_us.sim @@ -652,25 +652,25 @@ endi if $data01 != 1 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 1 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 1 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 1 then @@ -689,25 +689,25 @@ endi if $data01 != 0.000000000 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1.000000000 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 2.000000000 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 3.000000000 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 4.000000000 then @@ -722,25 +722,25 @@ endi if $data01 != 0 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 2 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 3 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 4 then @@ -755,25 +755,25 @@ endi if $data01 != 0 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 2 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 3 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 4 then @@ -788,25 +788,25 @@ endi if $data01 != 0 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 2 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 3 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 4 then @@ -821,25 +821,25 @@ endi if $data01 != 0 then return -1 endi -if $data11 != null then +if $data11 != NULL then return -1 endi if $data21 != 1 then return -1 endi -if $data31 != null then +if $data31 != NULL then return -1 endi if $data41 != 2 then return -1 endi -if $data51 != null then +if $data51 != NULL then return -1 endi if $data61 != 3 then return -1 endi -if $data71 != null then +if $data71 != NULL then return -1 endi if $data81 != 4 then diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index 2129c6b82a..d8c72349db 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/parser/limit1.sim b/tests/script/general/parser/limit1.sim index 84b760ced9..8b927b2a7f 100644 --- a/tests/script/general/parser/limit1.sim +++ b/tests/script/general/parser/limit1.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/parser/limit1_tblocks100.sim b/tests/script/general/parser/limit1_tblocks100.sim index 29b48bdd43..f1702924aa 100644 --- a/tests/script/general/parser/limit1_tblocks100.sim +++ b/tests/script/general/parser/limit1_tblocks100.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/parser/limit_stb.sim b/tests/script/general/parser/limit_stb.sim index 9983424dbf..c9df03ab2e 100644 --- a/tests/script/general/parser/limit_stb.sim +++ b/tests/script/general/parser/limit_stb.sim @@ -63,6 +63,11 @@ if $data41 != 9 then endi sql select * from $stb order by ts asc limit 5 +print select * from $stb order by ts asc limit 5 +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + if $rows != 5 then return -1 endi diff --git a/tests/script/general/parser/limit_tb.sim b/tests/script/general/parser/limit_tb.sim index 9e62d652e2..970c7b75c5 100644 --- a/tests/script/general/parser/limit_tb.sim +++ b/tests/script/general/parser/limit_tb.sim @@ -356,6 +356,11 @@ if $rows != 0 then return -1 endi sql select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 +print select top(c1, 5) from $tb where ts >= $ts0 and ts <= $tsu order by ts desc limit 3 offset 1 +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + if $rows != 3 then return -1 endi diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim index 9d7ac9678d..38af57c73d 100644 --- a/tests/script/general/parser/set_tag_vals.sim +++ b/tests/script/general/parser/set_tag_vals.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/stream/metrics_replica1_vnoden.sim b/tests/script/general/stream/metrics_replica1_vnoden.sim index ee071fd681..abe3e2ffdf 100644 --- a/tests/script/general/stream/metrics_replica1_vnoden.sim +++ b/tests/script/general/stream/metrics_replica1_vnoden.sim @@ -4,7 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start sleep 3000 diff --git a/tests/script/general/stream/table_replica1_vnoden.sim b/tests/script/general/stream/table_replica1_vnoden.sim index e1d5a9babf..ee0c871d98 100644 --- a/tests/script/general/stream/table_replica1_vnoden.sim +++ b/tests/script/general/stream/table_replica1_vnoden.sim @@ -4,7 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start sleep 3000 diff --git a/tests/script/general/table/limit.sim b/tests/script/general/table/limit.sim index 46fc5103c1..067182e1c8 100644 --- a/tests/script/general/table/limit.sim +++ b/tests/script/general/table/limit.sim @@ -3,6 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 129 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 8 system sh/exec.sh -n dnode1 -s start sleep 3000 @@ -36,7 +37,7 @@ while $x > $y endw sql show vgroups -if $rows != 2 then +if $rows != 8 then return -1 endi @@ -51,7 +52,7 @@ while $x > $y endw sql show vgroups -if $rows != 4 then +if $rows != 8 then return -1 endi @@ -66,7 +67,7 @@ while $x > $y endw sql show vgroups -if $rows != 6 then +if $rows != 8 then return -1 endi diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bead4bd095..3b9806558e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -312,9 +312,9 @@ cd ../../../debug; make #./test.sh -f general/parser/stream_on_sys.sim #./test.sh -f general/parser/repeatStream.sim -./test.sh -f general/stream/metrics_del.sim +#./test.sh -f general/stream/metrics_del.sim ./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim +#./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim @@ -324,7 +324,7 @@ cd ../../../debug; make ./test.sh -f general/stream/table_replica1_vnoden.sim ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -358,6 +358,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim + ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 37be89f8d6..8f9569e09d 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -133,11 +133,13 @@ echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG -echo "numOfLogLines 10000000" >> $TAOS_CFG -echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "numOfLogLines 10000000" >> $TAOS_CFG +echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG echo "clog 2" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG echo "numOfTotalVnodes 4" >> $TAOS_CFG +echo "maxVgroupsPerDb 4" >> $TAOS_CFG +echo "maxTablesPerVnode 1000" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "numOfMnodes 1" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh new file mode 100755 index 0000000000..7ba301617c --- /dev/null +++ b/tests/script/sh/exec-random-fail.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# if [ $# != 4 || $# != 5 ]; then + # echo "argument list need input : " + # echo " -n nodeName" + # echo " -s start/stop" + # echo " -c clear" + # exit 1 +# fi + +NODE_NAME= +EXEC_OPTON= +CLEAR_OPTION="false" +while getopts "n:s:u:x:ct" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + c) + CLEAR_OPTION="clear" + ;; + t) + SHELL_OPTION="true" + ;; + u) + USERS=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data +MGMT_DIR=$NODE_DIR/data/mgmt +TSDB_DIR=$NODE_DIR/data/tsdb + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg + +echo ------------ $EXEC_OPTON $NODE_NAME + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ]; then + EXE_DIR=/usr/local/bin/taos +fi + +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR + rm -rf $TSDB_DIR + rm -rf $MGMT_DIR +fi + +if [ "$EXEC_OPTON" = "start" ]; then + echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR + + if [ "$SHELL_OPTION" = "true" ]; then + nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 5 > /dev/null 2>&1 & + fi + +else + #relative path + RCFG_DIR=sim/$NODE_NAME/cfg + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGINT" ]; then + echo try to kill by signal SIGINT + kill -SIGINT $PID + else + echo try to kill by signal SIGKILL + kill -9 $PID + fi + sleep 1 + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + done +fi + diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 2480c20709..85146bd4c3 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -25,6 +25,12 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 20 + system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode2 -c http -v 1 -system sh/cfg.sh -n dnode3 -c http -v 1 \ No newline at end of file +system sh/cfg.sh -n dnode3 -c http -v 1 +system sh/cfg.sh -n dnode4 -c http -v 1 \ No newline at end of file diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim index af2b47d1ea..1b22db9d82 100644 --- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim +++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim @@ -161,7 +161,7 @@ sleep 10000 $loopCnt = 0 wait_dnode_offline_overtime_dropped: $loopCnt = $loopCnt + 1 -if $loopCnt == 10 then +if $loopCnt == 20 then return -1 endi sql show dnodes diff --git a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim index c3bae9005a..5827b25ff4 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -56,11 +62,10 @@ sql create dnode $hostname3 sql create dnode $hostname4 sleep 3000 -$totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 sql use $db # create table , insert data diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index 807709b675..dd101bc97c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -54,11 +60,10 @@ sql create dnode $hostname2 sql create dnode $hostname3 sleep 3000 -$totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data @@ -137,8 +142,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_2 -$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data9_2 if $dnode3Vtatus != offline then sleep 2000 @@ -204,8 +209,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_2 -$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data9_2 print dnode2Vtatus: $dnode3Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -319,8 +324,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_2 -$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index a05681ece2..6cd2e0cca3 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -56,11 +62,10 @@ sql create dnode $hostname3 sql create dnode $hostname4 sleep 3000 -$totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 sql use $db # create table , insert data @@ -139,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 @@ -206,8 +211,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -325,8 +330,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -386,8 +391,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim index cacbcbb132..26f45cf645 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim @@ -23,6 +23,12 @@ system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 16 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 16 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 16 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 16 +system sh/cfg.sh -n dnode5 -c maxVgroupsPerDb -v 16 + system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 @@ -215,6 +221,7 @@ system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 16 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 @@ -322,7 +329,7 @@ $totalRows = 0 $tsStart = 1420041600000 $db = db1 -sql create database $db replica 2 maxTables 4 +sql create database $db replica 2 sql use $db $stb = stb sql create table $stb (ts timestamp, c1 int) tags(t1 int) diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 9b01b2fae3..8abd4460c5 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -52,11 +58,10 @@ system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 sleep 3000 -$totalTableNum = 10 $sleepTimer = 10000 $db = db -sql create database $db replica 1 maxTables $totalTableNum +sql create database $db replica 1 sql use $db # create table , insert data @@ -352,16 +357,16 @@ if $loopCnt == 10 then endi sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 $data10_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 $data10_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 $data10_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data8_1 -$thirdDnode_3 = $data8_2 -$thirdDnode_4 = $data8_3 -$thirdDnode_5 = $data8_4 +$thirdDnode_2 = $data10_1 +$thirdDnode_3 = $data10_2 +$thirdDnode_4 = $data10_3 +$thirdDnode_5 = $data10_4 if $thirdDnode_2 != null then sleep 2000 @@ -405,10 +410,10 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data5_1 -$sencodDnode_3 = $data5_2 -$sencodDnode_4 = $data5_3 -$sencodDnode_5 = $data5_4 +$sencodDnode_2 = $data7_1 +$sencodDnode_3 = $data7_2 +$sencodDnode_4 = $data7_3 +$sencodDnode_5 = $data7_4 if $sencodDnode_2 != null then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index 9b001820f6..654983bb0f 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,7 +66,7 @@ $totalTableNum = 10 $sleepTimer = 10000 $db = db -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data diff --git a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim index 152a65a75a..32674acade 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim @@ -39,10 +39,10 @@ system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10 -system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10 -system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10 -system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10 +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 5 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 5 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 5 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 5 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1 @@ -182,7 +182,7 @@ system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 5 system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1 system sh/exec.sh -n dnode4 -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index 13c3517e24..3c47fdf93b 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -137,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 @@ -212,8 +218,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -286,8 +292,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 0b4e385785..6b107aed60 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,7 +66,7 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data @@ -143,8 +149,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_2 -$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data9_2 if $dnode3Vtatus != offline then sleep 2000 @@ -233,8 +239,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != master then sleep 2000 @@ -312,8 +318,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -392,8 +398,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index c464d4023f..7e71fe3a8d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -195,8 +201,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -268,8 +274,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -324,8 +330,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim index f2aa000170..8cd31e4de6 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,7 +66,7 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data @@ -157,8 +163,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -230,8 +236,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -286,8 +292,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index cf20122d4c..43538cf95a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -56,11 +62,10 @@ sql create dnode $hostname3 sql create dnode $hostname4 sleep 3000 -$totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 sql use $db # create table , insert data @@ -141,8 +146,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 @@ -201,8 +206,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -301,8 +306,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -417,8 +422,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index ce05b0b5ff..5b5a9deffa 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,7 +66,7 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data @@ -143,8 +149,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_2 -$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data9_2 if $dnode3Vtatus != offline then sleep 2000 @@ -233,8 +239,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != master then sleep 2000 @@ -312,8 +318,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus != offline then sleep 2000 @@ -391,8 +397,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data7_2 -$dnode3Vtatus = $data4_2 +$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data6_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index f5bcd055a0..7696168bb9 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -34,6 +34,17 @@ system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 20 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 + system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator @@ -56,11 +67,10 @@ sql create dnode $hostname3 sql create dnode $hostname4 sleep 3000 -$totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 sql use $db # create table , insert data @@ -146,8 +156,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 @@ -169,7 +179,7 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db1 -sql create database $db replica 2 maxTables $totalTableNum +sql create database $db replica 2 sql use $db # create table , insert data @@ -254,8 +264,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data4_3 -$dnode2Vtatus = $data7_3 +$dnode3Vtatus = $data6_3 +$dnode2Vtatus = $data9_3 if $dnode3Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index 18ffb32eed..ea5e7608ee 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -138,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index 6e85429b77..6fa1da4e1d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 #system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 #system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -58,11 +64,10 @@ sleep 3000 $totalTableNum = 10 $sleepTimer = 3000 -$maxTables = $totalTableNum * 2 $db = db -print create database $db replica 2 maxTables $maxTables -sql create database $db replica 2 maxTables $maxTables +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -143,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index 0356f72a97..1d2a83c825 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 20 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -59,11 +65,9 @@ sleep 3000 $totalTableNum = 10 $sleepTimer = 3000 -$maxTables = $totalTableNum * 2 - $db = db -print create database $db replica 2 maxTables $maxTables -sql create database $db replica 2 maxTables $maxTables +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -144,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index dc2586b43b..630b4eaab8 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 721c7d60ba..43dba536c7 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 897c662b51..09a2424c85 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index 95ae8047fa..d94f341236 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -19,20 +19,21 @@ system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 - #system sh/cfg.sh -n dnode1 -c alternativeRole -v 1 #system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 #system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 #system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 20 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -58,11 +59,10 @@ sleep 3000 $totalTableNum = 10 $sleepTimer = 3000 -$maxTables = $totalTableNum * 2 $db = db -print create database $db replica 3 maxTables $maxTables -sql create database $db replica 3 maxTables $maxTables +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index a60f810eea..fc1c69709a 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 20 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 20 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -61,8 +67,8 @@ $sleepTimer = 3000 $maxTables = $totalTableNum * 2 $db = db -print create database $db replica 3 maxTables $maxTables -sql create database $db replica 3 maxTables $maxTables +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -143,8 +149,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index a619cbe826..72727f3df0 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index 3b7ee869ed..7e57b0a971 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index 1b2c8d1db6..a16e765a30 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -34,10 +34,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 12 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -65,7 +71,7 @@ sql connect $totalTableNum = 12 $db = db -sql create database $db replica 1 maxTables $totalTableNum +sql create database $db replica 1 sql use $db # create table , insert data diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index 4f66e5c135..8775da607c 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index 7bcd148bc9..84ae9554f3 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index f61b1e3cb9..c9dc302d3c 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index b27558fdf0..b7392eb960 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 2 maxTables $totalTableNum -sql create database $db replica 2 maxTables $totalTableNum +print create database $db replica 2 +sql create database $db replica 2 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index 2a283a6a0c..2fde90f3db 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 7d321ed381..f31d01b327 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_createTable.sim b/tests/script/unique/arbitrator/sync_replica3_createTable.sim index 1b93cc09f2..a07e6dc93d 100644 --- a/tests/script/unique/arbitrator/sync_replica3_createTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_createTable.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim index f5ceced668..8a586321e1 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -137,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 @@ -205,8 +211,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != slave then sleep 2000 @@ -238,8 +244,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != master then sleep 2000 @@ -312,8 +318,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != slave then sleep 2000 @@ -345,8 +351,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != master then sleep 2000 @@ -435,8 +441,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != slave then sleep 2000 @@ -468,8 +474,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != master then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 0f7ee50c39..fd07fda2d4 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index 9c33dd8bb4..1749477d64 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -29,10 +29,16 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +$totalTableNum = 10 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v $totalTableNum + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -60,8 +66,8 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -print create database $db replica 3 maxTables $totalTableNum -sql create database $db replica 3 maxTables $totalTableNum +print create database $db replica 3 +sql create database $db replica 3 sql use $db # create table , insert data @@ -142,8 +148,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data4_2 -$dnode3Vtatus = $data7_2 +$dnode4Vtatus = $data6_2 +$dnode3Vtatus = $data9_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index b2a15b0b96..259951581e 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -1,23 +1,28 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 1000 system sh/deploy.sh -n dnode3 -i 3 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 1000 system sh/deploy.sh -n dnode5 -i 5 system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 1000 +system sh/cfg.sh -n dnode5 -c maxVgroupsPerDb -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000 print =============== prepare data system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index ab93474a43..9603b5de64 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -9,9 +9,11 @@ $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v $totalVnodes +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v $totalVnodes print ========== prepare data diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index 819da67c8a..f05d64e635 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -12,6 +12,12 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 2 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 2 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start @@ -23,7 +29,7 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start print ======== step1 -sql create database db replica 3 blocks 2 maxtables 1000 +sql create database db replica 3 blocks 2 sql create table db.mt (ts timestamp, tbcol int) TAGS(tgcol int) $tbPrefix = db.t diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index ac84382be4..bf74e04b7f 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -24,10 +24,15 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 8 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 8 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 8 +system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 8 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index e0b5e9b931..9d53c9968e 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -199,13 +199,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data4_4 -$d2v3status = $data4_2 -$d2v4status = $data4_3 +$d2v2status = $data6_4 +$d2v3status = $data6_2 +$d2v4status = $data6_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 +$d1v2status = $data9_4 +$d1v3status = $data9_2 +$d1v4status = $data9_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index ae7fc6af17..af457f4cba 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -201,13 +201,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data4_4 -$d2v3status = $data4_2 -$d2v4status = $data4_3 +$d2v2status = $data6_4 +$d2v3status = $data6_2 +$d2v4status = $data6_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 +$d1v2status = $data9_4 +$d1v3status = $data9_2 +$d1v4status = $data9_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim index dc9bc62696..e4ff2c23c8 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim @@ -166,13 +166,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data4_4 -$d2v3status = $data4_2 -$d2v4status = $data4_3 +$d2v2status = $data6_4 +$d2v3status = $data6_2 +$d2v4status = $data6_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 +$d1v2status = $data9_4 +$d1v3status = $data9_2 +$d1v4status = $data9_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index b754dc7a49..6736530240 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -199,13 +199,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data4_4 -$d2v3status = $data4_2 -$d2v4status = $data4_3 +$d2v2status = $data6_4 +$d2v3status = $data6_2 +$d2v4status = $data6_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 +$d1v2status = $data9_4 +$d1v3status = $data9_2 +$d1v4status = $data9_3 if $d2v2status != master then sleep 2000 diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index ecd481f869..293427d240 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -30,8 +30,8 @@ #define MAX_FILE_NAME_LEN 256 #define MAX_ERROR_LEN 1024 #define MAX_QUERY_VALUE_LEN 40 -#define MAX_QUERY_COL_NUM 10 -#define MAX_QUERY_ROW_NUM 10 +#define MAX_QUERY_COL_NUM 20 +#define MAX_QUERY_ROW_NUM 20 #define MAX_SYSTEM_RESULT_LEN 2048 #define MAX_VAR_LEN 100 #define MAX_VAR_NAME_LEN 32 diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index b077547709..7826735c2f 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -80,6 +80,23 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } } return "null"; + } else if (varName[6] == '_') { + int col = (varName[4] - '0') * 10 + (varName[5] - '0'); + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + char *keyName; + int keyLen; + paGetToken(varName + 7, &keyName, &keyLen); + + for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyName, script->data[i][0], keyLen) == 0) { + simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + return "null"; } else { int row = varName[4] - '0'; int col = varName[5] - '0';