From 357384f0b1ba823e6dba06b9e3b32ab9af35bc9f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 12 Jun 2020 14:04:42 +0000 Subject: [PATCH 01/24] [TD-570] the first version of sdb write queue --- src/dnode/src/dnodeMWrite.c | 1 + src/inc/taoserror.h | 8 +- src/mnode/inc/mnodeSdb.h | 4 +- src/mnode/inc/mnodeUser.h | 2 +- src/mnode/src/mnodeAcct.c | 4 +- src/mnode/src/mnodeDnode.c | 1 + src/mnode/src/mnodeSdb.c | 362 +++++++++++++++++++++++++++-------- src/mnode/src/mnodeUser.c | 23 ++- tests/script/tmp/prepare.sim | 8 +- 9 files changed, 317 insertions(+), 96 deletions(-) diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index eb9d8653e5..41714a0df7 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -128,6 +128,7 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) { void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) { SMnodeMsg *pWrite = pRaw; + if (pWrite == NULL) return; if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { dnodeReprocessMnodeWriteMsg(pWrite); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ac2af75742..fa1cd25b7c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -113,8 +113,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "mnode inva TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "mnode invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "mnode invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "mnode object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "mnode sdb error") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "[sdb] object already there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "[sdb] app error") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, 0, 0x0322, "[sdb] invalid table type") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, 0, 0x0323, "[sdb] object not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, 0, 0x0324, "[sdb] invalid meta row") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, 0, 0x0325, "[sdb] invalid key type") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "mnode dnode already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "mnode dnode not exist") diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 975206d52e..88578f79ca 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -48,8 +48,10 @@ typedef struct { ESdbOper type; void * table; void * pObj; - int32_t rowSize; + void * pMnodeMsg; void * rowData; + int32_t rowSize; + int32_t retCode; // for callback in sdb queue } SSdbOper; typedef struct { diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 338b43b47d..2ff0b4be81 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -28,7 +28,7 @@ void * mnodeGetNextUser(void *pIter, SUserObj **pUser); void mnodeIncUserRef(SUserObj *pUser); void mnodeDecUserRef(SUserObj *pUser); SUserObj *mnodeGetUserFromConn(void *pConn); -int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass); +int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); #ifdef __cplusplus diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 7ea5188b96..4c79564769 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -78,7 +78,9 @@ static int32_t mnodeAcctActionDecode(SSdbOper *pOper) { } static int32_t mnodeAcctActionRestored() { - if (dnodeIsFirstDeploy()) { + int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb); + if (numOfRows <= 0 && dnodeIsFirstDeploy()) { + mPrint("dnode first deploy, create root acct"); int32_t code = mnodeCreateRootAcct(); if (code != TSDB_CODE_SUCCESS) { mError("failed to create root account, reason:%s", tstrerror(code)); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index d2af86badb..b672140563 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -117,6 +117,7 @@ static int32_t mnodeDnodeActionDecode(SSdbOper *pOper) { static int32_t mnodeDnodeActionRestored() { int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { + mPrint("dnode first deploy, create dnode:%s", tsLocalEp); mnodeCreateDnode(tsLocalEp); SDnodeObj *pDnode = mnodeGetDnodeByEp(tsLocalEp); mnodeAddMnode(pDnode->dnodeId); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 659ac159a8..d4daefe25f 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -24,6 +24,7 @@ #include "tsync.h" #include "tglobal.h" #include "dnode.h" +#include "mnode.h" #include "mnodeDef.h" #include "mnodeInt.h" #include "mnodeMnode.h" @@ -83,8 +84,29 @@ typedef struct { void * row; } SSdbRow; +typedef struct { + pthread_t thread; + int32_t workerId; +} SSdbWriteWorker; + +typedef struct { + int32_t num; + SSdbWriteWorker *writeWorker; +} SSdbWriteWorkerPool; + static SSdbObject tsSdbObj = {0}; -static int sdbWrite(void *param, void *data, int type); +static taos_qset tsSdbWriteQset; +static taos_qall tsSdbWriteQall; +static taos_queue tsSdbWriteQueue; +static SSdbWriteWorkerPool tsSdbPool; + +static int sdbWrite(void *param, void *data, int type); +static int sdbWriteToQueue(void *param, void *data, int type); +static void * sdbWorkerFp(void *param); +static int32_t sdbInitWriteWorker(); +static void sdbCleanupWriteWorker(); +static int32_t sdbAllocWriteQueue(); +static void sdbFreeWritequeue(); int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; @@ -302,7 +324,7 @@ void sdbUpdateSync() { syncInfo.ahandle = NULL; syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.writeToCache = sdbWrite; + syncInfo.writeToCache = sdbWriteToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; tsSdbObj.cfg = syncCfg; @@ -319,10 +341,14 @@ int32_t sdbInit() { pthread_mutex_init(&tsSdbObj.mutex, NULL); sem_init(&tsSdbObj.sem, 0, 0); + if (sdbInitWriteWorker() != 0) { + return -1; + } + if (sdbInitWal() != 0) { return -1; } - + sdbRestoreTables(); if (mnodeGetMnodesNum() == 1) { @@ -340,6 +366,8 @@ void sdbCleanUp() { tsSdbObj.status = SDB_STATUS_CLOSING; + sdbCleanupWriteWorker(); + if (tsSdbObj.sync) { syncStop(tsSdbObj.sync); tsSdbObj.sync = NULL; @@ -494,9 +522,10 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { } static int sdbWrite(void *param, void *data, int type) { + SSdbOper *pOper = param; SWalHead *pHead = data; - int32_t tableId = pHead->msgType / 10; - int32_t action = pHead->msgType % 10; + int32_t tableId = pHead->msgType / 10; + int32_t action = pHead->msgType % 10; SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); @@ -531,21 +560,22 @@ static int sdbWrite(void *param, void *data, int type) { pthread_mutex_unlock(&tsSdbObj.mutex); return code; } - walFsync(tsSdbObj.wal); - + code = sdbForwardToPeer(pHead); pthread_mutex_unlock(&tsSdbObj.mutex); // from app, oper is created - if (param != NULL) { - //sdbTrace("request from app is disposed, version:%" PRIu64 " code:%s", pHead->version, tstrerror(code)); + if (pOper != NULL) { + sdbTrace("record from app is disposed, version:%" PRIu64 " code:%s", pHead->version, tstrerror(code)); return code; } // from wal or forward msg, oper not created, should add into hash if (tsSdbObj.sync != NULL) { - sdbTrace("forward request is received, version:%" PRIu64 " result:%s, confirm it", pHead->version, tstrerror(code)); + sdbTrace("record from wal forward is disposed, version:%" PRIu64 " confirm it", pHead->version); syncConfirmForward(tsSdbObj.sync, pHead->version, code); + } else { + sdbTrace("record from wal restore is disposed, version:%" PRIu64 , pHead->version); } if (action == SDB_ACTION_INSERT) { @@ -568,7 +598,7 @@ static int sdbWrite(void *param, void *data, int type) { int32_t sdbInsertRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; - if (pTable == NULL) return -1; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (sdbGetRowFromObj(pTable, pOper->pObj)) { sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj)); @@ -587,98 +617,132 @@ int32_t sdbInsertRow(SSdbOper *pOper) { pthread_mutex_unlock(&pTable->mutex); } - if (pOper->type == SDB_OPER_GLOBAL) { - int32_t size = sizeof(SWalHead) + pTable->maxRowSize; - SWalHead *pHead = taosAllocateQitem(size); - pHead->version = 0; - pHead->len = pOper->rowSize; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; - - pOper->rowData = pHead->cont; - (*pTable->encodeFp)(pOper); - pHead->len = pOper->rowSize; - - int32_t code = sdbWrite(pOper, pHead, pHead->msgType); - taosFreeQitem(pHead); - if (code < 0) return code; + int32_t code = sdbInsertHash(pTable, pOper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert into hash", pTable->tableName); + return code; } - return sdbInsertHash(pTable, pOper); + // just insert data into memory + if (pOper->type != SDB_OPER_GLOBAL) { + return TSDB_CODE_SUCCESS; + } + + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize; + SSdbOper *pNewOper = taosAllocateQitem(size); + + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + pHead->version = 0; + pHead->len = pOper->rowSize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; + + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + memcpy(pNewOper, pOper, sizeof(SSdbOper)); + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + return TSDB_CODE_SUCCESS; } int32_t sdbDeleteRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; - if (pTable == NULL) return -1; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); - return -1; + return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - void * pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - if (pOper->type == SDB_OPER_GLOBAL) { - void * key = sdbGetObjKey(pTable, pOper->pObj); - int32_t keySize = 0; - switch (pTable->keyType) { - case SDB_KEY_STRING: - case SDB_KEY_VAR_STRING: - keySize = strlen((char *)key) + 1; - break; - case SDB_KEY_INT: - case SDB_KEY_AUTO: - keySize = sizeof(uint32_t); - break; - default: - return -1; - } - - int32_t size = sizeof(SWalHead) + keySize; - SWalHead *pHead = taosAllocateQitem(size); - pHead->version = 0; - pHead->len = keySize; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; - memcpy(pHead->cont, key, keySize); - - int32_t code = sdbWrite(pOper, pHead, pHead->msgType); - taosFreeQitem(pHead); - if (code < 0) return code; + void *pMetaRow = pMeta->row; + if (pMetaRow == NULL) { + sdbError("table:%s, record meta is null", pTable->tableName); + return TSDB_CODE_MND_SDB_INVAID_META_ROW; } - return sdbDeleteHash(pTable, pOper); + int32_t code = sdbDeleteHash(pTable, pOper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to delete from hash", pTable->tableName); + return code; + } + + // just delete data from memory + if (pOper->type != SDB_OPER_GLOBAL) { + return TSDB_CODE_SUCCESS; + } + + void * key = sdbGetObjKey(pTable, pOper->pObj); + int32_t keySize = 0; + switch (pTable->keyType) { + case SDB_KEY_STRING: + case SDB_KEY_VAR_STRING: + keySize = strlen((char *)key) + 1; + break; + case SDB_KEY_INT: + case SDB_KEY_AUTO: + keySize = sizeof(uint32_t); + break; + default: + return TSDB_CODE_MND_SDB_INVAID_KEY_TYPE; + } + + + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize; + SSdbOper *pNewOper = taosAllocateQitem(size); + + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + pHead->version = 0; + pHead->len = keySize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; + memcpy(pHead->cont, key, keySize); + + memcpy(pNewOper, pOper, sizeof(SSdbOper)); + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + return TSDB_CODE_SUCCESS; } int32_t sdbUpdateRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; - if (pTable == NULL) return -1; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); - return -1; + sdbTrace("table:%s, record is not there, update failed", pTable->tableName); + return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - void * pMetaRow = pMeta->row; - assert(pMetaRow != NULL); + void *pMetaRow = pMeta->row; + if (pMetaRow == NULL) { + sdbError("table:%s, record meta is null", pTable->tableName); + return TSDB_CODE_MND_SDB_INVAID_META_ROW; + } - if (pOper->type == SDB_OPER_GLOBAL) { - int32_t size = sizeof(SWalHead) + pTable->maxRowSize; - SWalHead *pHead = taosAllocateQitem(size); - pHead->version = 0; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; + int32_t code = sdbUpdateHash(pTable, pOper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to update hash", pTable->tableName); + return code; + } - pOper->rowData = pHead->cont; - (*pTable->encodeFp)(pOper); - pHead->len = pOper->rowSize; + // just update data in memory + if (pOper->type != SDB_OPER_GLOBAL) { + return TSDB_CODE_SUCCESS; + } - int32_t code = sdbWrite(pOper, pHead, pHead->msgType); - taosFreeQitem(pHead); - if (code < 0) return code; - } - - return sdbUpdateHash(pTable, pOper); + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize; + SSdbOper *pNewOper = taosAllocateQitem(size); + + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + pHead->version = 0; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; + + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + memcpy(pNewOper, pOper, sizeof(SSdbOper)); + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + return TSDB_CODE_SUCCESS; } void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { @@ -775,3 +839,147 @@ void sdbCloseTable(void *handle) { free(pTable); } +int32_t sdbInitWriteWorker() { + tsSdbPool.num = 1; + tsSdbPool.writeWorker = (SSdbWriteWorker *)calloc(sizeof(SSdbWriteWorker), tsSdbPool.num); + + if (tsSdbPool.writeWorker == NULL) return -1; + for (int32_t i = 0; i < tsSdbPool.num; ++i) { + SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + pWorker->workerId = i; + } + + sdbAllocWriteQueue(); + + mPrint("sdb write is opened"); + return 0; +} + +void sdbCleanupWriteWorker() { + for (int32_t i = 0; i < tsSdbPool.num; ++i) { + SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + if (pWorker->thread) { + taosQsetThreadResume(tsSdbWriteQset); + } + } + + for (int32_t i = 0; i < tsSdbPool.num; ++i) { + SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + if (pWorker->thread) { + pthread_join(pWorker->thread, NULL); + } + } + + sdbFreeWritequeue(); + + mPrint("sdb write is closed"); +} + +int32_t sdbAllocWriteQueue() { + tsSdbWriteQueue = taosOpenQueue(); + if (tsSdbWriteQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; + + tsSdbWriteQset = taosOpenQset(); + if (tsSdbWriteQset == NULL) { + taosCloseQueue(tsSdbWriteQueue); + return TSDB_CODE_MND_OUT_OF_MEMORY; + } + taosAddIntoQset(tsSdbWriteQset, tsSdbWriteQueue, NULL); + + tsSdbWriteQall = taosAllocateQall(); + if (tsSdbWriteQall == NULL) { + taosCloseQset(tsSdbWriteQset); + taosCloseQueue(tsSdbWriteQueue); + return TSDB_CODE_MND_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < tsSdbPool.num; ++i) { + SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, sdbWorkerFp, pWorker) != 0) { + mError("failed to create thread to process sdb write queue, reason:%s", strerror(errno)); + taosFreeQall(tsSdbWriteQall); + taosCloseQset(tsSdbWriteQset); + taosCloseQueue(tsSdbWriteQueue); + return TSDB_CODE_MND_OUT_OF_MEMORY; + } + + pthread_attr_destroy(&thAttr); + mTrace("sdb write worker:%d is launched, total:%d", pWorker->workerId, tsSdbPool.num); + } + + mTrace("sdb write queue:%p is allocated", tsSdbWriteQueue); + return TSDB_CODE_SUCCESS; +} + +void sdbFreeWritequeue() { + taosCloseQset(tsSdbWriteQueue); + taosFreeQall(tsSdbWriteQall); + taosCloseQset(tsSdbWriteQset); + tsSdbWriteQall = NULL; + tsSdbWriteQset = NULL; + tsSdbWriteQueue = NULL; +} + +int sdbWriteToQueue(void *param, void *data, int type) { + SWalHead *pHead = data; + int size = sizeof(SWalHead) + pHead->len; + SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); + memcpy(pWal, pHead, size); + + taosWriteQitem(tsSdbWriteQueue, type, pWal); + return 0; +} + +static void *sdbWorkerFp(void *param) { + SWalHead *pHead; + SSdbOper *pOper; + int32_t type; + int32_t numOfMsgs; + void * item; + void * unUsed; + + while (1) { + numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed); + if (numOfMsgs == 0) { + sdbTrace("sdbWorkerFp: got no message from qset, exiting..."); + break; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(tsSdbWriteQall, &type, &item); + if (type == TAOS_QTYPE_RPC) { + pOper = (SSdbOper *)item; + pHead = (void *)pOper + sizeof(SSdbOper); + } else { + pHead = (SWalHead *)item; + pOper = NULL; + } + + int32_t code = sdbWrite(pOper, pHead, type); + if (pOper && code != TSDB_CODE_SUCCESS) { + pOper->retCode = code; + } + } + + walFsync(tsSdbObj.wal); + + // browse all items, and process them one by one + taosResetQitems(tsSdbWriteQall); + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(tsSdbWriteQall, &type, &item); + if (type == TAOS_QTYPE_RPC) { + pOper = (SSdbOper *)item; + dnodeSendRpcMnodeWriteRsp(pOper->pMnodeMsg, pOper->retCode); + } + taosFreeQitem(item); + } + } + + return NULL; +} diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 95457c83a0..24b8a67dc4 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -102,11 +102,13 @@ static int32_t mnodeUserActionDecode(SSdbOper *pOper) { } static int32_t mnodeUserActionRestored() { - if (dnodeIsFirstDeploy()) { + int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); + if (numOfRows <= 0 && dnodeIsFirstDeploy()) { + mPrint("dnode first deploy, create root user"); SAcctObj *pAcct = mnodeGetAcct("root"); - mnodeCreateUser(pAcct, "root", "taosdata"); - mnodeCreateUser(pAcct, "monitor", tsInternalPass); - mnodeCreateUser(pAcct, "_root", tsInternalPass); + mnodeCreateUser(pAcct, "root", "taosdata", NULL); + mnodeCreateUser(pAcct, "monitor", tsInternalPass, NULL); + mnodeCreateUser(pAcct, "_root", tsInternalPass, NULL); mnodeDecAcctRef(pAcct); } @@ -185,7 +187,7 @@ static int32_t mnodeUpdateUser(SUserObj *pUser) { return code; } -int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) { +int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_USER); if (code != TSDB_CODE_SUCCESS) { return code; @@ -226,16 +228,17 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) { .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, - .rowSize = sizeof(SUserObj) + .rowSize = sizeof(SUserObj), + .pMnodeMsg = pMsg }; code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { tfree(pUser); - code = TSDB_CODE_MND_SDB_ERROR; + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - return code; } static int32_t mnodeDropUser(SUserObj *pUser) { @@ -364,7 +367,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) { if (pOperUser->superAuth) { SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; - code = mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass); + code = mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg); if (code == TSDB_CODE_SUCCESS) { mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user); } diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index a71401d6e4..2480c20709 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -20,10 +20,10 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 20 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode2 -c http -v 1 From 585d95e91c317fa7bbc4299a6a09e3066b5e9ed6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 13 Jun 2020 03:30:39 +0000 Subject: [PATCH 02/24] [TD-570] add user/db into sdb write queue --- src/mnode/inc/mnodeSdb.h | 5 ++- src/mnode/inc/mnodeUser.h | 1 + src/mnode/src/mnodeDb.c | 95 ++++++++++++++++++++------------------- src/mnode/src/mnodeSdb.c | 5 ++- src/mnode/src/mnodeUser.c | 67 ++++++++++++++------------- 5 files changed, 93 insertions(+), 80 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 88578f79ca..df9dbf843e 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -20,6 +20,8 @@ extern "C" { #endif +struct SMnodeMsg; + typedef enum { SDB_TABLE_DNODE = 0, SDB_TABLE_MNODE = 1, @@ -48,10 +50,11 @@ typedef struct { ESdbOper type; void * table; void * pObj; - void * pMnodeMsg; void * rowData; int32_t rowSize; int32_t retCode; // for callback in sdb queue + void (*cb)(struct SMnodeMsg *pMsg, int32_t code); + struct SMnodeMsg *pMsg; } SSdbOper; typedef struct { diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 2ff0b4be81..073460f9d3 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -28,6 +28,7 @@ void * mnodeGetNextUser(void *pIter, SUserObj **pUser); void mnodeIncUserRef(SUserObj *pUser); void mnodeDecUserRef(SUserObj *pUser); SUserObj *mnodeGetUserFromConn(void *pConn); +char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 3666d61adc..21fcdc102c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -41,7 +41,7 @@ static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg); static int32_t mnodeDropDb(SMnodeMsg *newMsg); static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -308,7 +308,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->replications < 0) pCfg->replications = tsReplications; } -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); if (code != 0) return code; @@ -361,15 +361,17 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { .table = tsDbSdb, .pObj = pDb, .rowSize = sizeof(SDbObj), + .pMsg = pMsg }; code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { tfree(pDb); - code = TSDB_CODE_MND_SDB_ERROR; + return code; + } else { + mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - return code; } bool mnodeCheckIsMonitorDB(char *db, char *monitordb) { @@ -768,12 +770,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { } else if (!pMsg->pUser->writeAuth) { code = TSDB_CODE_MND_NO_RIGHTS; } else { - code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); - } else { - mError("db:%s, failed to create, reason:%s", pCreate->db, tstrerror(code)); - } + code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate, pMsg); } return code; @@ -890,7 +887,29 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return newCfg; } -static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { +static void mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) return; + SDbObj *pDb = pMsg->pDb; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + if (pVgroup->pDb == pDb) { + mnodeSendCreateVgroupMsg(pVgroup, NULL); + } + mnodeDecVgroupRef(pVgroup); + } + sdbFreeIter(pIter); + + mTrace("db:%s, all vgroups is altered", pDb->name); + mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); + + balanceNotify(); +} + +static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter); if (terrno != TSDB_CODE_SUCCESS) { return terrno; @@ -901,37 +920,25 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return code; } - int32_t oldReplica = pDb->cfg.replications; - if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; pDb->cfgVersion++; SSdbOper oper = { - .type = SDB_OPER_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, - .pObj = pDb + .pObj = pDb, + .pMsg = pMsg, + .cb = mnodeAlterDbCb }; int32_t code = sdbUpdateRow(&oper); if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; - mnodeSendCreateVgroupMsg(pVgroup, NULL); - mnodeDecVgroupRef(pVgroup); - } - sdbFreeIter(pIter); - - if (oldReplica != pDb->cfg.replications) { - balanceNotify(); - } - return TSDB_CODE_SUCCESS; } @@ -945,14 +952,7 @@ static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_INVALID_DB; } - int32_t code = mnodeAlterDb(pMsg->pDb, pAlter); - if (code != TSDB_CODE_SUCCESS) { - mError("db:%s, failed to alter, invalid db option", pAlter->db); - return code; - } - - mTrace("db:%s, all vgroups is altered", pMsg->pDb->name); - return TSDB_CODE_SUCCESS; + return mnodeAlterDb(pMsg->pDb, pAlter, pMsg); } static int32_t mnodeDropDb(SMnodeMsg *pMsg) { @@ -960,16 +960,19 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { mPrint("db:%s, drop db from sdb", pDb->name); SSdbOper oper = { - .type = SDB_OPER_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, - .pObj = pDb + .pObj = pDb, + .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&oper); - if (code != 0) { - code = TSDB_CODE_MND_SDB_ERROR; - } - return code; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return code; + } else { + mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } } static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index d4daefe25f..380a992473 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -975,7 +975,10 @@ static void *sdbWorkerFp(void *param) { taosGetQitem(tsSdbWriteQall, &type, &item); if (type == TAOS_QTYPE_RPC) { pOper = (SSdbOper *)item; - dnodeSendRpcMnodeWriteRsp(pOper->pMnodeMsg, pOper->retCode); + if (pOper->cb) { + (*pOper->cb)(pOper->pMsg, pOper->retCode); + } + dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode); } taosFreeQitem(item); } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 24b8a67dc4..57587fb65b 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -172,19 +172,21 @@ void mnodeDecUserRef(SUserObj *pUser) { return sdbDecRef(tsUserSdb, pUser); } -static int32_t mnodeUpdateUser(SUserObj *pUser) { +static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { SSdbOper oper = { - .type = SDB_OPER_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, - .pObj = pUser + .pObj = pUser, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_MND_SDB_ERROR; + return code; + } else { + mLPrint("user:%s, is altered by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - return code; } int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { @@ -225,11 +227,11 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { } SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsUserSdb, - .pObj = pUser, + .type = SDB_OPER_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, .rowSize = sizeof(SUserObj), - .pMnodeMsg = pMsg + .pMsg = pMsg }; code = sdbInsertRow(&oper); @@ -237,23 +239,26 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { tfree(pUser); return code; } else { + mLPrint("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } -static int32_t mnodeDropUser(SUserObj *pUser) { +static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { SSdbOper oper = { - .type = SDB_OPER_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, - .pObj = pUser + .pObj = pUser, + .pMsg = pMsg }; int32_t code = sdbDeleteRow(&oper); if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_MND_SDB_ERROR; + return code; + } else { + mLPrint("user:%s, is dropped by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - return code; } static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -361,22 +366,25 @@ SUserObj *mnodeGetUserFromConn(void *pConn) { } } +char *mnodeGetUserFromMsg(void *pMsg) { + SMnodeMsg *pMnodeMsg = pMsg; + if (pMnodeMsg != NULL &&pMnodeMsg->pUser != NULL) { + return pMnodeMsg->pUser->user; + } else { + return "system"; + } +} + static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) { - int32_t code; SUserObj *pOperUser = pMsg->pUser; if (pOperUser->superAuth) { SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; - code = mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user); - } + return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg); } else { mError("user:%s, no rights to create user", pOperUser->user); - code = TSDB_CODE_MND_NO_RIGHTS; + return TSDB_CODE_MND_NO_RIGHTS; } - - return code; } static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) { @@ -413,8 +421,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) { if (hasRight) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); - code = mnodeUpdateUser(pUser); - mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); + code = mnodeUpdateUser(pUser, pMsg); } else { mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_MND_NO_RIGHTS; @@ -454,8 +461,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) { pUser->writeAuth = 1; } - code = mnodeUpdateUser(pUser); - mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); + code = mnodeUpdateUser(pUser, pMsg); } else { mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_MND_NO_RIGHTS; @@ -501,10 +507,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { } if (hasRight) { - code = mnodeDropUser(pUser); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s, is dropped by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); - } + code = mnodeDropUser(pUser, pMsg); } else { code = TSDB_CODE_MND_NO_RIGHTS; } From fc82a732b85d2f151214a0eac405800fbd0a8812 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 13 Jun 2020 05:33:07 +0000 Subject: [PATCH 03/24] [TD-570] add dnode into sdb write queue --- src/mnode/inc/mnodeDnode.h | 2 +- src/mnode/src/mnodeDb.c | 25 +++++++++--------- src/mnode/src/mnodeDnode.c | 53 ++++++++++++++------------------------ src/mnode/src/mnodeUser.c | 21 ++++++++------- 4 files changed, 43 insertions(+), 58 deletions(-) diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index f95a163d83..9e21f8f56a 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -47,7 +47,7 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode); void * mnodeGetDnode(int32_t dnodeId); void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); -int32_t mnodeDropDnode(SDnodeObj *pDnode); +int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); extern int32_t tsAccessSquence; diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 21fcdc102c..7e0eb8ffc9 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -366,12 +366,13 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { - tfree(pDb); - return code; + tfree(pDb);; } else { mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } bool mnodeCheckIsMonitorDB(char *db, char *monitordb) { @@ -931,15 +932,13 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { .cb = mnodeAlterDbCb }; - int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return code; - } else { - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = sdbUpdateRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { @@ -967,12 +966,12 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { }; int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return code; - } else { + if (code == TSDB_CODE_SUCCESS) { mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index b672140563..816bcc6c5c 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -44,7 +44,7 @@ static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; extern void * tsVgroupSdb; -static int32_t mnodeCreateDnode(char *ep); +static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg); static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); @@ -118,7 +118,7 @@ static int32_t mnodeDnodeActionRestored() { int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mPrint("dnode first deploy, create dnode:%s", tsLocalEp); - mnodeCreateDnode(tsLocalEp); + mnodeCreateDnode(tsLocalEp, NULL); SDnodeObj *pDnode = mnodeGetDnodeByEp(tsLocalEp); mnodeAddMnode(pDnode->dnodeId); mnodeDecDnodeRef(pDnode); @@ -369,7 +369,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeCreateDnode(char *ep) { +static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { int32_t grantCode = grantCheck(TSDB_GRANT_DNODE); if (grantCode != TSDB_CODE_SUCCESS) { return grantCode; @@ -393,7 +393,8 @@ static int32_t mnodeCreateDnode(char *ep) { .type = SDB_OPER_GLOBAL, .table = tsDnodeSdb, .pObj = pDnode, - .rowSize = sizeof(SDnodeObj) + .rowSize = sizeof(SDnodeObj), + .pMsg = pMsg }; int32_t code = sdbInsertRow(&oper); @@ -401,30 +402,32 @@ static int32_t mnodeCreateDnode(char *ep) { int dnodeId = pDnode->dnodeId; tfree(pDnode); mError("failed to create dnode:%d, result:%s", dnodeId, tstrerror(code)); - return TSDB_CODE_MND_SDB_ERROR; + } else { + mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code)); + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code)); return code; } -int32_t mnodeDropDnode(SDnodeObj *pDnode) { +int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDnodeSdb, - .pObj = pDnode + .pObj = pDnode, + .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_MND_SDB_ERROR; + int32_t code = sdbDeleteRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code)); + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code)); return code; } -static int32_t mnodeDropDnodeByEp(char *ep) { +static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode == NULL) { mError("dnode:%s, is not exist", ep); @@ -439,7 +442,7 @@ static int32_t mnodeDropDnodeByEp(char *ep) { mPrint("dnode:%d, start to drop it", pDnode->dnodeId); #ifndef _SYNC - return mnodeDropDnode(pDnode); + return mnodeDropDnode(pDnode, pMsg); #else return balanceDropDnode(pDnode); #endif @@ -451,17 +454,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { if (strcmp(pMsg->pUser->user, "root") != 0) { return TSDB_CODE_MND_NO_RIGHTS; } else { - int32_t code = mnodeCreateDnode(pCreate->ep); - - if (code == TSDB_CODE_SUCCESS) { - SDnodeObj *pDnode = mnodeGetDnodeByEp(pCreate->ep); - mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user); - mnodeDecDnodeRef(pDnode); - } else { - mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(code)); - } - - return code; + return mnodeCreateDnode(pCreate->ep, pMsg); } } @@ -471,15 +464,7 @@ static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) { if (strcmp(pMsg->pUser->user, "root") != 0) { return TSDB_CODE_MND_NO_RIGHTS; } else { - int32_t code = mnodeDropDnodeByEp(pDrop->ep); - - if (code == TSDB_CODE_SUCCESS) { - mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user); - } else { - mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(code)); - } - - return code; + return mnodeDropDnodeByEp(pDrop->ep, pMsg); } } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 57587fb65b..10ed4e1da2 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -181,12 +181,12 @@ static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return code; - } else { + if (code == TSDB_CODE_SUCCESS) { mLPrint("user:%s, is altered by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { @@ -237,11 +237,12 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { tfree(pUser); - return code; } else { mLPrint("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { @@ -253,12 +254,12 @@ static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { }; int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return code; - } else { + if (code == TSDB_CODE_SUCCESS) { mLPrint("user:%s, is dropped by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { From d4bb3fd6a3f55558c81de3a68b923446ccc49820 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 13 Jun 2020 08:11:49 +0000 Subject: [PATCH 04/24] [TD-570] add vgroup/table into sdb write queue --- src/client/src/tscSecondaryMerge.c | 2 +- src/client/src/tscServer.c | 2 +- src/client/src/tscStream.c | 2 +- src/mnode/inc/mnodeSdb.h | 2 +- src/mnode/src/mnodeDb.c | 8 +- src/mnode/src/mnodeSdb.c | 6 +- src/mnode/src/mnodeTable.c | 195 +++++++++++++++++------------ src/mnode/src/mnodeVgroup.c | 43 +++++-- src/rpc/src/rpcTcp.c | 6 +- src/util/src/tmem.c | 2 +- src/vnode/src/vnodeMain.c | 2 +- 11 files changed, 158 insertions(+), 112 deletions(-) diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 52a06277e3..b62df73732 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -353,7 +353,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->numOfGroups = 0; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);; + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t revisedSTime = diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5913200ff6..bc3b12d088 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1473,7 +1473,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += len; } - pCmd->payloadLen = pMsg - (char*)pInfoMsg;; + pCmd->payloadLen = pMsg - (char*)pInfoMsg; pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; tfree(tmpData); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index c4413f8541..7ce86dc1a4 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -499,7 +499,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); tscFreeSqlObj(pSql); - return NULL;; + return NULL; } strtolower(pSql->sqlstr, sqlstr); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index df9dbf843e..ca2fffe24c 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -53,7 +53,7 @@ typedef struct { void * rowData; int32_t rowSize; int32_t retCode; // for callback in sdb queue - void (*cb)(struct SMnodeMsg *pMsg, int32_t code); + int32_t (*cb)(struct SMnodeMsg *pMsg, int32_t code); struct SMnodeMsg *pMsg; } SSdbOper; diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 7e0eb8ffc9..7e29ba5bcb 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -366,7 +366,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { - tfree(pDb);; + tfree(pDb); } else { mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -888,8 +888,8 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return newCfg; } -static void mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { - if (code != TSDB_CODE_SUCCESS) return; +static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) return code; SDbObj *pDb = pMsg->pDb; void *pIter = NULL; @@ -908,6 +908,8 @@ static void mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); balanceNotify(); + + return TSDB_CODE_SUCCESS; } static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 380a992473..aff13f46bc 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -962,9 +962,7 @@ static void *sdbWorkerFp(void *param) { } int32_t code = sdbWrite(pOper, pHead, type); - if (pOper && code != TSDB_CODE_SUCCESS) { - pOper->retCode = code; - } + if (pOper) pOper->retCode = code; } walFsync(tsSdbObj.wal); @@ -976,7 +974,7 @@ static void *sdbWorkerFp(void *param) { if (type == TAOS_QTYPE_RPC) { pOper = (SSdbOper *)item; if (pOper->cb) { - (*pOper->cb)(pOper->pMsg, pOper->retCode); + pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode); } dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode); } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 0e255d011b..79079f99e5 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -797,18 +797,20 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .rowSize = sizeof(SSuperTableObj) + schemaSize + .rowSize = sizeof(SSuperTableObj) + schemaSize, + .pMsg = pMsg }; int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { mnodeDestroySuperTable(pStable); mError("table:%s, failed to create, sdb error", pCreate->tableId); - return TSDB_CODE_MND_SDB_ERROR; } else { mLPrint("table:%s, is created, tags:%d fields:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } + + return code; } static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { @@ -840,11 +842,16 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbDeleteRow(&oper); - mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code)); + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code)); + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; } @@ -859,7 +866,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char * return -1; } -static int32_t mnodeAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { +static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { mError("stable:%s, add tag, too many tags", pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; @@ -893,19 +900,20 @@ static int32_t mnodeAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); - return TSDB_CODE_SUCCESS; + + return code; } -static int32_t mnodeDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { +static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *tagName) { int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName); @@ -920,19 +928,20 @@ static int32_t mnodeDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); - return TSDB_CODE_SUCCESS; + + return code; } -static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { +static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); @@ -956,16 +965,17 @@ static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTa SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); - return TSDB_CODE_SUCCESS; + + return code; } static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { @@ -979,7 +989,8 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col return -1; } -static int32_t mnodeAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { +static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { + SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId); return TSDB_CODE_MND_APP_ERROR; @@ -1021,19 +1032,21 @@ static int32_t mnodeAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SS SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("stable %s, succeed to add column", pStable->info.tableId); + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - mPrint("stable %s, succeed to add column", pStable->info.tableId); - return TSDB_CODE_SUCCESS; + + return code; } -static int32_t mnodeDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) { +static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *colName) { + SDbObj *pDb = pMsg->pDb; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName); @@ -1058,16 +1071,17 @@ static int32_t mnodeDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, c SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, - .pObj = pStable + .pObj = pStable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("stable %s, succeed to delete column", pStable->info.tableId); + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mPrint("stable %s, succeed to delete column", pStable->info.tableId); - return TSDB_CODE_SUCCESS; + return code; } // show super tables @@ -1428,7 +1442,30 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO return pCreate; } -static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { +static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) return code; + + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable); + if (pMDCreate == NULL) { + return terrno; + } + + SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcMsg rpcMsg = { + .handle = pMsg, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + + dnodeSendMsgToDnode(&ipSet, &rpcMsg); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, SVgObj *pVgroup, int32_t tid) { + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); @@ -1503,16 +1540,20 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb desc.type = SDB_OPER_GLOBAL; desc.pObj = pTable; desc.table = tsChildTableSdb; + desc.pMsg = pMsg; + desc.cb = mnodeDoCreateChildTableCb; - if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) { + int32_t code = sdbInsertRow(&desc); + if (code != TSDB_CODE_SUCCESS) { free(pTable); mError("table:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_MND_SDB_ERROR; return NULL; + } else { + mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pTable->info.tableId, pVgroup->vgId, pTable->sid, + pTable->uid); + return pTable; } - - mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64 , pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid); - return pTable; } static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { @@ -1537,7 +1578,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return mnodeCreateVgroup(pMsg, pMsg->pDb); } - pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pCreate, pVgroup, sid); + pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, pVgroup, sid); if (pMsg->pTable == NULL) { return terrno; } @@ -1550,25 +1591,9 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) { return terrno; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - - SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable); - if (pMDCreate == NULL) { - return terrno; - } - - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = { - .handle = pMsg, - .pCont = pMDCreate, - .contLen = htonl(pMDCreate->contLen), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE - }; - - dnodeSendMsgToDnode(&ipSet, &rpcMsg); - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { @@ -1622,7 +1647,8 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col return -1; } -static int32_t mnodeAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { +static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { + SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId); return TSDB_CODE_MND_APP_ERROR; @@ -1657,19 +1683,21 @@ static int32_t mnodeAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SS SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, - .pObj = pTable + .pObj = pTable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + if (code == TSDB_CODE_SUCCESS) { + mPrint("table %s, succeed to add column", pTable->info.tableId); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mPrint("table %s, succeed to add column", pTable->info.tableId); - return TSDB_CODE_SUCCESS; + return code; } -static int32_t mnodeDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) { +static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable, char *colName) { + SDbObj *pDb = pMsg->pDb; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName); @@ -1689,16 +1717,17 @@ static int32_t mnodeDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, c SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, - .pObj = pTable + .pObj = pTable, + .pMsg = pMsg }; int32_t code = sdbUpdateRow(&oper); if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_MND_SDB_ERROR; + mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); - return TSDB_CODE_SUCCESS; + return code; } static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { @@ -2237,32 +2266,32 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; mTrace("table:%s, start to alter stable", pAlter->tableId); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - code = mnodeAddSuperTableTag(pTable, pAlter->schema, 1); + code = mnodeAddSuperTableTag(pMsg, pTable, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - code = mnodeDropSuperTableTag(pTable, pAlter->schema[0].name); + code = mnodeDropSuperTableTag(pMsg, pTable, pAlter->schema[0].name); } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - code = mnodeModifySuperTableTagName(pTable, pAlter->schema[0].name, pAlter->schema[1].name); + code = mnodeModifySuperTableTagName(pMsg, pTable, pAlter->schema[0].name, pAlter->schema[1].name); } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - code = mnodeAddSuperTableColumn(pMsg->pDb, pTable, pAlter->schema, 1); + code = mnodeAddSuperTableColumn(pMsg, pTable, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - code = mnodeDropSuperTableColumn(pMsg->pDb, pTable, pAlter->schema[0].name); + code = mnodeDropSuperTableColumn(pMsg, pTable, pAlter->schema[0].name); } else { } } else { mTrace("table:%s, start to alter ctable", pAlter->tableId); SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - char *tagVal = (char*)(pAlter->schema + pAlter->numOfCols); + char *tagVal = (char *)(pAlter->schema + pAlter->numOfCols); code = mnodeModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal); } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - code = mnodeAddNormalTableColumn(pMsg->pDb, pTable, pAlter->schema, 1); + code = mnodeAddNormalTableColumn(pMsg, pTable, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - code = mnodeDropNormalTableColumn(pMsg->pDb, pTable, pAlter->schema[0].name); + code = mnodeDropNormalTableColumn(pMsg, pTable, pAlter->schema[0].name); } else { } } - return code; + return code; } static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 25b9daf00c..cdb494e298 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -299,6 +299,27 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); } +static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + pMsg->pVgroup = NULL; + return code; + } + + SVgObj *pVgroup = pMsg->pVgroup; + SDbObj *pDb = pMsg->pDb; + + mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId); + } + + mnodeIncVgroupRef(pVgroup); + pMsg->expected = pVgroup->numOfVnodes; + mnodeSendCreateVgroupMsg(pVgroup, pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); @@ -314,26 +335,22 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, - .rowSize = sizeof(SVgObj) + .rowSize = sizeof(SVgObj), + .pMsg = pMsg, + .cb = mnodeCreateVgroupCb }; + pMsg->pVgroup = pVgroup; + int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { + pMsg->pVgroup = NULL; tfree(pVgroup); - return TSDB_CODE_MND_SDB_ERROR; + } else { + if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId); - } - - mnodeIncVgroupRef(pVgroup); - pMsg->pVgroup = pVgroup; - pMsg->expected = pVgroup->numOfVnodes; - mnodeSendCreateVgroupMsg(pVgroup, pMsg); - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 7035b30b66..0ca73c1b40 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -117,7 +117,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread code = pthread_mutex_init(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); - break;; + break; } pThreadObj->pollFd = epoll_create(10); // size does not matter @@ -367,7 +367,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { recvInfo.ip = 0; recvInfo.port = 0; recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; + recvInfo.thandle = pFdObj->thandle; recvInfo.chandle = NULL; recvInfo.connType = RPC_CONN_TCP; (*(pThreadObj->processData))(&recvInfo); @@ -414,7 +414,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->ip = pFdObj->ip; pInfo->port = pFdObj->port; pInfo->shandle = pThreadObj->shandle; - pInfo->thandle = pFdObj->thandle;; + pInfo->thandle = pFdObj->thandle; pInfo->chandle = pFdObj; pInfo->connType = RPC_CONN_TCP; diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index 38d930ac4e..ec5f90990b 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -29,7 +29,7 @@ static FILE* fpAllocLog = NULL; // memory allocator which fails randomly extern int32_t taosGetTimestampSec(); -static int32_t startTime = INT32_MAX;; +static int32_t startTime = INT32_MAX; static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { if (taosGetTimestampSec() < startTime) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 67ba891f93..6c7e475b64 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -119,7 +119,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; - tsdbCfg.compression = pVnodeCfg->cfg.compression;; + tsdbCfg.compression = pVnodeCfg->cfg.compression; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); From 5f14bab1f057b92d28b186b479f7e121d9cfc04d Mon Sep 17 00:00:00 2001 From: dengyihao Date: Sun, 14 Jun 2020 07:52:05 +0800 Subject: [PATCH 05/24] fixbug revealed by coverity --- src/query/src/qExecutor.c | 10 ++++++---- src/query/src/tvariant.c | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3e0ff479e9..d9c725a4f5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3335,7 +3335,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL); + //assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL); setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb); @@ -4230,7 +4230,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); qTrace("QInfo:%p last_row query on group:%d, total group:%d, current group:%d", pQInfo, pQInfo->groupIndex, - numOfGroups); + numOfGroups, group); STsdbQueryCond cond = { .twindow = pQuery->window, @@ -4264,10 +4264,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb); - taosArrayDestroy(s); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); } + taosArrayDestroy(s); // here we simply set the first table as current table pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info; @@ -5380,6 +5380,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) && (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { + assert(rangeFilterArray != NULL); if (lower == TSDB_RELATION_GREATER_EQUAL) { if (upper == TSDB_RELATION_LESS_EQUAL) { pSingleColFilter->fp = rangeFilterArray[4]; @@ -5394,11 +5395,12 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } } } else { // set callback filter function + assert(filterArray != NULL); if (lower != TSDB_RELATION_INVALID) { pSingleColFilter->fp = filterArray[lower]; if (upper != TSDB_RELATION_INVALID) { - qError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type); + qError("pQInfo:%p failed to get filter function, invalid filter condition: %d", pQInfo, type); return TSDB_CODE_QRY_INVALID_MSG; } } else { diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index 32d06225c5..7ba4e7899e 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -887,7 +887,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { free(pVariant->pz); pVariant->dKey = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - pVariant->dKey = pVariant->i64Key; + pVariant->dKey = (double)(pVariant->i64Key); } pVariant->nType = TSDB_DATA_TYPE_DOUBLE; From d0f955f98b1d58676b62d94965963afb12e81fe3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 15 Jun 2020 10:20:24 +0000 Subject: [PATCH 06/24] [TD-570] un reference vgroup while create table --- src/mnode/src/mnodeTable.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 79079f99e5..22917f559c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1464,7 +1464,8 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, SVgObj *pVgroup, int32_t tid) { +static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { + SVgObj *pVgroup = pMsg->pVgroup; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { @@ -1578,7 +1579,9 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return mnodeCreateVgroup(pMsg, pMsg->pDb); } - pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, pVgroup, sid); + pMsg->pVgroup = pVgroup; + mnodeIncVgroupRef(pVgroup); + pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, sid); if (pMsg->pTable == NULL) { return terrno; } From edfb1859a1a98d134efae9db04829b017730a3d9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Jun 2020 01:48:42 +0000 Subject: [PATCH 07/24] [TD-570] lost while create vgroup --- src/mnode/src/mnodeTable.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 9c9efebc32..bffb002cbd 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1581,8 +1581,11 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return mnodeCreateVgroup(pMsg, pMsg->pDb); } - pMsg->pVgroup = pVgroup; - mnodeIncVgroupRef(pVgroup); + if (pMsg->pVgroup == NULL) { + pMsg->pVgroup = pVgroup; + mnodeIncVgroupRef(pVgroup); + } + pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, sid); if (pMsg->pTable == NULL) { return terrno; From 99ebc7e07357eeefa769be5efb23a30aa1283998 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Jun 2020 02:59:43 +0000 Subject: [PATCH 08/24] [TD-570] wrong errocde while drop stable --- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index aff13f46bc..86df1f5303 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -503,7 +503,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 "version:%" PRIu64, pTable->tableName, + sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index bffb002cbd..6e72285c5a 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -849,7 +849,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { }; int32_t code = sdbDeleteRow(&oper); - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code == TSDB_CODE_SUCCESS) { mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code)); if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1585,7 +1585,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { pMsg->pVgroup = pVgroup; mnodeIncVgroupRef(pVgroup); } - + pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, sid); if (pMsg->pTable == NULL) { return terrno; From 15f0e77ca5b2ebcd269ae33881272b82e6853375 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Jun 2020 16:39:12 +0800 Subject: [PATCH 09/24] [TD-570] fix wrong response handle when create child tables --- src/dnode/src/dnodeMWrite.c | 22 ++++-- src/mnode/src/mnodeSdb.c | 28 +++++++- src/mnode/src/mnodeTable.c | 92 ++++++++++++++----------- src/mnode/src/mnodeWrite.c | 16 +++-- tests/script/jenkins/basic.txt | 48 +++++++------- tests/test/c/insertPerRow.c | 118 +++++++++++++++++++++------------ tests/test/c/insertPerTable.c | 50 ++++++++++---- 7 files changed, 241 insertions(+), 133 deletions(-) diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 41714a0df7..e3a228919f 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -118,6 +118,8 @@ void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) { SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); mnodeCreateMsg(pWrite, pMsg); + + dTrace("app:%p:%p, msg:%s is put into mwrite queue", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType]); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } @@ -147,19 +149,21 @@ void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) { } static void *dnodeProcessMnodeWriteQueue(void *param) { - SMnodeMsg *pWriteMsg; + SMnodeMsg *pWrite; int32_t type; void * unUsed; while (1) { - if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWriteMsg, &unUsed) == 0) { + if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) { dTrace("dnodeProcessMnodeWriteQueue: got no message from qset, exiting..."); break; } - dTrace("%p, msg:%s will be processed in mwrite queue", pWriteMsg->rpcMsg.ahandle, taosMsg[pWriteMsg->rpcMsg.msgType]); - int32_t code = mnodeProcessWrite(pWriteMsg); - dnodeSendRpcMnodeWriteRsp(pWriteMsg, code); + dTrace("app:%p:%p, msg:%s will be processed in mwrite queue", pWrite->rpcMsg.ahandle, pWrite, + taosMsg[pWrite->rpcMsg.msgType]); + + int32_t code = mnodeProcessWrite(pWrite); + dnodeSendRpcMnodeWriteRsp(pWrite, code); } return NULL; @@ -169,9 +173,15 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; if (!mnodeIsRunning() || tsMWriteQueue == NULL) { + dTrace("app:%p:%p, msg:%s is redirected for mnode not running, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, + taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); + dnodeSendRedirectMsg(pMsg, true); dnodeFreeMnodeWriteMsg(pWrite); - } else { + } else { + dTrace("app:%p:%p, msg:%s is reput into mwrite queue, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, + taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 86df1f5303..efcadccb1c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -641,6 +641,11 @@ int32_t sdbInsertRow(SSdbOper *pOper) { pHead->len = pOper->rowSize; memcpy(pNewOper, pOper, sizeof(SSdbOper)); + + if (pNewOper->pMsg != NULL) { + sdbTrace("app:%p:%p, insert action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + } + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_SUCCESS; } @@ -687,7 +692,6 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { return TSDB_CODE_MND_SDB_INVAID_KEY_TYPE; } - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize; SSdbOper *pNewOper = taosAllocateQitem(size); @@ -698,6 +702,11 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { memcpy(pHead->cont, key, keySize); memcpy(pNewOper, pOper, sizeof(SSdbOper)); + + if (pNewOper->pMsg != NULL) { + sdbTrace("app:%p:%p, delete action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + } + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_SUCCESS; } @@ -740,7 +749,12 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { (*pTable->encodeFp)(pOper); pHead->len = pOper->rowSize; - memcpy(pNewOper, pOper, sizeof(SSdbOper)); + memcpy(pNewOper, pOper, sizeof(SSdbOper)); + + if (pNewOper->pMsg != NULL) { + sdbTrace("app:%p:%p, update action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + } + taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_SUCCESS; } @@ -961,6 +975,10 @@ static void *sdbWorkerFp(void *param) { pOper = NULL; } + if (pOper != NULL && pOper->pMsg != NULL) { + sdbTrace("app:%p:%p, will be processed in sdb queue", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg); + } + int32_t code = sdbWrite(pOper, pHead, type); if (pOper) pOper->retCode = code; } @@ -976,6 +994,12 @@ static void *sdbWorkerFp(void *param) { if (pOper->cb) { pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode); } + + if (pOper != NULL && pOper->pMsg != NULL) { + sdbTrace("app:%p:%p, msg is processed, result:%s", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, + tstrerror(pOper->retCode)); + } + dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode); } taosFreeQitem(item); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 6e72285c5a..7f2f1f3faf 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -688,10 +688,12 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { } if (pCreate->numOfTags != 0) { - mTrace("table:%s, create stable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle); + mTrace("app:%p:%p, table:%s, create stable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { - mTrace("table:%s, create ctable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle); + mTrace("app:%p:%p, table:%s, create ctable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } } @@ -1466,14 +1468,13 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { +static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SVgObj *pVgroup = pMsg->pVgroup; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); - terrno = TSDB_CODE_MND_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_MND_OUT_OF_MEMORY; } if (pCreate->numOfColumns == 0) { @@ -1493,8 +1494,7 @@ static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { if (pSuperTable == NULL) { mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name); mnodeDestroyChildTable(pTable); - terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; - return NULL; + return TSDB_CODE_MND_INVALID_TABLE_NAME; } mnodeDecTableRef(pSuperTable); @@ -1513,8 +1513,7 @@ static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { pTable->schema = (SSchema *) calloc(1, schemaSize); if (pTable->schema == NULL) { free(pTable); - terrno = TSDB_CODE_MND_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_MND_OUT_OF_MEMORY; } memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); @@ -1530,15 +1529,17 @@ static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { pTable->sql = calloc(1, pTable->sqlLen); if (pTable->sql == NULL) { free(pTable); - terrno = TSDB_CODE_MND_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_MND_OUT_OF_MEMORY; } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; mTrace("table:%s, stream sql len:%d sql:%s", pTable->info.tableId, pTable->sqlLen, pTable->sql); } } - + + pMsg->pTable = (STableObj *)pTable; + mnodeIncTableRef(pMsg->pTable); + SSdbOper desc = {0}; desc.type = SDB_OPER_GLOBAL; desc.pObj = pTable; @@ -1550,12 +1551,12 @@ static SChildTableObj* mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { if (code != TSDB_CODE_SUCCESS) { free(pTable); mError("table:%s, update sdb error", pCreate->tableId); - terrno = TSDB_CODE_MND_SDB_ERROR; - return NULL; + pMsg->pTable = NULL; + return code; } else { mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid); - return pTable; + return TSDB_CODE_SUCCESS; } } @@ -1563,13 +1564,14 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to create, grant timeseries failed", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, grant timeseries failed", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId); return code; } SVgObj *pVgroup = mnodeGetAvailableVgroup(pMsg->pDb); if (pVgroup == NULL) { - mTrace("table:%s, start to create a new vgroup", pCreate->tableId); + mTrace("app:%p:%p, table:%s, start to create a new vgroup", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return mnodeCreateVgroup(pMsg, pMsg->pDb); } @@ -1577,7 +1579,8 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid <= 0) { - mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId); + mTrace("app:%p:%p, table:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + pVgroup->vgId); return mnodeCreateVgroup(pMsg, pMsg->pDb); } @@ -1586,21 +1589,27 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { mnodeIncVgroupRef(pVgroup); } - pMsg->pTable = (STableObj *)mnodeDoCreateChildTable(pMsg, sid); - if (pMsg->pTable == NULL) { - return terrno; - } + mTrace("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + pVgroup->vgId, sid); - mnodeIncTableRef(pMsg->pTable); + code = mnodeDoCreateChildTable(pMsg, sid); + if (code != TSDB_CODE_SUCCESS) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } } } else { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); } if (pMsg->pTable == NULL) { + mError("app:%p:%p, table:%s, object not found, retry:%d reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + tstrerror(terrno)); return terrno; } else { - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mTrace("app:%p:%p, table:%s, send create msg to vnode again", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + return mnodeDoCreateChildTableCb(pMsg, TSDB_CODE_SUCCESS); } } @@ -2007,8 +2016,10 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } -// handle create table response from dnode -// if failed, drop the table cached +/* + * handle create table response from dnode + * if failed, drop the table cached + */ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->handle == NULL) return; @@ -2017,18 +2028,18 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); - mTrace("table:%s, create table rsp received, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle, - tstrerror(rpcMsg->code)); - - if (rpcMsg->code != TSDB_CODE_SUCCESS) { + + if (rpcMsg->code != TSDB_CODE_SUCCESS && rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { if (mnodeMsg->retry++ < 10) { - mTrace("table:%s, create table rsp received, retry:%d thandle:%p result:%s", pTable->info.tableId, - mnodeMsg->retry, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + mTrace("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p", + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry,tstrerror(rpcMsg->code), + mnodeMsg->rpcMsg.handle); + dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); } else { - mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId, - mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); - + mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, @@ -2037,16 +2048,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { sdbDeleteRow(&oper); dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); - } + } } else { - mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle, - tstrerror(rpcMsg->code)); SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont; if (pCreate->getMeta) { - mTrace("table:%s, continue to get meta", pTable->info.tableId); + mTrace("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p", mnodeMsg->rpcMsg.ahandle, + mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle); + mnodeMsg->retry = 0; dnodeReprocessMnodeWriteMsg(mnodeMsg); } else { + mTrace("app:%p:%p, table:%s, created in dnode, thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + pTable->info.tableId, mnodeMsg->rpcMsg.handle); + dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); } } diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 29b2e6c82b..cd3d722480 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -43,7 +43,7 @@ void mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg)) int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (pMsg->rpcMsg.pCont == NULL) { - mError("%p, msg:%s in mwrite queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("app:%p:%p, msg:%s content is null", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_INVALID_MSG_LEN; } @@ -54,27 +54,31 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { rpcRsp->rsp = ipSet; rpcRsp->len = sizeof(SRpcIpSet); - mTrace("%p, msg:%s in mwrite queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); + mTrace("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], + ipSet->inUse); for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mTrace("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); + mTrace("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i], + htons(ipSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; } if (tsMnodeProcessWriteMsgFp[pMsg->rpcMsg.msgType] == NULL) { - mError("%p, msg:%s in mwrite queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("app:%p:%p, msg:%s not processed", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_MSG_NOT_PROCESSED; } int32_t code = mnodeInitMsg(pMsg); if (code != TSDB_CODE_SUCCESS) { - mError("%p, msg:%s in mwrite queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code)); + mError("app:%p:%p, msg:%s not processed, reason:%s", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], + tstrerror(code)); return code; } if (!pMsg->pUser->writeAuth) { - mError("%p, msg:%s in mwrite queue, not processed, no write auth", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("app:%p:%p, msg:%s not processed, no write auth", pMsg->rpcMsg.ahandle, pMsg, + taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_NO_RIGHTS; } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index fe91e4bbde..eb330a27a1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -333,28 +333,28 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim +#./test.sh -f unique/arbitrator/insert_duplicationTs.sim +#./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim +#./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim +#./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim +#./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim +#./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim +#./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim +#./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim +#./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim +#./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim +#./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim +#./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim +#./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim +#./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim +#./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim +#./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim +#./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim +#./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim +#./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim +#./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index 6b5a678093..906956b998 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -33,6 +33,7 @@ typedef struct { int threadIndex; char dbName[32]; char stableName[64]; + float createTableSpeed; pthread_t thread; } SInfo; @@ -49,8 +50,8 @@ int64_t numOfThreads = 1; int64_t numOfTablesPerThread = 200; char dbName[32] = "db"; char stableName[64] = "st"; -int32_t cache = 16384; -int32_t tables = 1000; +int32_t cache = 16; +int32_t tables = 5000; int main(int argc, char *argv[]) { shellParseArgument(argc, argv); @@ -63,9 +64,8 @@ int main(int argc, char *argv[]) { void createDbAndTable() { pPrint("start to create table"); + TAOS_RES * pSql; TAOS * con; - struct timeval systemTime; - int64_t st, et; char qstr[64000]; char fqdn[TSDB_FQDN_LEN]; @@ -77,22 +77,24 @@ void createDbAndTable() { exit(1); } - sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); - if (taos_query(con, qstr)) { - pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); + sprintf(qstr, "create database if not exists %s cache %d maxtables %d", dbName, cache, tables); + pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); exit(0); } + taos_free_result(pSql); sprintf(qstr, "use %s", dbName); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } + taos_free_result(pSql); - gettimeofday(&systemTime, NULL); - st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - int64_t totalTables = numOfTablesPerThread * numOfThreads; - if (strcmp(stableName, "no") != 0) { int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName); for (int64_t f = 0; f < pointsPerTable; ++f) { @@ -100,36 +102,14 @@ void createDbAndTable() { } sprintf(qstr + len, ") tags(t int)"); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } - - for (int64_t t = 0; t < totalTables; ++t) { - sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); - if (taos_query(con, qstr)) { - pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); - exit(0); - } - } - } else { - for (int64_t t = 0; t < totalTables; ++t) { - int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t); - for (int64_t f = 0; f < pointsPerTable; ++f) { - len += sprintf(qstr + len, ", f%ld double", f); - } - sprintf(qstr + len, ")"); - - if (taos_query(con, qstr)) { - pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); - exit(0); - } - } + taos_free_result(pSql); } - - gettimeofday(&systemTime, NULL); - et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - pPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables); } void insertData() { @@ -144,7 +124,7 @@ void insertData() { pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + SInfo *pInfo = (SInfo *)calloc(numOfThreads, sizeof(SInfo)); // Start threads to write for (int i = 0; i < numOfThreads; ++i) { @@ -173,10 +153,15 @@ void insertData() { double speedOfRows = totalRows / seconds; double speedOfPoints = totalPoints / seconds; + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + pPrint( "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf " - "PointsPerSecond:%.1lf%s", - GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC); + "PointsPerSecond:%.1lf CreateTableSpeed:%.1f t/s %s", + GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, createTableSpeed, NC); pPrint("threads exit"); @@ -191,6 +176,7 @@ void *syncTest(void *param) { int64_t st, et; char qstr[65000]; int maxBytes = 60000; + int code; pPrint("thread:%d, start to run", pInfo->threadIndex); @@ -210,6 +196,48 @@ void *syncTest(void *param) { gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + if (strcmp(stableName, "no") != 0) { + for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + } + } else { + for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ")"); + + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + } + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float seconds = (et - st) / 1000.0 / 1000.0; + int64_t tables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + pInfo->createTableSpeed = (float)tables / seconds; + pPrint("thread:%d, %.1f seconds to create %ld tables, speed:%.1f", pInfo->threadIndex, seconds, tables, + pInfo->createTableSpeed); + + if (pInfo->rowsPerTable == 0) return NULL; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t start = 1430000000000; int64_t interval = 1000; // 1000 ms @@ -227,10 +255,13 @@ void *syncTest(void *param) { } len += sprintf(sql + len, ")"); if (len > maxBytes) { - if (taos_query(con, qstr)) { + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { pError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, table, row, taos_errstr(con)); } + taos_free_result(pSql); // "insert into" len = sprintf(sql, "%s", inserStr); @@ -239,7 +270,8 @@ void *syncTest(void *param) { } if (len != strlen(inserStr)) { - taos_query(con, qstr); + TAOS_RES *pSql = taos_query(con, qstr); + taos_free_result(pSql); } gettimeofday(&systemTime, NULL); diff --git a/tests/test/c/insertPerTable.c b/tests/test/c/insertPerTable.c index a439abec88..af92927a9e 100644 --- a/tests/test/c/insertPerTable.c +++ b/tests/test/c/insertPerTable.c @@ -49,8 +49,8 @@ int64_t numOfThreads = 1; int64_t numOfTablesPerThread = 1; char dbName[32] = "db"; char stableName[64] = "st"; -int32_t cache = 16384; -int32_t tables = 1000; +int32_t cache = 16; +int32_t tables = 5000; int main(int argc, char *argv[]) { shellParseArgument(argc, argv); @@ -63,6 +63,7 @@ int main(int argc, char *argv[]) { void createDbAndTable() { pPrint("start to create table"); + TAOS_RES * pSql; TAOS * con; struct timeval systemTime; int64_t st, et; @@ -79,17 +80,22 @@ void createDbAndTable() { exit(1); } - sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); - if (taos_query(con, qstr)) { - pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); + sprintf(qstr, "create database if not exists %s cache %d maxtables %d", dbName, cache, tables); + pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); exit(0); } sprintf(qstr, "use %s", dbName); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } + taos_stop_query(pSql); gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; @@ -102,17 +108,23 @@ void createDbAndTable() { } sprintf(qstr + len, ") tags(t int)"); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); exit(0); } - + taos_stop_query(pSql); + for (int64_t t = 0; t < totalTables; ++t) { sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); exit(0); } + taos_stop_query(pSql); } } else { for (int64_t t = 0; t < totalTables; ++t) { @@ -122,16 +134,20 @@ void createDbAndTable() { } sprintf(qstr + len, ")"); - if (taos_query(con, qstr)) { + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); exit(0); } + taos_stop_query(pSql); } } gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - pPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables); + float seconds = (et - st) / 1000.0 / 1000.0; + pPrint("%.1f seconds to create %ld tables, speed:%.1f", seconds, totalTables, totalTables / seconds); } void insertData() { @@ -141,7 +157,12 @@ void insertData() { gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - pPrint("%d threads are spawned to insert data", numOfThreads); + if (rowsPerTable <= 0) { + pPrint("not insert data for rowsPerTable is :%d", rowsPerTable); + exit(0); + } else { + pPrint("%d threads are spawned to insert data", numOfThreads); + } pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -230,10 +251,13 @@ void *syncTest(void *param) { } len += sprintf(sql + len, ")"); if (len > maxBytes) { - if (taos_query(con, qstr)) { + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { pError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, table, row, taos_errstr(con)); } + taos_stop_query(pSql); // "insert into" len = sprintf(sql, "%s", inserStr); From 1da2179942f60fb36d243dbd64c49e6ed7644a5f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Jun 2020 17:51:04 +0800 Subject: [PATCH 10/24] [TD-570] update logs --- src/mnode/inc/mnodeInt.h | 8 +- src/mnode/src/mnodeSdb.c | 6 +- src/mnode/src/mnodeTable.c | 213 ++++++++++++++++++++++--------------- 3 files changed, 134 insertions(+), 93 deletions(-) diff --git a/src/mnode/inc/mnodeInt.h b/src/mnode/inc/mnodeInt.h index 7405cef6f3..708f11d1f7 100644 --- a/src/mnode/inc/mnodeInt.h +++ b/src/mnode/inc/mnodeInt.h @@ -36,10 +36,10 @@ extern int32_t sdbDebugFlag; #define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) } #define mLPrint(...) { monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) } -#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); }} -#define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); }} -#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__);}} -#define sdbPrint(...) { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); } +#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR SDB ", 255, __VA_ARGS__); }} +#define sdbWarn(...) { if (sdbDebugFlag & DEBUG_WARN) { taosPrintLog("WARN SDB ", sdbDebugFlag, __VA_ARGS__); }} +#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__);}} +#define sdbPrint(...) { taosPrintLog("SDB ", 255, __VA_ARGS__); } #define sdbLError(...) { monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__) } #define sdbLWarn(...) { monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__) } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index efcadccb1c..1d428ab9a2 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -643,7 +643,7 @@ int32_t sdbInsertRow(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbTrace("app:%p:%p, insert action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + sdbTrace("app:%p:%p, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); } taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); @@ -704,7 +704,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbTrace("app:%p:%p, delete action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + sdbTrace("app:%p:%p, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); } taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); @@ -752,7 +752,7 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbTrace("app:%p:%p, update action is add to write queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); + sdbTrace("app:%p:%p, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg); } taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7f2f1f3faf..309a387195 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -669,20 +669,21 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { - mError("table:%s, failed to create, db not selected", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pCreate->getMeta) { - mTrace("table:%s, continue to get meta", pCreate->tableId); + mTrace("app:%p:%p, table:%s, continue to get meta", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return mnodeGetChildTableMeta(pMsg); } else if (pCreate->igExists) { - mTrace("table:%s, is already exist", pCreate->tableId); + mTrace("app:%p:%p, table:%s, is already exist", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_SUCCESS; } else { - mError("table:%s, failed to create, table already exist", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, table already exist", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } @@ -702,31 +703,32 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { - mError("table:%s, failed to drop table, db not selected", pDrop->tableId); + mError("app:%p:%p, table:%s, failed to drop table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); + mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg, + pDrop->tableId); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mTrace("table:%s, table is not exist, think drop success", pDrop->tableId); + mTrace("app:%p:%p, table:%s, table is not exist, think drop success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return TSDB_CODE_SUCCESS; } else { - mError("table:%s, failed to drop table, table not exist", pDrop->tableId); + mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - mPrint("table:%s, start to drop stable", pDrop->tableId); + mPrint("app:%p:%p, table:%s, start to drop stable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return mnodeProcessDropSuperTableMsg(pMsg); } else { - mPrint("table:%s, start to drop ctable", pDrop->tableId); + mPrint("app:%p:%p, table:%s, start to drop ctable", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return mnodeProcessDropChildTableMsg(pMsg); } } @@ -734,21 +736,25 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); - mTrace("table:%s, table meta msg is received from thandle:%p, createFlag:%d", pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); + mTrace("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg, + pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { - mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); + mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg, + pInfo->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId); if (pMsg->pTable == NULL) { if (!pInfo->createFlag) { - mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); + mError("app:%p:%p, table:%s, failed to get table meta, table not exist", pMsg->rpcMsg.ahandle, pMsg, + pInfo->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - mTrace("table:%s, failed to get table meta, start auto create table ", pInfo->tableId); + mTrace("app:%p:%p, table:%s, failed to get table meta, start auto create table ", pMsg->rpcMsg.ahandle, pMsg, + pInfo->tableId); return mnodeAutoCreateChildTable(pMsg); } } else { @@ -762,9 +768,9 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - SSuperTableObj *pStable = calloc(1, sizeof(SSuperTableObj)); + SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { - mError("table:%s, failed to create, no enough memory", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -782,7 +788,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { free(pStable); - mError("table:%s, failed to create, no schema input", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, no schema input", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); @@ -808,9 +814,10 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { mnodeDestroySuperTable(pStable); - mError("table:%s, failed to create, sdb error", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); } else { - mLPrint("table:%s, is created, tags:%d fields:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); + mLPrint("app:%p:%p, table:%s, is created, tags:%d fields:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + pStable->numOfTags, pStable->numOfColumns); if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -831,10 +838,11 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); - - mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, pVgroup->vgId); + + mPrint("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + pVgroup->vgId); SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&ipSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } @@ -872,18 +880,20 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char * static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { - mError("stable:%s, add tag, too many tags", pStable->info.tableId); + mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; } for (int32_t i = 0; i < ntags; i++) { if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name); + mError("app:%p:%p, stable:%s, add tag, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_TAG_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { - mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name); + mError("app:%p:%p, stable:%s, add tag, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } } @@ -910,17 +920,19 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, S int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); + mPrint("app:%p:%p, stable %s, succeed to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + schema[0].name); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - + return code; } static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *tagName) { int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { - mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName); + mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tagName); return TSDB_CODE_MND_TAG_NOT_EXIST; } @@ -938,17 +950,19 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); + mPrint("app:%p:%p, stable %s, succeed to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } return code; } -static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *oldTagName, char *newTagName) { +static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *oldTagName, + char *newTagName) { int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { - mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); + mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_MND_TAG_NOT_EXIST; } @@ -975,7 +989,8 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pSt int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); + mPrint("app:%p:%p, stable %s, succeed to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + oldTagName, newTagName); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -996,18 +1011,20 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { - mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId); + mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name); + mError("app:%p:%p, stable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { - mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name); + mError("app:%p:%p, stable:%s, add column, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_TAG_ALREAY_EXIST; } } @@ -1042,7 +1059,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("stable %s, succeed to add column", pStable->info.tableId); + mPrint("app:%p:%p, stable %s, succeed to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1053,7 +1070,8 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStabl SDbObj *pDb = pMsg->pDb; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { - mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName); + mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + colName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -1081,10 +1099,10 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStabl int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("stable %s, succeed to delete column", pStable->info.tableId); + mPrint("app:%p:%p, stable %s, succeed to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } - + return code; } @@ -1274,8 +1292,9 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMeta->contLen = htons(pMeta->contLen); pMsg->rpcRsp.rsp = pMeta; - - mTrace("stable:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); + + mTrace("app:%p:%p, stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pTable->uid); return TSDB_CODE_SUCCESS; } @@ -1306,12 +1325,13 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN)*i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { - mError("stable:%s, not exist while get stable vgroup info", stableName); + mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); mnodeDecTableRef(pTable); continue; } if (pTable->vgHash == NULL) { - mError("stable:%s, not vgroup exist while get stable vgroup info", stableName); + mError("app:%p:%p, stable:%s, not vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, + stableName); mnodeDecTableRef(pTable); // even this super table has no corresponding table, still return @@ -1473,7 +1493,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { - mError("table:%s, failed to alloc memory", pCreate->tableId); + mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1492,7 +1512,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { STagData *pTagData = (STagData *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mnodeGetSuperTable(pTagData->name); if (pSuperTable == NULL) { - mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name); + mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg, + pCreate->tableId, pTagData->name); mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -1533,7 +1554,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mTrace("table:%s, stream sql len:%d sql:%s", pTable->info.tableId, pTable->sqlLen, pTable->sql); + mTrace("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + pTable->sqlLen, pTable->sql); } } @@ -1550,12 +1572,13 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { int32_t code = sdbInsertRow(&desc); if (code != TSDB_CODE_SUCCESS) { free(pTable); - mError("table:%s, update sdb error", pCreate->tableId); + mError("app:%p:%p, table:%s, update sdb error, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + tstrerror(code)); pMsg->pTable = NULL; return code; } else { - mTrace("table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pTable->info.tableId, pVgroup->vgId, pTable->sid, - pTable->uid); + mTrace("app:%p:%p, table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid); return TSDB_CODE_SUCCESS; } } @@ -1617,13 +1640,15 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { - mError("table:%s, failed to drop ctable, vgroup not exist", pTable->info.tableId); + mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId); return TSDB_CODE_MND_APP_ERROR; } SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("table:%s, failed to drop ctable, no enough memory", pTable->info.tableId); + mError("app:%p:%p, table:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1635,7 +1660,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); - mPrint("table:%s, send drop ctable msg", pDrop->tableId); + mPrint("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); SRpcMsg rpcMsg = { .handle = pMsg, .pCont = pDrop, @@ -1667,13 +1692,14 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { - mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId); + mError("app:%p:%p, table:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name); + mError("app:%p:%p, table:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } } @@ -1706,10 +1732,10 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("table %s, succeed to add column", pTable->info.tableId); + mPrint("app:%p:%p, table %s, succeed to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - + return code; } @@ -1717,7 +1743,8 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTabl SDbObj *pDb = pMsg->pDb; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { - mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName); + mError("app:%p:%p, table:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + colName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -1740,10 +1767,10 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTabl int32_t code = sdbUpdateRow(&oper); if (code != TSDB_CODE_SUCCESS) { - mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); + mPrint("app:%p:%p, table %s, succeed to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } - + return code; } @@ -1783,10 +1810,11 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable); } - + if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { - mError("table:%s, failed to get table meta, vgroup not exist", pTable->info.tableId); + mError("app:%p:%p, table:%s, failed to get table meta, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -1800,7 +1828,8 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { } pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); - mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); + mTrace("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pTable->uid); return TSDB_CODE_SUCCESS; } @@ -1812,7 +1841,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen); SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); if (pCreateMsg == NULL) { - mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); + mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle, + pMsg, pInfo->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1824,7 +1854,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { pCreateMsg->contLen = htonl(contLen); memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg)); - mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, ((STagData *)(pCreateMsg->schema))->name); + mTrace("app:%p:%p, table:%s, start to create on demand, stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, + ((STagData *)(pCreateMsg->schema))->name); rpcFreeCont(pMsg->rpcMsg.pCont); pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE; @@ -1835,9 +1866,11 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { } static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) { - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); + STableMetaMsg *pMeta = + rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); if (pMeta == NULL) { - mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId); + mError("app:%p:%p, table:%s, failed to get table meta, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + pMsg->pTable->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1952,11 +1985,13 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); pCfg->sid = htonl(pCfg->sid); - mTrace("dnode:%d, vgId:%d sid:%d, receive table config msg", pCfg->dnodeId, pCfg->vgId, pCfg->sid); + mTrace("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, + pCfg->vgId, pCfg->sid); SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); if (pTable == NULL) { - mError("dnode:%d, vgId:%d sid:%d, table not found", pCfg->dnodeId, pCfg->vgId, pCfg->sid); + mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, + pCfg->vgId, pCfg->sid); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -1980,36 +2015,39 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); - mPrint("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + mPrint("app:%p:%p, table:%s, drop table rsp received, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to drop in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); + mError("app:%p:%p, table:%s, failed to drop in dnode, reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + pTable->info.tableId, tstrerror(rpcMsg->code)); dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); return; } if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (mnodeMsg->pVgroup == NULL) { - mError("table:%s, failed to get vgroup", pTable->info.tableId); + mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); return; } - + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable }; - + int32_t code = sdbDeleteRow(&oper); if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, update ctables sdb error", pTable->info.tableId); + mError("app:%p:%p, table:%s, update ctables sdb error", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_SDB_ERROR); return; } if (mnodeMsg->pVgroup->numOfTables <= 0) { - mPrint("vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->pVgroup->vgId); + mPrint("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + mnodeMsg->pVgroup->vgId); mnodeDropVgroup(mnodeMsg->pVgroup, NULL); } @@ -2028,11 +2066,11 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); - - if (rpcMsg->code != TSDB_CODE_SUCCESS && rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + + if (!(rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code != TSDB_CODE_TDB_TABLE_ALREADY_EXIST)) { if (mnodeMsg->retry++ < 10) { mTrace("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry,tstrerror(rpcMsg->code), + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); @@ -2061,7 +2099,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mTrace("app:%p:%p, table:%s, created in dnode, thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } } } @@ -2251,22 +2289,24 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; - mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->rpcMsg.handle); + mTrace("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + pAlter->tableId, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { - mError("table:%s, failed to alter table, db not selected", pAlter->tableId); + mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("table:%s, failed to alter table, its log db", pAlter->tableId); + mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId); if (pMsg->pTable == NULL) { - mError("table:%s, failed to alter table, table not exist", pMsg->pTable->tableId); + mError("app:%p:%p, table:%s, failed to alter table, table not exist", pMsg->rpcMsg.ahandle, pMsg, + pMsg->pTable->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -2275,7 +2315,8 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { pAlter->tagValLen = htonl(pAlter->tagValLen); if (pAlter->numOfCols > 2) { - mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); + mError("app:%p:%p, table:%s, error numOfCols:%d in alter table", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId, + pAlter->numOfCols); return TSDB_CODE_MND_APP_ERROR; } @@ -2286,7 +2327,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT; if (pMsg->pTable->type == TSDB_SUPER_TABLE) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; - mTrace("table:%s, start to alter stable", pAlter->tableId); + mTrace("app:%p:%p, table:%s, start to alter stable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { code = mnodeAddSuperTableTag(pMsg, pTable, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { @@ -2300,7 +2341,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { } else { } } else { - mTrace("table:%s, start to alter ctable", pAlter->tableId); + mTrace("app:%p:%p, table:%s, start to alter ctable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { char *tagVal = (char *)(pAlter->schema + pAlter->numOfCols); From 3d56707049707a4d67880b09de98c11f09216a31 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 16 Jun 2020 18:19:03 +0800 Subject: [PATCH 11/24] fix td-642 --- src/client/src/tscSQLParser.c | 9 ++++-- src/tsdb/src/tsdbMeta.c | 4 +++ src/vnode/src/vnodeMain.c | 10 +++--- tests/script/general/parser/stream.sim | 44 ++++++++++---------------- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 64c38f011c..db12693c29 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6082,11 +6082,14 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* } } + if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if (pRight->nodeType == TSQL_NODE_VALUE) { - if ( pRight->pVal->nType == TSDB_DATA_TYPE_BOOL - || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY - || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) { + if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { + return TSDB_CODE_TSC_INVALID_SQL; + } + if ((pRight->pVal->nType == TSDB_DATA_TYPE_BINARY || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) + && (*pExpr)->_node.optr != TSDB_RELATION_LIKE) { return TSDB_CODE_TSC_INVALID_SQL; } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7795f25b71..1f31f18ce1 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -600,6 +600,10 @@ int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { return -1; } + if (pTable->cqhandle != NULL) { + pRepo->appH.cqDropFunc(pTable->cqhandle); + } + tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), tableId.tid, tableId.uid); if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0882ee983d..fef0b071a6 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -325,6 +325,11 @@ void vnodeRelease(void *pVnodeRaw) { tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; + // stop continuous query + if (pVnode->cq) + cqClose(pVnode->cq); + pVnode->cq = NULL; + if (pVnode->wal) walClose(pVnode->wal); pVnode->wal = NULL; @@ -436,11 +441,6 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { pVnode->sync = NULL; } - // stop continuous query - if (pVnode->cq) - cqClose(pVnode->cq); - pVnode->cq = NULL; - // release local resources only after cutting off outside connections vnodeRelease(pVnode); } diff --git a/tests/script/general/parser/stream.sim b/tests/script/general/parser/stream.sim index 6c8d8f93ea..fca5f37d74 100644 --- a/tests/script/general/parser/stream.sim +++ b/tests/script/general/parser/stream.sim @@ -51,19 +51,20 @@ sql drop table strm ## [TBASE304] print ====== TBASE-304 sleep 10000 -print create mt -sql create table mt (ts timestamp, c1 int) tags(t1 int, t2 int) +# we cannot reset query cache in server side, as a workaround, +# set super table name to mt304, need to change back to mt later +print create mt304 +sql create table mt304 (ts timestamp, c1 int) tags(t1 int, t2 int) print create tb1 -sql create table tb1 using mt tags(1, 1) +sql create table tb1 using mt304 tags(1, 1) print create tb2 -sql create table tb2 using mt tags(1, -1) +sql create table tb2 using mt304 tags(1, -1) print create strm -sql create table strm as select count(*), avg(c1) from mt where t2 >= 0 interval(4s) sliding(2s) +sql create table strm as select count(*), avg(c1) from mt304 where t2 >= 0 interval(4s) sliding(2s) sql insert into tb1 values (now,1) sql insert into tb2 values (now,2) -sleep 20000 +sleep 100000 sql select * from strm; -sleep 1000 if $rows != 2 then print ==== expect rows = 2, actually returned rows = $rows return -1 @@ -75,11 +76,11 @@ print data02 = $data02 if $data02 != 1.000000000 then return -1 endi -sql alter table mt drop tag t2; +sql alter table mt304 drop tag t2; sql insert into tb2 values (now,2); sql insert into tb1 values (now,1); sql select * from strm; -sql alter table mt add tag t2 int; +sql alter table mt304 add tag t2 int; sleep 10000 sql select * from strm @@ -98,7 +99,7 @@ sleep 4000 sql insert into tb2 values (now, 2, 'tb2') sleep 4000 sql insert into tb3 values (now, 0, 'tb3') -sleep 6000 +sleep 60000 sql describe strm if $rows == 0 then @@ -134,11 +135,9 @@ endi ## The vnode client needs to refresh metadata cache to allow strm calculate tb4's data. But the current refreshing frequency is every 10 min ## commented out the case below to save running time sql create table tb4 using stb tags('a4') -sleep 6000 sql insert into tb4 values(now, 4, 'tb4') -sleep 10000 +sleep 60000 sql select * from strm order by ts desc -sleep 1000 print ======== data0: $data00 $data01 $data02 $data03 #print ======== data1: $data10 $data11 $data12 $data13 #print ======== data2: $data20 $data21 $data22 $data23 @@ -160,7 +159,7 @@ sleep 3000 # waiting for new tag valid sql insert into tb1 values (now, 1, 'tb1_a1') sleep 4000 sql insert into tb4 values (now, -4, 'tb4_b4') -sleep 10000 +sleep 100000 sql select * from strm order by ts desc sleep 1000 print ======== data0: $data00 $data01 $data02 $data03 @@ -191,9 +190,9 @@ sql create table tb3 using stb tags(3, 'tb3') sql create table tb4 using stb tags(4, 'tb4') sql create table strm0 as select count(ts), count(c1), max(c2), min(c4), first(c5), last(c6) from stb where ts < now + 30s interval(4s) sliding(2s) -sleep 10000 +sleep 1000 sql insert into tb0 values (now, 0, 0, 0, 0, 'binary0', '涛思0', true) tb1 values (now, 1, 1, 1, 1, 'binary1', '涛思1', false) tb2 values (now, 2, 2, 2, 2, 'binary2', '涛思2', true) tb3 values (now, 3, 3, 3, 3, 'binary3', '涛思3', false) tb4 values (now, 4, 4, 4, 4, 'binary4', '涛思4', true) -sleep 5000 +sleep 20000 sql select * from strm0 order by ts desc sleep 1000 if $rows != 2 then @@ -202,7 +201,7 @@ if $rows != 2 then endi sql insert into tb0 values (now, 10, 10, 10, 10, 'binary0', '涛思0', true) tb1 values (now, 11, 11, 11, 11, 'binary1', '涛思1', false) tb2 values (now, 12, 12, 12, 12, 'binary2', '涛思2', true) tb3 values (now, 13, 13, 13, 13, 'binary3', '涛思3', false) tb4 values (now, 14, 14, 14, 14, 'binary4', '涛思4', true) -sleep 5000 +sleep 30000 sql select * from strm0 order by ts desc sleep 1000 if $rows != 4 then @@ -210,15 +209,4 @@ if $rows != 4 then return -1 endi -sql drop database if exists strm_db_0 -sql show databases -if $rows != 0 then - return -1 -endi - -sql create database $db -sql use $db -sql create table stb (ts timestamp, c1 int) tags(t1 int) -sql create table tb1 using stb tags(1) - system sh/exec.sh -n dnode1 -s stop -x SIGINT From 646b5e53a7aba1d1e861c614b67040ab7939d5a1 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 11:03:23 +0000 Subject: [PATCH 12/24] remove race condition in retry timer --- src/rpc/src/rpcMain.c | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 9bb63b751a..f812d95188 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -555,18 +555,10 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, return pConn; } -static void rpcCloseConn(void *thandle) { - SRpcConn *pConn = (SRpcConn *)thandle; +static void rpcReleaseConn(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; if (pConn->user[0] == 0) return; - rpcLockConn(pConn); - - if (pConn->user[0] == 0) { - rpcUnlockConn(pConn); - return; - } - pConn->user[0] = 0; if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle); @@ -591,7 +583,21 @@ static void rpcCloseConn(void *thandle) { taosFreeId(pRpc->idPool, pConn->sid); pConn->pContext = NULL; - tTrace("%s, rpc connection is closed", pConn->info); + tTrace("%s, rpc connection is released", pConn->info); +} + +static void rpcCloseConn(void *thandle) { + SRpcConn *pConn = (SRpcConn *)thandle; + if (pConn->user[0] == 0) return; + + rpcLockConn(pConn); + + if (pConn->user[0] == 0) { + rpcUnlockConn(pConn); + return; + } + + rpcReleaseConn(pConn); rpcUnlockConn(pConn); } @@ -911,8 +917,8 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + rpcReleaseConn(pConn); rpcUnlockConn(pConn); - rpcCloseConn(pConn); } static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { @@ -1217,7 +1223,6 @@ static void rpcProcessConnError(void *param, void *id) { static void rpcProcessRetryTimer(void *param, void *tmrId) { SRpcConn *pConn = (SRpcConn *)param; SRpcInfo *pRpc = pConn->pRpc; - int reportDisc = 0; rpcLockConn(pConn); @@ -1233,19 +1238,17 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { } else { // close the connection tTrace("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); - reportDisc = 1; + if (pConn->pContext) { + pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + rpcProcessConnError(pConn->pContext, NULL); + rpcReleaseConn(pConn); + } } } else { tTrace("%s, retry timer not processed", pConn->info); } rpcUnlockConn(pConn); - - if (reportDisc && pConn->pContext) { - pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcProcessConnError(pConn->pContext, NULL); - rpcCloseConn(pConn); - } } static void rpcProcessIdleTimer(void *param, void *tmrId) { From 4e87a28e8d0c980f2ea1c5d00cf530145d88a955 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 12:03:36 +0000 Subject: [PATCH 13/24] fix deadlock --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index f812d95188..d455cd645f 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1240,7 +1240,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { tTrace("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcProcessConnError(pConn->pContext, NULL); + taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); rpcReleaseConn(pConn); } } From 58922d86002a7d3772faba8229ef48e0951e693a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 12:05:25 +0000 Subject: [PATCH 14/24] compiling error --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d455cd645f..2e2202a31b 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1240,7 +1240,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { tTrace("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); + taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl); rpcReleaseConn(pConn); } } From 387f162ddf2f0e562858eafd37827e8cf8df602f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 12:36:58 +0000 Subject: [PATCH 15/24] add lock in all timer processing --- src/rpc/src/rpcMain.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2e2202a31b..90f11bb3de 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -490,6 +490,7 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg int rpcReportProgress(void *handle, char *pCont, int contLen) { SRpcConn *pConn = (SRpcConn *)handle; + rpcLockConn(pConn); if (pConn->user[0]) { // pReqMsg and reqMsgLen is re-used to store the context from app server pConn->pReqMsg = pCont; @@ -499,6 +500,8 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { tTrace("%s, rpc connection is already released", pConn->info); rpcFreeCont(pCont); + rpcUnlockConn(pConn); + return -1; } @@ -1254,13 +1257,17 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { static void rpcProcessIdleTimer(void *param, void *tmrId) { SRpcConn *pConn = (SRpcConn *)param; + rpcLockConn(pConn); + if (pConn->user[0]) { tTrace("%s, close the connection since no activity", pConn->info); if (pConn->inType) rpcReportBrokenLinkToServer(pConn); - rpcCloseConn(pConn); + rpcReleaseConn(pConn); } else { tTrace("%s, idle timer:%p not processed", pConn->info, tmrId); } + + rpcUnlockConn(pConn); } static void rpcProcessProgressTimer(void *param, void *tmrId) { From f3f0e39ab16f4bf6fb4675c8b62e28b6a8480a47 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 13:12:27 +0000 Subject: [PATCH 16/24] deadlock --- src/rpc/src/rpcMain.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 90f11bb3de..1558de37ea 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -489,20 +489,22 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg // this API is used by server app to keep an APP context in case connection is broken int rpcReportProgress(void *handle, char *pCont, int contLen) { SRpcConn *pConn = (SRpcConn *)handle; + int code = 0; rpcLockConn(pConn); + if (pConn->user[0]) { // pReqMsg and reqMsgLen is re-used to store the context from app server pConn->pReqMsg = pCont; pConn->reqMsgLen = contLen; - return 0; - } + } else { + tTrace("%s, rpc connection is already released", pConn->info); + rpcFreeCont(pCont); + code = -1; + } - tTrace("%s, rpc connection is already released", pConn->info); - rpcFreeCont(pCont); rpcUnlockConn(pConn); - - return -1; + return code; } /* todo: cancel process may have race condition, pContext may have been released From a703510f1bd0598634dfa04b5f54fa533e6c4e03 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 16 Jun 2020 13:40:18 +0000 Subject: [PATCH 17/24] tune up code --- src/rpc/src/rpcMain.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 1558de37ea..989021eb52 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -593,16 +593,11 @@ static void rpcReleaseConn(SRpcConn *pConn) { static void rpcCloseConn(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; - if (pConn->user[0] == 0) return; rpcLockConn(pConn); - if (pConn->user[0] == 0) { - rpcUnlockConn(pConn); - return; - } - - rpcReleaseConn(pConn); + if (pConn->user[0]) + rpcReleaseConn(pConn); rpcUnlockConn(pConn); } From c0786acc6419ac4bd240f6211cb57560f5e8c2d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Jun 2020 15:36:15 +0000 Subject: [PATCH 18/24] [TD-570] fix bug while sdb sync --- src/mnode/src/mnodeSdb.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1d428ab9a2..d8fc9aee5a 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -32,6 +32,7 @@ #include "mnodeSdb.h" #define SDB_TABLE_LEN 12 +#define SDB_SYNC_HACK 16 typedef enum { SDB_ACTION_INSERT, @@ -566,7 +567,7 @@ static int sdbWrite(void *param, void *data, int type) { // from app, oper is created if (pOper != NULL) { - sdbTrace("record from app is disposed, version:%" PRIu64 " code:%s", pHead->version, tstrerror(code)); + sdbTrace("record from app is disposed, version:%" PRIu64 " result:%s", pHead->version, tstrerror(code)); return code; } @@ -628,10 +629,10 @@ int32_t sdbInsertRow(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize; + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; SSdbOper *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; pHead->version = 0; pHead->len = pOper->rowSize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; @@ -692,10 +693,10 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { return TSDB_CODE_MND_SDB_INVAID_KEY_TYPE; } - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize; + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize + SDB_SYNC_HACK; SSdbOper *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; pHead->version = 0; pHead->len = keySize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; @@ -738,10 +739,10 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize; + int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; SSdbOper *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper); + SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; @@ -969,7 +970,7 @@ static void *sdbWorkerFp(void *param) { taosGetQitem(tsSdbWriteQall, &type, &item); if (type == TAOS_QTYPE_RPC) { pOper = (SSdbOper *)item; - pHead = (void *)pOper + sizeof(SSdbOper); + pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; } else { pHead = (SWalHead *)item; pOper = NULL; @@ -991,10 +992,10 @@ static void *sdbWorkerFp(void *param) { taosGetQitem(tsSdbWriteQall, &type, &item); if (type == TAOS_QTYPE_RPC) { pOper = (SSdbOper *)item; - if (pOper->cb) { + if (pOper != NULL && pOper->cb != NULL) { pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode); } - + if (pOper != NULL && pOper->pMsg != NULL) { sdbTrace("app:%p:%p, msg is processed, result:%s", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, tstrerror(pOper->retCode)); From 8035567babed25f262f0181a86edeb5505e76a9c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 17 Jun 2020 11:21:17 +0800 Subject: [PATCH 19/24] [modify cluster sim cases] --- tests/script/jenkins/basic.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index fe91e4bbde..773125e06d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -323,19 +323,20 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync_second.sim +./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim +./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim # TODO: check file number using sim ? ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim ./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim # fail ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim # fail ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim # unsupport ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim ./test.sh -f unique/arbitrator/insert_duplicationTs.sim @@ -349,7 +350,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim ./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim ./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim +#./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim # fail ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim From 975e652a3c51f5f9af54085de03c16f1985f02ab Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 17 Jun 2020 11:23:09 +0800 Subject: [PATCH 20/24] [modify cases] --- .../arbitrator/check_cluster_cfg_para.sim | 39 +++++++---- .../arbitrator/dn2_mn1_cache_file_sync.sim | 12 ++++ .../dn3_mn1_full_createTableFail.sim | 3 +- .../arbitrator/dn3_mn1_full_dropDnodeFail.sim | 2 + .../dn3_mn1_multiCreateDropTable.sim | 3 +- ...3_mn1_nw_disable_timeout_autoDropDnode.sim | 15 ++-- .../dn3_mn1_replica2_wal1_AddDelDnode.sim | 59 +++++++++++----- .../arbitrator/dn3_mn1_replica_change.sim | 70 +++++++++++++++++-- .../dn3_mn1_replica_change_dropDnod.sim | 10 ++- .../arbitrator/dn3_mn1_stopDnode_timeout.sim | 31 ++++---- .../arbitrator/dn3_mn1_vnode_change.sim | 55 +++++++++------ .../dn3_mn1_vnode_corruptFile_offline.sim | 54 ++++++++++++-- .../dn3_mn1_vnode_corruptFile_online.sim | 24 +++++-- .../arbitrator/dn3_mn1_vnode_nomaster.sim | 29 +++++++- .../unique/arbitrator/dn3_mn2_killDnode.sim | 3 +- .../arbitrator/insert_duplicationTs.sim | 15 +++- .../offline_replica2_alterTable_online.sim | 11 +++ .../offline_replica2_alterTag_online.sim | 15 ++++ .../offline_replica2_createTable_online.sim | 15 ++++ .../offline_replica2_dropDb_online.sim | 10 +++ .../offline_replica2_dropTable_online.sim | 15 ++++ .../offline_replica3_alterTable_online.sim | 15 ++++ .../offline_replica3_alterTag_online.sim | 15 ++++ .../offline_replica3_createTable_online.sim | 15 ++++ .../offline_replica3_dropDb_online.sim | 10 +++ .../offline_replica3_dropTable_online.sim | 15 ++++ .../replica_changeWithArbitrator.sim | 36 +++++++--- .../sync_replica2_alterTable_add.sim | 10 +++ .../sync_replica2_alterTable_drop.sim | 12 +++- .../arbitrator/sync_replica2_dropDb.sim | 10 +++ .../arbitrator/sync_replica2_dropTable.sim | 10 +++ .../sync_replica3_alterTable_add.sim | 16 ++++- .../sync_replica3_alterTable_drop.sim | 10 +++ .../arbitrator/sync_replica3_dropDb.sim | 10 +++ .../arbitrator/sync_replica3_dropTable.sim | 22 ++++-- tests/script/unique/arbitrator/testSuite.sim | 6 +- 36 files changed, 578 insertions(+), 124 deletions(-) diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim index e74717c077..af2b47d1ea 100644 --- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim +++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim @@ -7,84 +7,83 @@ system sh/deploy.sh -n dnode5 -i 5 system sh/deploy.sh -n dnode6 -i 6 system sh/deploy.sh -n dnode7 -i 7 - system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode1 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode1 -c statusInterval -v 3 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode1 -c timezone -v "" #system sh/cfg.sh -n dnode1 -c locale -v "" #system sh/cfg.sh -n dnode1 -c charset -v "" -system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 3 ######## dnode 2 the same with dnode1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode2 -c statusInterval -v 3 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode2 -c timezone -v "" #system sh/cfg.sh -n dnode2 -c locale -v "" #system sh/cfg.sh -n dnode2 -c charset -v "" -system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 3 ######## dnode 3 one para no same with dnode1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode3 -c statusInterval -v 3 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode3 -c timezone -v "" #system sh/cfg.sh -n dnode3 -c locale -v "" #system sh/cfg.sh -n dnode3 -c charset -v "" -system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 3 ######## dnode 4 one para no same with dnode1 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 5 -system sh/cfg.sh -n dnode4 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode4 -c statusInterval -v 3 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode4 -c timezone -v "" #system sh/cfg.sh -n dnode4 -c locale -v "" #system sh/cfg.sh -n dnode4 -c charset -v "" -system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 3 ######## dnode 5 one para no same with dnode1 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode5 -c offlineThreshold -v 16 +system sh/cfg.sh -n dnode5 -c offlineThreshold -v 7 system sh/cfg.sh -n dnode5 -c statusInterval -v 3 system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode5 -c timezone -v "" #system sh/cfg.sh -n dnode5 -c locale -v "" #system sh/cfg.sh -n dnode5 -c charset -v "" -system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 3 ######## dnode 6 one para no same with dnode1 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode6 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode6 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode6 -c statusInterval -v 2 system sh/cfg.sh -n dnode6 -c arbitrator -v $arbitrator #system sh/cfg.sh -n dnode6 -c timezone -v "" #system sh/cfg.sh -n dnode6 -c locale -v "" #system sh/cfg.sh -n dnode6 -c charset -v "" -system sh/cfg.sh -n dnode6 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode6 -c balanceInterval -v 3 ######## dnode 7 one para no same with dnode1 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode7 -c offlineThreshold -v 15 +system sh/cfg.sh -n dnode7 -c offlineThreshold -v 8 system sh/cfg.sh -n dnode7 -c statusInterval -v 3 system sh/cfg.sh -n dnode7 -c arbitrator -v "plum-VirtualBox:8001" #system sh/cfg.sh -n dnode7 -c timezone -v "" #system sh/cfg.sh -n dnode7 -c locale -v "" #system sh/cfg.sh -n dnode7 -c charset -v "" -system sh/cfg.sh -n dnode7 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode7 -c balanceInterval -v 3 print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -109,7 +108,12 @@ sql create dnode $hostname6 sql create dnode $hostname7 sleep 10000 +$loopCnt = 0 wait_dnode_created: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 7 then sleep 2000 @@ -154,7 +158,12 @@ endi sleep 10000 +$loopCnt = 0 wait_dnode_offline_overtime_dropped: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim index 2620746d38..ae73fc9423 100644 --- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim +++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim @@ -117,7 +117,13 @@ print ============== step4: stop dnode3, so date rows falling disc, generate two system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep $sleepTimer +$loopCnt = 0 wait_dnode3_offline: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -179,7 +185,13 @@ print ============== step7: restart dnode3, waiting sync end system sh/exec.sh -n dnode3 -s start sleep 3000 +$loopCnt = 0 wait_dnode3_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim index 7bcde02ba8..c373bce188 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim index 97c2f02427..29fa60de94 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -37,6 +38,7 @@ system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim index 67164908c7..c3bae9005a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim index 9b1e320946..ac2b8365f9 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim @@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/deploy.sh -n dnode5 -i 5 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode5 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode4 -c walLevel -v 1 -system sh/cfg.sh -n dnode5 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 @@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10 -system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1 -system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1 print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -126,7 +117,13 @@ system sh/exec.sh -n dnode4 -s stop #system sh/port.sh -p 7400 -s down sleep 12000 +$loopCnt = 0 wait_dnode4_dropped: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim index ef6954d4ee..cacbcbb132 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim @@ -36,14 +36,16 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200 -#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200 -#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200 -#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 200 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 +system sh/cfg.sh -n dnode5 -c alternativeRole -v 2 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator @@ -71,7 +73,7 @@ system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect -print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data +print ============== step2: start dnode2/dnode3 and add into cluster, then create database replica 2, create table , and insert data system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 @@ -84,7 +86,7 @@ $totalRows = 0 $tsStart = 1420041600000 $db = db -sql create database $db replica 2 maxTables 4 +sql create database $db replica 2 sql use $db # create table , insert data @@ -164,7 +166,13 @@ endi sleep 15000 +$loopCnt = 0 wait_drop: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 3000 @@ -193,14 +201,17 @@ if $dnode4Status != ready then endi print ============== step4-1: restart dnode3, adn add into cluster -system rm -rf ../../sim/dnode3 -sleep 3000 +system rm -rf ../../../sim/dnode3 +sleep 1000 system sh/deploy.sh -n dnode3 -i 3 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1 @@ -209,7 +220,13 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 sleep 3000 +$loopCnt = 0 wait_dnode3_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes print rows: $rows if $rows != 4 then @@ -228,19 +245,19 @@ $dnode4Status = $data4_4 $dnode5Status = $data4_5 if $dnode1Status != ready then - return -1 + goto wait_dnode3_ready endi if $dnode2Status != ready then - return -1 + goto wait_dnode3_ready endi if $dnode3Status != null then - return -1 + goto wait_dnode3_ready endi if $dnode4Status != ready then - return -1 + goto wait_dnode3_ready endi if $dnode5Status != ready then - return -1 + goto wait_dnode3_ready endi sql select count(*) from $stb @@ -255,7 +272,13 @@ sleep 1000 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 sleep 3000 +$loopCnt = 0 wait_dnode5: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 5 then sleep 3000 @@ -274,22 +297,22 @@ $dnode5Status = $data4_5 $dnode6Status = $data4_6 if $dnode1Status != ready then - return -1 + goto wait_dnode5 endi if $dnode2Status != ready then - return -1 + goto wait_dnode5 endi if $dnode3Status != null then - return -1 + goto wait_dnode5 endi if $dnode4Status != ready then - return -1 + goto wait_dnode5 endi if $dnode5Status != ready then - return -1 + goto wait_dnode5 endi if $dnode6Status != ready then - return -1 + goto wait_dnode5 endi print ============== step6: create database and table until not free vnodes diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 1652470346..9b01b2fae3 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -92,7 +93,14 @@ endi print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer + + +$loopCnt = 0 wait_dnode2_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 2 then sleep 2000 @@ -117,7 +125,14 @@ endi system sh/exec.sh -n dnode2 -s start sleep $sleepTimer + + +$loopCnt = 0 wait_dnode2_reready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 2 then sleep 2000 @@ -155,8 +170,13 @@ sleep 3000 sql alter database $db replica 2 sleep $sleepTimer - +$loopCnt = 0 wait_dnode3_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -195,8 +215,13 @@ endi print ============== step4: stop dnode2 for checking if sync ok system sh/exec.sh -n dnode2 -s stop sleep $sleepTimer - +$loopCnt = 0 wait_dnode2_offline: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -235,7 +260,13 @@ print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start sleep 3000 +$loopCnt = 0 wait_dnode2_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -273,8 +304,13 @@ sleep 3000 sql alter database $db replica 3 sleep $sleepTimer - +$loopCnt = 0 wait_dnode4_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 4 then sleep 2000 @@ -308,8 +344,13 @@ endi print ============== step7: alter replica from 3 to 2, and waiting sync sql alter database $db replica 2 sleep $sleepTimer - +$loopCnt = 0 wait_vgroups_replic_to_2: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 @@ -350,8 +391,13 @@ endi print ============== step8: alter replica from 2 to 1, and waiting sync sql alter database $db replica 1 sleep $sleepTimer - +$loopCnt = 0 wait_vgroups_replic_to_1: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 @@ -381,7 +427,13 @@ if $sencodDnode_5 != null then goto wait_vgroups_replic_to_1 endi +$loopCnt = 0 all_dnodes_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 4 then sleep 2000 @@ -429,7 +481,13 @@ sql drop dnode $hostname2 sql drop dnode $hostname3 sleep $sleepTimer +$loopCnt = 0 wait_dnode23_dropped: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 2 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index b870d07c1d..9b001820f6 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -97,8 +98,13 @@ print ============== step3: drop dnode4, then check rows #system sh/exec.sh -n dnode4 -s stop -x SIGINT sql drop dnode $hostname4 sleep $sleepTimer - +$loopCnt = 0 wait_dnode4_dropped: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim index 1b0a184690..152a65a75a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim @@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/deploy.sh -n dnode5 -i 5 -system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode5 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode4 -c walLevel -v 1 -system sh/cfg.sh -n dnode5 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 @@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10 -system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1 system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1 -system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1 print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -124,7 +115,13 @@ print ============== step3: stop dnode4, after timerout dnode4 will be auto-drop system sh/exec.sh -n dnode4 -s stop sleep 12000 +$loopCnt = 0 wait_dnode4_dropped: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -178,7 +175,7 @@ system sh/exec.sh -n dnode4 -s stop system rm -rf ../../../sim/dnode4 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 @@ -192,7 +189,13 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 sleep 6000 +$loopCnt = 0 wait_dnode4_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 4 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index d2bd5c6b26..77dbdb0c4c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect -print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data +print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start @@ -59,7 +60,7 @@ $totalTableNum = 10 $sleepTimer = 3000 $db = db -sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 2 maxTables $totalTableNum sql use $db # create table , insert data @@ -87,13 +88,18 @@ while $i < $tblNum endw sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 endi -print ============== step3: stop dnode4, and remove its vnodeX subdirector +sql show dnodes +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 + +print ============== step3: stop dnode4, then destroy the contents of its data file system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer $loopCnt = 0 @@ -110,14 +116,11 @@ endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +$dnode1Status = $data4_1 $dnode2Status = $data4_2 $dnode3Status = $data4_3 $dnode4Status = $data4_4 -#$dnode5Status = $data4_5 if $dnode4Status != offline then sleep 2000 @@ -146,10 +149,20 @@ if $dnode3Vtatus != master then goto wait_dnode4_vgroup_offline endi - -system rm -rf ../../../sim/dnode4/data/vnode/* +system echo "haha, nothing......" > ../../../sim/dnode4/data/vnode/vnode2/tsdb/data/f1643.data +#system rm -rf ../../../sim/dnode4/data/vnode/* sleep 1000 +print ============== step3-1: insert new data +sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x ) +$totalRows = $totalRows + 3 + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + print ============== step4: restart dnode4, waiting sync end system sh/exec.sh -n dnode4 -s start sleep $sleepTimer @@ -213,11 +226,16 @@ if $dnode3Vtatus != master then goto wait_dnode4_vgroup_slave endi -print ============== step5: stop dnode3/dnode2, and check rows -system sh/exec.sh -n dnode2 -s stop -system sh/exec.sh -n dnode3 -s stop -sleep $sleepTimer +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi +print ============== step5: stop dnode3/dnode2, and check rows +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep $sleepTimer $loopCnt = 0 wait_dnode23_offline: @@ -234,13 +252,10 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 +$dnode1Status = $data4_1 $dnode2Status = $data4_2 $dnode3Status = $data4_3 $dnode4Status = $data4_4 -#$dnode5Status = $data4_5 if $dnode2Status != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index a7529d5bbc..40ba2f6e9a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -96,7 +97,14 @@ endi print ============== step3: stop dnode3 for falling disc, then corrupt vnode data file in dnode3 system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep $sleepTimer + +$loopCnt = 0 wait_dnode3_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -119,7 +127,13 @@ if $dnode3Status != offline then goto wait_dnode3_offline_0 endi +$loopCnt = 0 wait_dnode3_vgroup_offline: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups if $rows != 1 then sleep 2000 @@ -154,14 +168,25 @@ print ---->dnode3 data files: [ $system_content ], expect is 3 # return -1 #endi -#system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data -#sleep 1000 +system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data +sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x ) +$totalRows = $totalRows + 3 +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi print ============== step4: restart dnode3, and run query system sh/exec.sh -n dnode3 -s start sleep $sleepTimer - +$loopCnt = 0 wait_dnode3_reready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -183,8 +208,13 @@ if $dnode3Status != ready then sleep 2000 goto wait_dnode3_reready endi - +$loopCnt = 0 wait_dnode3_vgroup_slave: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups if $rows != 1 then sleep 2000 @@ -225,12 +255,16 @@ print ----> dnode3 data files: [ $system_content ], expect is 0 # return -1 #endi -return -1 - print ============== step5: stop dnode2, and check if dnode3 sync ok system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer + +$loopCnt = 0 wait_dnode2_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -253,7 +287,13 @@ if $dnode2Status != offline then goto wait_dnode2_offline_0 endi +$loopCnt = 0 wait_dnode3_vgroup_master: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index fc8577964d..477f4684b5 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -94,11 +95,13 @@ if $data00 != $totalRows then endi print ============== step3: corrupt vnode data file in dnode3, not stop dnode3 -# del the second row -system sed '2d' ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/v1849.data +system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data sleep 1000 -print ============== step4: run query +print ============== step4: insert new data, and run query +sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x ) +$totalRows = $totalRows + 3 + sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then @@ -108,7 +111,14 @@ endi print ============== step5: stop dnode2, and check if dnode3 sync ok system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer + +$loopCnt = 0 wait_dnode2_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -131,7 +141,13 @@ if $dnode2Status != offline then goto wait_dnode2_offline_0 endi +$loopCnt = 0 wait_dnode3_vgroup_master: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 4e9afbf31a..f5bcd055a0 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -96,7 +97,13 @@ print ============== step3: stop dnode4/dnode2 system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer + +$loopCnt = 0 wait_dnode4_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -123,7 +130,13 @@ if $dnode2Status != offline then goto wait_dnode4_offline_0 endi +$loopCnt = 0 wait_dnode4_vgroup_offline: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show vgroups if $rows != 1 then sleep 2000 @@ -192,7 +205,12 @@ endi print ============== step5: stop dnode3 system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep $sleepTimer +$loopCnt = 0 wait_dnode3_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -221,7 +239,12 @@ if $dnode2Status != ready then goto wait_dnode3_offline_0 endi +$loopCnt = 0 wait_dnode2_vgroup_master: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -243,8 +266,8 @@ if $dnode2Vtatus != master then goto wait_dnode2_vgroup_master endi -sql insert into tb98 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002) tb99 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002) -$totalRows = $totalRows + 6 +sql insert into $tb values (now, 9000) (now + 1s, 9001) (now + 2s, 9002) +$totalRows = $totalRows + 3 sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then diff --git a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim index 57a833b8de..acf157a93a 100644 --- a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start diff --git a/tests/script/unique/arbitrator/insert_duplicationTs.sim b/tests/script/unique/arbitrator/insert_duplicationTs.sim index aae1f46180..b9aff29f0c 100644 --- a/tests/script/unique/arbitrator/insert_duplicationTs.sim +++ b/tests/script/unique/arbitrator/insert_duplicationTs.sim @@ -43,11 +43,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -117,7 +117,13 @@ print ============== step4: stop dnode2, so date rows falling disc, generate two system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer +$loopCnt = 0 wait_dnode2_offline: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 @@ -160,8 +166,13 @@ $totalRows = $totalRows + 2 print ============== step5: restart dnode2, waiting sync end system sh/exec.sh -n dnode2 -s start sleep 3000 - +$loopCnt = 0 wait_dnode2_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi + sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index 26a6359a5c..18ffb32eed 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -99,7 +99,13 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer + +$loopCnt = 0 wait_dnode4_offline_0: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -160,7 +166,12 @@ $totalRows = $totalRows + $addRows print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$loopCnt = 0 wait_dnode4_ready: +$loopCnt = $loopCnt + 1 +if $loopCnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index b883f077c2..6e85429b77 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -100,7 +100,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -123,7 +128,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -187,7 +197,12 @@ endi print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index e7b2c70c78..0356f72a97 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -101,7 +101,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -124,7 +129,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -174,7 +184,12 @@ endi print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 20c575d382..dc2586b43b 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 9f72cde440..721c7d60ba 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -158,7 +168,12 @@ $tblNum = $tblNum - 5 print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 43f9dfff14..897c662b51 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -160,7 +170,12 @@ $totalRows = $totalRows + $addRows print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index e27bbafba7..95ae8047fa 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -100,7 +100,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -123,7 +128,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -187,7 +197,12 @@ endi print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index 1e47157975..a60f810eea 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -100,7 +100,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -123,7 +128,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -173,7 +183,12 @@ endi print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index c9fc91527d..a619cbe826 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index aecf41b892..3b7ee869ed 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -158,7 +168,12 @@ $tblNum = $tblNum - 5 print ============== step5: restart dnode4, waiting dnode4 synced system sh/exec.sh -n dnode4 -s start +$cnt = 0 wait_dnode4_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index 4a133d2aac..a5816b589b 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 @@ -37,11 +38,21 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8 + +system sh/cfg.sh -n dnode1 -c statusInterval -v 3 +system sh/cfg.sh -n dnode2 -c statusInterval -v 3 +system sh/cfg.sh -n dnode3 -c statusInterval -v 3 +system sh/cfg.sh -n dnode4 -c statusInterval -v 3 print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start @@ -96,7 +107,12 @@ sql create dnode $hostname2 sleep 3000 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2 +$cnt = 0 wait_dnode2_ready: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 2 then sleep 2000 @@ -105,14 +121,8 @@ endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 $dnode1Status = $data4_1 $dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 if $dnode1Status != ready then sleep 2000 @@ -123,7 +133,7 @@ if $dnode2Status != ready then goto wait_dnode2_ready endi -print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2 +print ============== step3: stop dnode1/dnode2, modify cfg numOfMnodes to 2, and restart dnode1/dnode2 system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop sleep 3000 @@ -143,7 +153,12 @@ sleep 5000 print ============= step4: wait dnode ready +$cnt = 0 wait_dnode_ready: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 2 then sleep 2000 @@ -180,7 +195,12 @@ print ============== step5: stop dnode1 system sh/exec.sh -n dnode1 -s stop sleep 3000 +$cnt = 0 wait_dnode2_master: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show mnodes if $rows != 2 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index 9730842938..4f66e5c135 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index 21957871a5..7bcd148bc9 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start run_back unique/arbitrator/sync_replica_alterTable_background_drop.sim print ============== step6: check result -#in background.sim, drop one column and add one new column, then insert 200 rows +#in background.sim, drop one column and add one new column, then insert 36 rows $totalRows = $totalRows + 36 $cnt = 0 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index 9836284af9..f61b1e3cb9 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index 4793e8e535..b27558fdf0 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 3 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index 9277ad2c85..2a283a6a0c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -162,7 +172,7 @@ while $i < $tblNum endw sql select count(*) from $stb -print data00 $data00 +print data00:$data00 totalRows:$totalRows if $data00 != $totalRows then return -1 endi @@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start run_back unique/arbitrator/sync_replica_alterTable_background_add.sim print ============== step6: check result -#in background.sim, add one column and insert 200 rows +#in background.sim, add one column and insert 36 rows $totalRows = $totalRows + 36 $cnt = 0 @@ -183,7 +193,7 @@ if $cnt == 20 then endi sql select count(*) from $stb if $data00 != $totalRows then - print data00: $data00 totalRows: $totalRows + print data00:$data00 totalRows:$totalRows sleep 2000 goto wait_table_altered endi diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 6593d6933b..7d321ed381 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 20 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 7099b1dc8e..0f7ee50c39 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index f902b41de5..9c33dd8bb4 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -99,7 +99,12 @@ endi print ============== step3: stop dnode4 system sh/exec.sh -n dnode4 -s stop -x SIGINT sleep $sleepTimer +$cnt = 0 wait_dnode4_offline_0: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show dnodes if $rows != 4 then sleep 2000 @@ -122,7 +127,12 @@ if $dnode4Status != offline then goto wait_dnode4_offline_0 endi +$cnt = 0 wait_dnode4_vgroup_offline: +$cnt = $cnt + 1 +if $cnt == 10 then + return -1 +endi sql show vgroups if $rows != 1 then sleep 2000 @@ -143,7 +153,7 @@ if $dnode3Vtatus != master then sleep 2000 goto wait_dnode4_vgroup_offline endi -sleep 2000 + print ============== step4: insert more data rows $tsStart = $tsEnd + 1000 $i = 0 @@ -162,17 +172,17 @@ while $i < $tblNum endw sql select count(*) from $stb -print data00 $data00 +print data00:$data00 totalRows:$totalRows if $data00 != $totalRows then return -1 endi -print ============== step5: restart dnode4, while drop database in other thead when dnode4 is syncing +print ============== step5: restart dnode4, while drop some tables in other thread when dnode4 is syncing system sh/exec.sh -n dnode4 -s start run_back unique/arbitrator/sync_replica_dropTable_background.sim print ============== step6: check result -#in background.sim, drop 10 tables +#in background.sim, drop 5 tables $totalRows = $totalRows - 5400 $cnt = 0 @@ -183,7 +193,7 @@ if $cnt == 20 then endi sql select count(*) from $stb if $data00 != $totalRows then - print data00: $data00 + print data00:$data00 totalRows:$totalRows sleep 2000 goto wait_table_dropped endi @@ -191,7 +201,7 @@ endi $tblNum = $tblNum - 5 sql select count(tbname) from $stb if $data00 != $tblNum then - print data00: $data00 + print data00: $data00 tblNum: $tblNum sleep 2000 goto wait_table_dropped endi diff --git a/tests/script/unique/arbitrator/testSuite.sim b/tests/script/unique/arbitrator/testSuite.sim index d286ecfaf2..9593137bdd 100644 --- a/tests/script/unique/arbitrator/testSuite.sim +++ b/tests/script/unique/arbitrator/testSuite.sim @@ -1,5 +1,5 @@ +run unique/arbitrator/check_cluster_cfg_para.sim run unique/arbitrator/dn2_mn1_cache_file_sync.sim -run unique/arbitrator/dn2_mn1_cache_file_sync_second.sim run unique/arbitrator/dn3_mn1_full_createTableFail.sim run unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim run unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -11,7 +11,7 @@ run unique/arbitrator/dn3_mn1_stopDnode_timeout.sim run unique/arbitrator/dn3_mn1_vnode_change.sim run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim run unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -####run unique/arbitrator/dn3_mn1_vnode_delDir.sim +run unique/arbitrator/dn3_mn1_vnode_delDir.sim run unique/arbitrator/dn3_mn1_vnode_nomaster.sim run unique/arbitrator/dn3_mn2_killDnode.sim run unique/arbitrator/insert_duplicationTs.sim @@ -33,4 +33,4 @@ run unique/arbitrator/sync_replica2_dropTable.sim run unique/arbitrator/sync_replica3_alterTable_add.sim run unique/arbitrator/sync_replica3_alterTable_drop.sim run unique/arbitrator/sync_replica3_dropDb.sim -run unique/arbitrator/sync_replica3_dropTable.sim +run unique/arbitrator/sync_replica3_dropTable.sim \ No newline at end of file From ff25ab6b14ce5535219438e3a985aa101f7a9a9f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 17 Jun 2020 11:54:18 +0800 Subject: [PATCH 21/24] fix TD-645, TD-660 --- src/cq/src/cqMain.c | 21 ++++++++++++++++++++- src/inc/taos.h | 2 +- tests/script/general/parser/alter.sim | 14 +++++++------- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 13f3c0a2b0..769dfea774 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -256,11 +256,30 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SDataRow trow = (SDataRow)pBlk->data; tdInitDataRow(trow, pSchema); + union { + char buf[sizeof(int64_t)]; + tstr str; + } nullVal; + for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; char* val = (char*)row[i]; if (IS_VAR_DATA_TYPE(c->type)) { - val -= sizeof(VarDataLenT); + if (val == NULL) { + val = nullVal.buf; + if (c->type == TSDB_DATA_TYPE_BINARY) { + setNull(nullVal.str.data, TSDB_DATA_TYPE_BINARY, 1); + nullVal.str.len = 1; + } else { + setNull(nullVal.str.data, TSDB_DATA_TYPE_NCHAR, 4); + nullVal.str.len = 4; + } + } else { + val -= sizeof(VarDataLenT); + } + } else if (val == NULL) { + val = nullVal.buf; + setNull(val, c->type, c->bytes); } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); } diff --git a/src/inc/taos.h b/src/inc/taos.h index bf6942be8b..9ac97a24e1 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -55,7 +55,7 @@ typedef enum { typedef struct taosField { char name[65]; uint8_t type; - short bytes; + uint16_t bytes; } TAOS_FIELD; #ifdef _TD_GO_DLL_ diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index 44c12fa968..c30cc171bc 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -137,7 +137,7 @@ sleep 6000 sql insert into tb1 values (now, 2, 'taos') sleep 3000 sql select * from strm -if $rows != 1 then +if $rows != 2 then return -1 endi if $data04 != 1 then @@ -148,7 +148,7 @@ sleep 6000 sql insert into tb1 values (now, 3, 'taos', 3); sleep 3000 sql select * from strm -if $rows != 1 then +if $rows != 3 then return -1 endi if $data04 != 1 then @@ -188,9 +188,9 @@ sql create table tb using mt tags(1) sleep 3000 sql insert into tb values ('2018-11-01 16:30:00.000', 1, 'insert', 1) sql alter table mt drop column c3 -# the below query should be deleted after bug fix + sql insert into tb values ('2018-11-01 16:29:59.000', 1, 'insert') -sql_error import into tb values ('2018-11-01 16:29:59.000', 1, 'import') +sql import into tb values ('2018-11-01 16:29:59.000', 1, 'import') sql select * from tb order by ts desc if $data01 != 1 then return -1 @@ -203,13 +203,13 @@ sql select * from tb order by ts desc if $data03 != NULL then return -1 endi -# the query below should be deleted after bug fix + sql reset query cache sql insert into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3) -sql_error import into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3) +sql import into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3) sql import into tb values ('2018-11-01 16:39:58.000', 2, 'import', 3) sql select * from tb order by ts desc -if $rows != 2 then +if $rows != 4 then return -1 endi if $data03 != 3 then From 70e5284476192398155cd326eb9c2766e86a48e6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 17 Jun 2020 04:05:28 +0000 Subject: [PATCH 22/24] [TD-570] errno process while create vnode --- src/mnode/src/mnodeTable.c | 39 +++++++++++++++++-------------------- src/mnode/src/mnodeVgroup.c | 7 +++++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index e71c477b8f..f0e3104a9a 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2067,7 +2067,22 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; assert(pTable); - if (!(rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code != TSDB_CODE_TDB_TABLE_ALREADY_EXIST)) { + if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont; + if (pCreate->getMeta) { + mTrace("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p result:%s", + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle, + tstrerror(rpcMsg->code)); + + mnodeMsg->retry = 0; + dnodeReprocessMnodeWriteMsg(mnodeMsg); + } else { + mTrace("app:%p:%p, table:%s, created in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, + pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + + dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + } + } else { if (mnodeMsg->retry++ < 10) { mTrace("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), @@ -2078,29 +2093,11 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; + SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&oper); - + dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); } - } else { - SCMCreateTableMsg *pCreate = mnodeMsg->rpcMsg.pCont; - if (pCreate->getMeta) { - mTrace("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p", mnodeMsg->rpcMsg.ahandle, - mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle); - - mnodeMsg->retry = 0; - dnodeReprocessMnodeWriteMsg(mnodeMsg); - } else { - mTrace("app:%p:%p, table:%s, created in dnode, thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - pTable->info.tableId, mnodeMsg->rpcMsg.handle); - - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); - } } } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 828d77b3ce..d46d98774a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -613,7 +613,6 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) { } void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { - mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p db:%s", pVgroup->vgId, pVgroup->vgId, ahandle, pVgroup->dbName); SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .handle = ahandle, @@ -626,9 +625,12 @@ void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { } void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { - mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); + mTrace("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, + pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + mTrace("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId, + i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle); mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); } } @@ -746,6 +748,7 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { } mnodeDecVgroupRef(pVgroup); + mTrace("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp); SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL); From 1768b59fcabccb590dac608f7711ad7393c65461 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 17 Jun 2020 14:25:13 +0800 Subject: [PATCH 23/24] [modify] --- .../dn3_mn1_vnode_corruptFile_offline.sim | 51 +++++++++++-------- .../replica_changeWithArbitrator.sim | 6 +-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 40ba2f6e9a..a0d8acb272 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -155,22 +155,31 @@ if $dnode2Vtatus != master then goto wait_dnode3_vgroup_offline endi -#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | sed 's/^[ \t]*//g' -#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | sed 's/[ \t]*$//g' +#$expectCnt = 3 . : +#print expectCnt: [ $expectCnt ] +#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':' +#system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr '\n' ':' #print --2-->dnode3 data files: [ $system_content ] -system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l -print ---->dnode2 data files: [ $system_content ], expect is 0 -system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l -print ---->dnode3 data files: [ $system_content ], expect is 3 -#if $system_content != 3 then -# return -1 -#endi +system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n' +print ---->dnode2 data files: $system_content expect: 0 +if $system_content != 0 then + return -1 +endi + +system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l | grep "^-" | wc -l | tr -d '\n' +print ---->dnode3 data files: $system_content expect: 3 +if $system_content != 3 then + return -1 +endi system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data + +print ============== step3-1: insert some news data for let version changed sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x ) -$totalRows = $totalRows + 3 +sql insert into $tb values ( now + 10a , $x ) ( now + 11a , $x ) ( now + 12a , $x ) +$totalRows = $totalRows + 6 sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then @@ -242,18 +251,18 @@ if $data00 != $totalRows then return -1 endi -system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l -print ----> dnode2 data files: [ $system_content ], expect is 0 -#if $system_content != 0 then -# return -1 -#endi +system_content ls ../../../sim/dnode2/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n' +print ----> dnode2 data files: $system_content expect: 0 +if $system_content != 0 then + return -1 +endi -system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l -print ----> dnode3 data files: [ $system_content ], expect is 0 -#if $system_content != 0 then -# print there should be no data file in dnode3 after sync -# return -1 -#endi +system_content ls ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/ -l |grep "^-"|wc -l | tr -d '\n' +print ----> dnode3 data files: $system_content expect: 0 +if $system_content != 0 then + print there should be no data file in dnode3 after sync + return -1 +endi print ============== step5: stop dnode2, and check if dnode3 sync ok system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index a5816b589b..1b2c8d1db6 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -25,9 +25,9 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 8 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 8 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 From 7e0d1ccf1c4822a7ff086722ee59cdbb2e663949 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 17 Jun 2020 06:43:43 +0000 Subject: [PATCH 24/24] When modifying the table schema, the calculation of memory size is wrong --- src/mnode/src/mnodeTable.c | 157 +++++++++++++++++-------- src/mnode/src/mnodeVgroup.c | 2 +- tests/script/unique/dnode/balance3.sim | 1 - 3 files changed, 111 insertions(+), 49 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index f0e3104a9a..a421b01d7a 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -185,7 +185,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; void *oldSchema = pTable->schema; - memcpy(pTable, pNew, pOper->rowSize); + memcpy(pTable, pNew, sizeof(SChildTableObj)); pTable->sql = pNew->sql; pTable->schema = pNew->schema; free(pNew); @@ -439,7 +439,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { if (pTable != pNew) { void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; - memcpy(pTable, pNew, pOper->rowSize); + memcpy(pTable, pNew, sizeof(SSuperTableObj)); pTable->schema = pNew->schema; free(pNew->vgHash); free(pNew); @@ -878,7 +878,16 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char * return -1; } -static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { +static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + + return code; +} + +static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; @@ -911,24 +920,34 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, S pStable->numOfTags += ntags; pStable->tversion++; + mPrint("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + schema[0].name); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeAddSuperTableTagCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, stable %s, succeed to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, - schema[0].name); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } return code; } -static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *tagName) { +static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -941,24 +960,33 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, SSuperTableObj *pStable, pStable->numOfTags--; pStable->tversion++; + mPrint("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeDropSuperTableTagCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, stable %s, succeed to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } return code; } -static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *oldTagName, - char *newTagName) { +static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -975,22 +1003,24 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, SSuperTableObj *pSt if (mnodeFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_MND_TAG_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); tstrncpy(schema->name, newTagName, sizeof(schema->name)); + mPrint("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + oldTagName, newTagName); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeModifySuperTableTagNameCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, stable %s, succeed to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, - oldTagName, newTagName); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1008,8 +1038,16 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col return -1; } -static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { +static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { SDbObj *pDb = pMsg->pDb; + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (ncols <= 0) { mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_APP_ERROR; @@ -1050,24 +1088,34 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable mnodeDecAcctRef(pAcct); } + mPrint("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeAddSuperTableColumnCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, stable %s, succeed to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } return code; } -static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable, char *colName) { +static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -1090,16 +1138,18 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStabl mnodeDecAcctRef(pAcct); } + mPrint("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeDropSuperTableColumnCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, stable %s, succeed to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); code = TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1674,10 +1724,6 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mnodeModifyChildTableTagValue(SChildTableObj *pTable, char *tagName, char *nContent) { - return TSDB_CODE_COM_OPS_NOT_SUPPORT; -} - static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t col = 0; col < pTable->numOfColumns; col++) { @@ -1689,16 +1735,24 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col return -1; } -static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { +static int32_t mnodeAddNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, ctable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { - mError("app:%p:%p, table:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - mError("app:%p:%p, table:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("app:%p:%p, ctable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } @@ -1722,28 +1776,38 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable pAcct->acctInfo.numOfTimeSeries += ncols; mnodeDecAcctRef(pAcct); } - + + mPrint("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeAddNormalTableColumnCb }; int32_t code = sdbUpdateRow(&oper); if (code == TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, table %s, succeed to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } return code; } -static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTable, char *colName) { +static int32_t mnodeDropNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + mLPrint("app:%p:%p, ctable %s, drop column result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { - mError("app:%p:%p, table:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -1757,17 +1821,19 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, SChildTableObj *pTabl pAcct->acctInfo.numOfTimeSeries--; mnodeDecAcctRef(pAcct); } - + + mPrint("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, - .pMsg = pMsg + .pMsg = pMsg, + .cb = mnodeDropNormalTableColumnCb }; int32_t code = sdbUpdateRow(&oper); if (code != TSDB_CODE_SUCCESS) { - mPrint("app:%p:%p, table %s, succeed to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -2323,30 +2389,27 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT; if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; mTrace("app:%p:%p, table:%s, start to alter stable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - code = mnodeAddSuperTableTag(pMsg, pTable, pAlter->schema, 1); + code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - code = mnodeDropSuperTableTag(pMsg, pTable, pAlter->schema[0].name); + code = mnodeDropSuperTableTag(pMsg, pAlter->schema[0].name); } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - code = mnodeModifySuperTableTagName(pMsg, pTable, pAlter->schema[0].name, pAlter->schema[1].name); + code = mnodeModifySuperTableTagName(pMsg, pAlter->schema[0].name, pAlter->schema[1].name); } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - code = mnodeAddSuperTableColumn(pMsg, pTable, pAlter->schema, 1); + code = mnodeAddSuperTableColumn(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - code = mnodeDropSuperTableColumn(pMsg, pTable, pAlter->schema[0].name); + code = mnodeDropSuperTableColumn(pMsg, pAlter->schema[0].name); } else { } } else { mTrace("app:%p:%p, table:%s, start to alter ctable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - char *tagVal = (char *)(pAlter->schema + pAlter->numOfCols); - code = mnodeModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal); + return TSDB_CODE_COM_OPS_NOT_SUPPORT; } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - code = mnodeAddNormalTableColumn(pMsg, pTable, pAlter->schema, 1); + code = mnodeAddNormalTableColumn(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - code = mnodeDropNormalTableColumn(pMsg, pTable, pAlter->schema[0].name); + code = mnodeDropNormalTableColumn(pMsg, pAlter->schema[0].name); } else { } } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index d46d98774a..0475a48ce2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -149,7 +149,7 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { } } - memcpy(pVgroup, pNew, pOper->rowSize); + memcpy(pVgroup, pNew, sizeof(SVgObj)); free(pNew); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index a09fd7e4ea..b2adb24dfa 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -177,7 +177,6 @@ endi print ========== step5 sql create dnode $hostname6 -system sh/deploy.sh -n dnode6 -i 6 system sh/exec.sh -n dnode6 -s start $x = 0