From 53774be0cadc03d31a4188c2a113afa97480b1ba Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 14 Oct 2020 22:41:17 +0800 Subject: [PATCH 01/75] add debug info for sync module --- src/sync/inc/syncInt.h | 3 +++ src/sync/src/syncMain.c | 44 +++++++++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index f681810646..96f1629f72 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -65,6 +65,9 @@ typedef struct { typedef struct { int8_t role; int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; uint64_t version; SPeerStatus peersStatus[]; } SPeersStatus; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ef635e6efc..9c4397e2a0 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -48,7 +48,7 @@ static void * vgIdHash; static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack); +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(void *param); static int syncProcessPeerMsg(void *param, void *buffer); static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp); @@ -71,6 +71,28 @@ char* syncRole[] = { "master" }; +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP +} ESyncStatusType; + +char *statusType[] = { + "broadcast", + "broadcast-rsp", + "setup-conn", + "setup-conn-rsp", + "exchange-data", + "exchange-data-rsp" +}; + +uint16_t syncGenTranId() { + return taosRand() & 0XFFFF; +} + int32_t syncInit() { SPoolInfo info; @@ -539,7 +561,7 @@ void syncBroadcastStatus(SSyncNode *pNode) { for (int i = 0; i < pNode->replica; ++i) { if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; - syncSendPeersStatusMsgToPeer(pPeer, 1); + syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId()); } } @@ -894,14 +916,14 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id, - syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack); + sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, + syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); pPeer->version = pPeersStatus->version; syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role); if (pPeersStatus->ack) { - syncSendPeersStatusMsgToPeer(pPeer, 0); + syncSendPeersStatusMsgToPeer(pPeer, 0, pPeersStatus->type + 1, pPeersStatus->tranId); } } @@ -958,7 +980,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) { #define statusMsgLen sizeof(SSyncHead) + sizeof(SPeersStatus) + sizeof(SPeerStatus) * TAOS_SYNC_MAX_REPLICA -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { SSyncNode *pNode = pPeer->pSyncNode; char msg[statusMsgLen] = {0}; @@ -973,6 +995,8 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { pPeersStatus->version = nodeVersion; pPeersStatus->role = nodeRole; pPeersStatus->ack = ack; + pPeersStatus->type = type; + pPeersStatus->tranId = tranId; for (int i = 0; i < pNode->replica; ++i) { pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role; @@ -981,8 +1005,8 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { int retLen = write(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role], - pPeersStatus->version, pPeersStatus->ack); + sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, syncRole[pPeersStatus->role], + pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); @@ -997,7 +1021,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); if (pPeer->peerFd >= 0) { sDebug("%s, send role version to peer", pPeer->id); - syncSendPeersStatusMsgToPeer(pPeer, 1); + syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_SETUP_CONN, syncGenTranId()); return; } @@ -1110,7 +1134,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); syncAddPeerRef(pPeer); sDebug("%s, ready to exchange data", pPeer->id); - syncSendPeersStatusMsgToPeer(pPeer, 1); + syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); } } From ec8ac48a606e311c3c030a408510e74df30cd073 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 13 Nov 2020 12:44:53 +0000 Subject: [PATCH 02/75] return value is pointer --- src/sync/src/syncMain.c | 2 +- src/util/tests/trefTest.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8d90315c48..4b51c770e7 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1097,7 +1097,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return; + if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 6887b24abd..454860410b 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -77,8 +77,8 @@ void *acquireRelease(void *param) { printf("a"); id = random() % pSpace->refNum; - code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); - if (code >= 0) { + void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + if (p) { usleep(id % 5 + 1); taosReleaseRef(pSpace->rsetId, pSpace->p[id]); } From 9d6e37762319eea1425cb83cb8cdbb6c82149e2f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 14:25:16 +0800 Subject: [PATCH 03/75] TD-1898 --- src/mnode/src/mnodeSdb.c | 130 ++++++++++++++++++------------------ src/sync/inc/syncInt.h | 2 +- src/sync/src/syncMain.c | 31 +++++---- src/sync/src/syncRestore.c | 2 +- src/sync/src/syncRetrieve.c | 2 +- src/wal/src/walWrite.c | 4 +- 6 files changed, 86 insertions(+), 85 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 003ecd0d24..9c11fa80e8 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -166,7 +166,7 @@ static char *sdbGetKeyStr(SSdbTable *pTable, void *key) { } } -static char *sdbGetKeyStrFromObj(SSdbTable *pTable, void *key) { +static char *sdbGetObjStr(SSdbTable *pTable, void *key) { return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key)); } @@ -176,18 +176,18 @@ static void *sdbGetTableFromId(int32_t tableId) { static int32_t sdbInitWal() { SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; - char temp[TSDB_FILENAME_LEN]; + char temp[TSDB_FILENAME_LEN] = {0}; sprintf(temp, "%s/wal", tsMnodeDir); tsSdbObj.wal = walOpen(temp, &walCfg); if (tsSdbObj.wal == NULL) { - sdbError("failed to open sdb wal in %s", tsMnodeDir); + sdbError("vgId:1, failed to open wal in %s", tsMnodeDir); return -1; } - sdbInfo("open sdb wal for restore"); + sdbInfo("vgId:1, open wal for restore"); int code = walRestore(tsSdbObj.wal, NULL, sdbWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("failed to open wal for restore, reason:%s", tstrerror(code)); + sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code)); return -1; } return 0; @@ -205,10 +205,10 @@ static void sdbRestoreTables() { totalRows += pTable->numOfRows; numOfTables++; - sdbDebug("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows); + sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->tableName, pTable->numOfRows); } - sdbInfo("sdb is restored, ver:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables); + sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbObj.version, totalRows, numOfTables); } void sdbUpdateMnodeRoles() { @@ -217,12 +217,12 @@ void sdbUpdateMnodeRoles() { SNodesRole roles = {0}; syncGetNodesRole(tsSdbObj.sync, &roles); - sdbInfo("update mnodes sync roles, total:%d", tsSdbObj.cfg.replica); + sdbInfo("vgId:1, update mnodes roles, replica:%d", tsSdbObj.cfg.replica); for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) { SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]); if (pMnode != NULL) { pMnode->role = roles.role[i]; - sdbInfo("mnode:%d, role:%s", pMnode->mnodeId, mnodeGetMnodeRoleStr(pMnode->role)); + sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, mnodeGetMnodeRoleStr(pMnode->role)); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbObj.role = pMnode->role; mnodeDecMnodeRef(pMnode); } @@ -242,7 +242,7 @@ static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { } static void sdbNotifyRole(void *ahandle, int8_t role) { - sdbInfo("mnode role changed from %s to %s", mnodeGetMnodeRoleStr(tsSdbObj.role), mnodeGetMnodeRoleStr(role)); + sdbInfo("vgId:1, mnode role changed from %s to %s", mnodeGetMnodeRoleStr(tsSdbObj.role), mnodeGetMnodeRoleStr(role)); if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) { balanceReset(); @@ -262,24 +262,21 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1); if (processedCount <= 1) { if (pMsg != NULL) { - sdbDebug("app:%p:%p, waiting for confirm this operation, count:%d result:%s", pMsg->rpcMsg.ahandle, pMsg, - processedCount, tstrerror(code)); + sdbDebug("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, processedCount, code); } return; } if (pMsg != NULL) { - sdbDebug("app:%p:%p, is confirmed and will do callback func, result:%s", pMsg->rpcMsg.ahandle, pMsg, - tstrerror(code)); + sdbDebug("vgId:1, msg:%p is confirmed, code:%x", pMsg, code); } // failed to forward, need revert insert if (pOper->retCode != TSDB_CODE_SUCCESS) { SWalHead *pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; int32_t action = pHead->msgType % 10; - sdbError("table:%s record:%p:%s ver:%" PRIu64 ", action:%d failed to foward reason:%s", - ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont), - pHead->version, action, tstrerror(pOper->retCode)); + sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pOper->pObj, + sdbGetKeyStr(pOper->table, pHead->cont), pHead->version, action, tstrerror(pOper->retCode)); if (action == SDB_ACTION_INSERT) { // It's better to create a table in two stages, create it first and then set it success //sdbDeleteHash(pOper->table, pOper); @@ -314,11 +311,11 @@ void sdbUpdateAsync() { void sdbUpdateSync(void *pMnodes) { SMnodeInfos *mnodes = pMnodes; if (!mnodeIsRunning()) { - mDebug("mnode not start yet, update sync config later"); + mDebug("vgId:1, mnode not start yet, update sync config later"); return; } - mDebug("update sync config in sync module, mnodes:%p", pMnodes); + mDebug("vgId:1, update sync config in sync module, mnodes:%p", pMnodes); SSyncCfg syncCfg = {0}; int32_t index = 0; @@ -344,7 +341,7 @@ void sdbUpdateSync(void *pMnodes) { } sdbFreeIter(pIter); syncCfg.replica = index; - mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); + mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { for (index = 0; index < mnodes->mnodeNum; ++index) { SMnodeInfo *node = &mnodes->mnodeInfos[index]; @@ -353,7 +350,7 @@ void sdbUpdateSync(void *pMnodes) { syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; } syncCfg.replica = index; - mDebug("mnodes info input, numOfMnodes:%d", syncCfg.replica); + mDebug("vgId:1, mnodes info input, numOfMnodes:%d", syncCfg.replica); } syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2; @@ -367,18 +364,19 @@ void sdbUpdateSync(void *pMnodes) { } if (!hasThisDnode) { - sdbDebug("update sync config, this dnode not exist"); + sdbDebug("vgId:1, update sync config, this dnode not exist"); return; } if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) { - sdbDebug("update sync config, info not changed"); + sdbDebug("vgId:1, update sync config, info not changed"); return; } - sdbInfo("work as mnode, replica:%d", syncCfg.replica); + sdbInfo("vgId:1, work as mnode, replica:%d", syncCfg.replica); for (int32_t i = 0; i < syncCfg.replica; ++i) { - sdbInfo("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort); + sdbInfo("vgId:1, mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, + syncCfg.nodeInfo[i].nodePort); } SSyncInfo syncInfo = {0}; @@ -427,9 +425,9 @@ void sdbCleanUp() { if (tsSdbObj.status != SDB_STATUS_SERVING) return; tsSdbObj.status = SDB_STATUS_CLOSING; - + sdbCleanupWriteWorker(); - sdbDebug("sdb will be closed, ver:%" PRId64, tsSdbObj.version); + sdbDebug("vgId:1, sdb will be closed, mver:%" PRIu64, tsSdbObj.version); if (tsSdbObj.sync) { syncStop(tsSdbObj.sync); @@ -450,7 +448,7 @@ void sdbIncRef(void *handle, void *pObj) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); - sdbTrace("add ref to table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount); + sdbTrace("vgId:1, sdb:%s, inc ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); } void sdbDecRef(void *handle, void *pObj) { @@ -459,11 +457,11 @@ void sdbDecRef(void *handle, void *pObj) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount); + sdbTrace("vgId:1, sdb:%s, dec ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); int32_t *updateEnd = pObj + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount); + sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); SSdbOper oper = {.pObj = pObj}; (*pTable->destroyFp)(&oper); } @@ -523,13 +521,13 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { atomic_add_fetch_32(&pTable->autoIndex, 1); } - sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, pOper->pMsg); + sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, pOper->pMsg); int32_t code = (*pTable->insertFp)(pOper); if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to insert record:%s to hash, remove it", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj)); sdbDeleteHash(pTable, pOper); } @@ -540,8 +538,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { int32_t *updateEnd = pOper->pObj + pTable->refCountPos - 4; bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { - sdbError("table:%s, failed to delete record:%s from hash, for it already removed", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } @@ -558,9 +556,9 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { pthread_mutex_unlock(&pTable->mutex); atomic_sub_fetch_32(&pTable->numOfRows, 1); - - sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); + + sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); sdbDecRef(pTable, pOper->pObj); @@ -568,8 +566,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { } static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { - sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); + sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; @@ -594,12 +592,12 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { // for data from WAL or forward, version may be smaller if (pHead->version <= tsSdbObj.version) { pthread_mutex_unlock(&tsSdbObj.mutex); - sdbDebug("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64, + sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version); return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbObj.version + 1) { pthread_mutex_unlock(&tsSdbObj.mutex); - sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64, + sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version); return TSDB_CODE_SYN_INVALID_VERSION; } else { @@ -623,19 +621,19 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { if (syncCode <= 0) pOper->processedCount = 1; if (syncCode < 0) { - sdbError("table:%s, failed to forward request, result:%s action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName, + sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); } else if (syncCode > 0) { - sdbDebug("table:%s, forward request is sent, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); } else { - sdbTrace("table:%s, no need to send fwd request, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName, + sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); } return syncCode; } - sdbDebug("table:%s, record from wal/fwd is disposed, action:%s record:%s ver:%" PRId64, pTable->tableName, + sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync @@ -649,7 +647,7 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { } else if (action == SDB_ACTION_DELETE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { - sdbDebug("table:%s, object:%s not exist in hash, ignore delete action", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->tableName, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } @@ -658,7 +656,7 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { } else if (action == SDB_ACTION_UPDATE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { - sdbDebug("table:%s, object:%s not exist in hash, ignore update action", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->tableName, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } @@ -675,8 +673,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (sdbGetRowFromObj(pTable, pOper->pObj)) { - sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, - sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbError("vgId:1, sdb:%s, failed to insert key:%s, already exist", pTable->tableName, + sdbGetObjStr(pTable, pOper->pObj)); sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } @@ -692,7 +690,7 @@ int32_t sdbInsertRow(SSdbOper *pOper) { int32_t code = sdbInsertHash(pTable, pOper); if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to insert into hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to insert into hash", pTable->tableName); return code; } @@ -727,8 +725,8 @@ int32_t sdbInsertRowImp(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbDebug("app:%p:%p, table:%s record:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, + pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); } sdbIncRef(pNewOper->table, pNewOper->pObj); @@ -751,7 +749,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pRow == NULL) { - sdbDebug("table:%s, record is not there, delete failed", pTable->tableName); + sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } @@ -759,7 +757,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { int32_t code = sdbDeleteHash(pTable, pOper); if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to delete from hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->tableName); sdbDecRef(pTable, pOper->pObj); return code; } @@ -795,8 +793,8 @@ int32_t sdbDeleteRowImp(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbDebug("app:%p:%p, table:%s record:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, + pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); } taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); @@ -810,13 +808,13 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pRow == NULL) { - sdbDebug("table:%s, record is not there, update failed", pTable->tableName); + sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } int32_t code = sdbUpdateHash(pTable, pOper); if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to update hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to update hash", pTable->tableName); return code; } @@ -850,8 +848,8 @@ int32_t sdbUpdateRowImp(SSdbOper *pOper) { memcpy(pNewOper, pOper, sizeof(SSdbOper)); if (pNewOper->pMsg != NULL) { - sdbDebug("app:%p:%p, table:%s record:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, + pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); } sdbIncRef(pNewOper->table, pNewOper->pObj); @@ -948,7 +946,7 @@ void sdbCloseTable(void *handle) { taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); - sdbDebug("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables); + sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables); free(pTable); } @@ -964,7 +962,7 @@ int32_t sdbInitWriteWorker() { sdbAllocWriteQueue(); - mInfo("sdb write is opened"); + mInfo("vgId:1, sdb write is opened"); return 0; } @@ -986,7 +984,7 @@ void sdbCleanupWriteWorker() { sdbFreeWritequeue(); tfree(tsSdbPool.writeWorker); - mInfo("sdb write is closed"); + mInfo("vgId:1, sdb write is closed"); } int32_t sdbAllocWriteQueue() { @@ -1072,7 +1070,7 @@ static void *sdbWorkerFp(void *param) { pOper->processedCount = 1; pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; if (pOper->pMsg != NULL) { - sdbDebug("app:%p:%p, table:%s record:%p:%s ver:%" PRIu64 ", will be processed in sdb queue", + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont), pHead->version); } diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7156a2d08a..309d5b1a75 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -106,7 +106,7 @@ typedef struct { int8_t nacks; int8_t confirmed; int32_t code; - uint64_t time; + int64_t time; } SFwdInfo; typedef struct { diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 6ff04aad64..5c56e90af6 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -689,7 +689,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, ver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; @@ -697,7 +697,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (nodeVersion < pMaster->version) { syncRequired = 1; } else { - sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version); + sInfo("%s is master, work as slave, sver:%" PRIu64, pMaster->id, pMaster->version); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); } @@ -854,7 +854,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo * pFwdInfo; - sDebug("%s, forward-rsp is received, code:%x ver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version); + sDebug("%s, forward-rsp is received, code:%x hver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version); SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { @@ -891,7 +891,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id, + sDebug("%s, status msg is received, self:%s sver:%" PRIu64 " peer:%s sver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack); pPeer->version = pPeersStatus->version; @@ -979,7 +979,7 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role], + sDebug("%s, status msg is sent, self:%s sver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack); } else { sDebug("%s, failed to send status msg, restart", pPeer->id); @@ -1154,7 +1154,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { pFwdInfo->time = time; pSyncFwds->fwds++; - sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); + sDebug("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); } static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { @@ -1168,7 +1168,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; pSyncFwds->fwds--; if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last; - // sDebug("vgId:%d, fwd info is removed, ver:%d, fwds:%d", + // sDebug("vgId:%d, fwd info is removed, hver:%d, fwds:%d", // pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); memset(pFwdInfo, 0, sizeof(SFwdInfo)); } @@ -1191,7 +1191,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } if (confirm && pFwdInfo->confirmed == 0) { - sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); + sDebug("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } @@ -1204,14 +1204,17 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - if (pSyncFwds) {; - uint64_t time = taosGetTimestampMs(); + if (pSyncFwds) { + int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds > 0) { pthread_mutex_lock(&(pNode->mutex)); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; - if (time - pFwdInfo->time < 2000) break; + if (ABS(time - pFwdInfo->time) < 2000) break; + + sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, + pFwdInfo->version, time, pFwdInfo->time); syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); } @@ -1234,7 +1237,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if (pWalHead->version > nodeVersion + 1) { - sError("vgId:%d, hver:%" PRIu64 ", inconsistent with ver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion); + sError("vgId:%d, hver:%" PRIu64 ", inconsistent with sver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { sInfo("vgId:%d, restart connection", pNode->vgId); for (int32_t i = 0; i < pNode->replica; ++i) { @@ -1277,9 +1280,9 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + sDebug("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); } else { - sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); + sError("%s, failed to forward, hver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); syncRestartConnection(pPeer); } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 5d7b9eac9b..33bd96ebb3 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -214,7 +214,7 @@ int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { memcpy(pRecv->offset, pHead, len); pRecv->offset += len; pRecv->forwards++; - sDebug("%s, fwd is saved into queue, ver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards); + sDebug("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards); } else { sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize); pRecv->code = -1; // set error code diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 21151f1199..52d1bded31 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -268,7 +268,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi break; } - sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) break; pPeer->sversion = pHead->version; diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 72464d4309..36b3dba165 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -144,9 +144,9 @@ void walFsync(void *handle, bool forceFsync) { if (pWal == NULL || pWal->fd < 0) return; if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name); + wTrace("vgId:%d, fileId:%" PRId64 ", do fsync", pWal->vgId, pWal->fileId); if (fsync(pWal->fd) < 0) { - wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno)); + wError("vgId:%d, fileId:%" PRId64 ", fsync failed since %s", pWal->vgId, pWal->fileId, strerror(errno)); } } } From f38b218247c7466b2c4e5b708dffa7460c2a1c51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 14:45:00 +0800 Subject: [PATCH 04/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 26 +- src/mnode/src/mnodeAcct.c | 28 +-- src/mnode/src/mnodeCluster.c | 28 +-- src/mnode/src/mnodeDb.c | 36 +-- src/mnode/src/mnodeDnode.c | 32 +-- src/mnode/src/mnodeMnode.c | 32 +-- src/mnode/src/mnodeSdb.c | 396 +++++++++++++++--------------- src/mnode/src/mnodeTable.c | 102 ++++---- src/mnode/src/mnodeUser.c | 34 +-- src/mnode/src/mnodeVgroup.c | 48 ++-- src/plugins/http/inc/httpInt.h | 2 +- src/plugins/http/src/httpHandle.c | 4 +- 12 files changed, 384 insertions(+), 384 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index f4854f69a0..938a8a40fd 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -47,7 +47,7 @@ typedef enum { SDB_OPER_LOCAL } ESdbOper; -typedef struct SSdbOper { +typedef struct SSWriteMsg { ESdbOper type; int32_t rowSize; int32_t retCode; // for callback in sdb queue @@ -58,7 +58,7 @@ typedef struct SSdbOper { void * pObj; void * rowData; struct SMnodeMsg *pMsg; -} SSdbOper; +} SSWriteMsg; typedef struct { char *tableName; @@ -67,13 +67,13 @@ typedef struct { int32_t refCountPos; ESdbTable tableId; ESdbKey keyType; - int32_t (*insertFp)(SSdbOper *pOper); - int32_t (*deleteFp)(SSdbOper *pOper); - int32_t (*updateFp)(SSdbOper *pOper); - int32_t (*encodeFp)(SSdbOper *pOper); - int32_t (*decodeFp)(SSdbOper *pDesc); - int32_t (*destroyFp)(SSdbOper *pDesc); - int32_t (*restoredFp)(); + int32_t (*fpInsert)(SSWriteMsg *pWrite); + int32_t (*fpDelete)(SSWriteMsg *pWrite); + int32_t (*fpUpdate)(SSWriteMsg *pWrite); + int32_t (*fpEncode)(SSWriteMsg *pWrite); + int32_t (*fpDecode)(SSWriteMsg *pWrite); + int32_t (*fpDestroy)(SSWriteMsg *pWrite); + int32_t (*fpDestored)(); } SSdbTableDesc; int32_t sdbInit(); @@ -84,10 +84,10 @@ bool sdbIsMaster(); bool sdbIsServing(); void sdbUpdateMnodeRoles(); -int32_t sdbInsertRow(SSdbOper *pOper); -int32_t sdbDeleteRow(SSdbOper *pOper); -int32_t sdbUpdateRow(SSdbOper *pOper); -int32_t sdbInsertRowImp(SSdbOper *pOper); +int32_t sdbInsertRow(SSWriteMsg *pWrite); +int32_t sdbDeleteRow(SSWriteMsg *pWrite); +int32_t sdbUpdateRow(SSWriteMsg *pWrite); +int32_t sdbInsertRowImp(SSWriteMsg *pWrite); void *sdbGetRow(void *handle, void *key); void *sdbFetchRow(void *handle, void *pIter, void **ppRow); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 365cf656de..c6d366ebb3 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -31,14 +31,14 @@ void * tsAcctSdb = NULL; static int32_t tsAcctUpdateSize; static int32_t mnodeCreateRootAcct(); -static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) { +static int32_t mnodeAcctActionDestroy(SSWriteMsg *pOper) { SAcctObj *pAcct = pOper->pObj; pthread_mutex_destroy(&pAcct->mutex); tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionInsert(SSdbOper *pOper) { +static int32_t mnodeAcctActionInsert(SSWriteMsg *pOper) { SAcctObj *pAcct = pOper->pObj; memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo)); pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS; @@ -46,14 +46,14 @@ static int32_t mnodeAcctActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDelete(SSdbOper *pOper) { +static int32_t mnodeAcctActionDelete(SSWriteMsg *pOper) { SAcctObj *pAcct = pOper->pObj; mnodeDropAllUsers(pAcct); mnodeDropAllDbs(pAcct); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) { +static int32_t mnodeAcctActionUpdate(SSWriteMsg *pOper) { SAcctObj *pAcct = pOper->pObj; SAcctObj *pSaved = mnodeGetAcct(pAcct->user); if (pAcct != pSaved) { @@ -64,14 +64,14 @@ static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionEncode(SSdbOper *pOper) { +static int32_t mnodeAcctActionEncode(SSWriteMsg *pOper) { SAcctObj *pAcct = pOper->pObj; memcpy(pOper->rowData, pAcct, tsAcctUpdateSize); pOper->rowSize = tsAcctUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDecode(SSdbOper *pOper) { +static int32_t mnodeAcctActionDecode(SSWriteMsg *pOper) { SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj)); if (pAcct == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -106,13 +106,13 @@ int32_t mnodeInitAccts() { .maxRowSize = tsAcctUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_STRING, - .insertFp = mnodeAcctActionInsert, - .deleteFp = mnodeAcctActionDelete, - .updateFp = mnodeAcctActionUpdate, - .encodeFp = mnodeAcctActionEncode, - .decodeFp = mnodeAcctActionDecode, - .destroyFp = mnodeAcctActionDestroy, - .restoredFp = mnodeAcctActionRestored + .fpInsert = mnodeAcctActionInsert, + .fpDelete = mnodeAcctActionDelete, + .fpUpdate = mnodeAcctActionUpdate, + .fpEncode = mnodeAcctActionEncode, + .fpDecode = mnodeAcctActionDecode, + .fpDestroy = mnodeAcctActionDestroy, + .fpDestored = mnodeAcctActionRestored }; tsAcctSdb = sdbOpenTable(&tableDesc); @@ -226,7 +226,7 @@ static int32_t mnodeCreateRootAcct() { pAcct->acctId = sdbGetId(tsAcctSdb); pAcct->createdTime = taosGetTimestampMs(); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsAcctSdb, .pObj = pAcct, diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 98587cf53d..3b2b668a1f 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -32,31 +32,31 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) { +static int32_t mnodeClusterActionDestroy(SSWriteMsg *pOper) { tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionInsert(SSdbOper *pOper) { +static int32_t mnodeClusterActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDelete(SSdbOper *pOper) { +static int32_t mnodeClusterActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionUpdate(SSdbOper *pOper) { +static int32_t mnodeClusterActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionEncode(SSdbOper *pOper) { +static int32_t mnodeClusterActionEncode(SSWriteMsg *pOper) { SClusterObj *pCluster = pOper->pObj; memcpy(pOper->rowData, pCluster, tsClusterUpdateSize); pOper->rowSize = tsClusterUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDecode(SSdbOper *pOper) { +static int32_t mnodeClusterActionDecode(SSWriteMsg *pOper) { SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj)); if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -91,13 +91,13 @@ int32_t mnodeInitCluster() { .maxRowSize = tsClusterUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_STRING, - .insertFp = mnodeClusterActionInsert, - .deleteFp = mnodeClusterActionDelete, - .updateFp = mnodeClusterActionUpdate, - .encodeFp = mnodeClusterActionEncode, - .decodeFp = mnodeClusterActionDecode, - .destroyFp = mnodeClusterActionDestroy, - .restoredFp = mnodeClusterActionRestored + .fpInsert = mnodeClusterActionInsert, + .fpDelete = mnodeClusterActionDelete, + .fpUpdate = mnodeClusterActionUpdate, + .fpEncode = mnodeClusterActionEncode, + .fpDecode = mnodeClusterActionDecode, + .fpDestroy = mnodeClusterActionDestroy, + .fpDestored = mnodeClusterActionRestored }; tsClusterSdb = sdbOpenTable(&tableDesc); @@ -145,7 +145,7 @@ static int32_t mnodeCreateCluster() { mDebug("uid is %s", pCluster->uid); } - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsClusterSdb, .pObj = pCluster, diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 3b3a21aeb1..403213c22f 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -56,7 +56,7 @@ static void mnodeDestroyDb(SDbObj *pDb) { tfree(pDb); } -static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { +static int32_t mnodeDbActionDestroy(SSWriteMsg *pOper) { mnodeDestroyDb(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -65,7 +65,7 @@ int64_t mnodeGetDbNum() { return sdbGetNumOfRows(tsDbSdb); } -static int32_t mnodeDbActionInsert(SSdbOper *pOper) { +static int32_t mnodeDbActionInsert(SSWriteMsg *pOper) { SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); @@ -91,7 +91,7 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDelete(SSdbOper *pOper) { +static int32_t mnodeDbActionDelete(SSWriteMsg *pOper) { SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); @@ -107,7 +107,7 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionUpdate(SSdbOper *pOper) { +static int32_t mnodeDbActionUpdate(SSWriteMsg *pOper) { SDbObj *pNew = pOper->pObj; SDbObj *pDb = mnodeGetDb(pNew->name); if (pDb != NULL && pNew != pDb) { @@ -120,14 +120,14 @@ static int32_t mnodeDbActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionEncode(SSdbOper *pOper) { +static int32_t mnodeDbActionEncode(SSWriteMsg *pOper) { SDbObj *pDb = pOper->pObj; memcpy(pOper->rowData, pDb, tsDbUpdateSize); pOper->rowSize = tsDbUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDecode(SSdbOper *pOper) { +static int32_t mnodeDbActionDecode(SSWriteMsg *pOper) { SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); if (pDb == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -151,13 +151,13 @@ int32_t mnodeInitDbs() { .maxRowSize = tsDbUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_STRING, - .insertFp = mnodeDbActionInsert, - .deleteFp = mnodeDbActionDelete, - .updateFp = mnodeDbActionUpdate, - .encodeFp = mnodeDbActionEncode, - .decodeFp = mnodeDbActionDecode, - .destroyFp = mnodeDbActionDestroy, - .restoredFp = mnodeDbActionRestored + .fpInsert = mnodeDbActionInsert, + .fpDelete = mnodeDbActionDelete, + .fpUpdate = mnodeDbActionUpdate, + .fpEncode = mnodeDbActionEncode, + .fpDecode = mnodeDbActionDecode, + .fpDestroy = mnodeDbActionDestroy, + .fpDestored = mnodeDbActionRestored }; tsDbSdb = sdbOpenTable(&tableDesc); @@ -412,7 +412,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * pMsg->pDb = pDb; mnodeIncDbRef(pDb); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, @@ -807,7 +807,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { if (pDb->status) return TSDB_CODE_SUCCESS; pDb->status = true; - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb @@ -1019,7 +1019,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; pDb->cfgVersion++; - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, @@ -1071,7 +1071,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { SDbObj *pDb = pMsg->pDb; mInfo("db:%s, drop db from sdb", pDb->name); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, @@ -1134,7 +1134,7 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { if (pDb->pAcct == pAcct) { mInfo("db:%s, drop db from sdb for acct:%s is dropped", pDb->name, pAcct->user); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsDbSdb, .pObj = pDb diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 7e34e08373..0dd43fef89 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -87,12 +87,12 @@ static char* offlineReason[] = { "unknown", }; -static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) { +static int32_t mnodeDnodeActionDestroy(SSWriteMsg *pOper) { tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { +static int32_t mnodeDnodeActionInsert(SSWriteMsg *pOper) { SDnodeObj *pDnode = pOper->pObj; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; @@ -107,7 +107,7 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) { +static int32_t mnodeDnodeActionDelete(SSWriteMsg *pOper) { SDnodeObj *pDnode = pOper->pObj; #ifndef _SYNC @@ -121,7 +121,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { +static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pOper) { SDnodeObj *pNew = pOper->pObj; SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId); if (pDnode != NULL && pNew != pDnode) { @@ -134,14 +134,14 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionEncode(SSdbOper *pOper) { +static int32_t mnodeDnodeActionEncode(SSWriteMsg *pOper) { SDnodeObj *pDnode = pOper->pObj; memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize); pOper->rowSize = tsDnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDecode(SSdbOper *pOper) { +static int32_t mnodeDnodeActionDecode(SSWriteMsg *pOper) { SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); if (pDnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -178,13 +178,13 @@ int32_t mnodeInitDnodes() { .maxRowSize = tsDnodeUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_AUTO, - .insertFp = mnodeDnodeActionInsert, - .deleteFp = mnodeDnodeActionDelete, - .updateFp = mnodeDnodeActionUpdate, - .encodeFp = mnodeDnodeActionEncode, - .decodeFp = mnodeDnodeActionDecode, - .destroyFp = mnodeDnodeActionDestroy, - .restoredFp = mnodeDnodeActionRestored + .fpInsert = mnodeDnodeActionInsert, + .fpDelete = mnodeDnodeActionDelete, + .fpUpdate = mnodeDnodeActionUpdate, + .fpEncode = mnodeDnodeActionEncode, + .fpDecode = mnodeDnodeActionDecode, + .fpDestroy = mnodeDnodeActionDestroy, + .fpDestored = mnodeDnodeActionRestored }; tsDnodeSdb = sdbOpenTable(&tableDesc); @@ -296,7 +296,7 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode) { } void mnodeUpdateDnode(SDnodeObj *pDnode) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDnodeSdb, .pObj = pDnode @@ -644,7 +644,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDnodeSdb, .pObj = pDnode, @@ -665,7 +665,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { } int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsDnodeSdb, .pObj = pDnode, diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 092f246c13..bf9033c5cd 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -58,12 +58,12 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #define mnodeMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock) #endif -static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) { +static int32_t mnodeMnodeActionDestroy(SSWriteMsg *pOper) { tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) { +static int32_t mnodeMnodeActionInsert(SSWriteMsg *pOper) { SMnodeObj *pMnode = pOper->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -76,7 +76,7 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDelete(SSdbOper *pOper) { +static int32_t mnodeMnodeActionDelete(SSWriteMsg *pOper) { SMnodeObj *pMnode = pOper->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); @@ -88,7 +88,7 @@ static int32_t mnodeMnodeActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionUpdate(SSdbOper *pOper) { +static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pOper) { SMnodeObj *pMnode = pOper->pObj; SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId); if (pMnode != pSaved) { @@ -99,14 +99,14 @@ static int32_t mnodeMnodeActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionEncode(SSdbOper *pOper) { +static int32_t mnodeMnodeActionEncode(SSWriteMsg *pOper) { SMnodeObj *pMnode = pOper->pObj; memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize); pOper->rowSize = tsMnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDecode(SSdbOper *pOper) { +static int32_t mnodeMnodeActionDecode(SSWriteMsg *pOper) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); if (pMnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -144,13 +144,13 @@ int32_t mnodeInitMnodes() { .maxRowSize = tsMnodeUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_INT, - .insertFp = mnodeMnodeActionInsert, - .deleteFp = mnodeMnodeActionDelete, - .updateFp = mnodeMnodeActionUpdate, - .encodeFp = mnodeMnodeActionEncode, - .decodeFp = mnodeMnodeActionDecode, - .destroyFp = mnodeMnodeActionDestroy, - .restoredFp = mnodeMnodeActionRestored + .fpInsert = mnodeMnodeActionInsert, + .fpDelete = mnodeMnodeActionDelete, + .fpUpdate = mnodeMnodeActionUpdate, + .fpEncode = mnodeMnodeActionEncode, + .fpDecode = mnodeMnodeActionDecode, + .fpDestroy = mnodeMnodeActionDestroy, + .fpDestored = mnodeMnodeActionRestored }; tsMnodeSdb = sdbOpenTable(&tableDesc); @@ -329,7 +329,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { pMnode->mnodeId = dnodeId; pMnode->createdTime = taosGetTimestampMs(); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsMnodeSdb, .pObj = pMnode, @@ -356,7 +356,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { void mnodeDropMnodeLocal(int32_t dnodeId) { SMnodeObj *pMnode = mnodeGetMnode(dnodeId); if (pMnode != NULL) { - SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode}; + SSWriteMsg oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode}; sdbDeleteRow(&oper); mnodeDecMnodeRef(pMnode); } @@ -371,7 +371,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsMnodeSdb, .pObj = pMnode diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 9c11fa80e8..59812068d4 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -48,7 +48,7 @@ typedef enum { SDB_STATUS_CLOSING } ESdbStatus; -typedef struct _SSdbTable { +typedef struct SSdbTable { char tableName[SDB_TABLE_LEN]; ESdbTable tableId; ESdbKey keyType; @@ -58,13 +58,13 @@ typedef struct _SSdbTable { int32_t autoIndex; int64_t numOfRows; void * iHandle; - int32_t (*insertFp)(SSdbOper *pDesc); - int32_t (*deleteFp)(SSdbOper *pOper); - int32_t (*updateFp)(SSdbOper *pOper); - int32_t (*decodeFp)(SSdbOper *pOper); - int32_t (*encodeFp)(SSdbOper *pOper); - int32_t (*destroyFp)(SSdbOper *pOper); - int32_t (*restoredFp)(); + int32_t (*fpInsert)(SSWriteMsg *pWrite); + int32_t (*fpDelete)(SSWriteMsg *pWrite); + int32_t (*fpUpdate)(SSWriteMsg *pWrite); + int32_t (*fpDecode)(SSWriteMsg *pWrite); + int32_t (*fpEncode)(SSWriteMsg *pWrite); + int32_t (*fpDestroy)(SSWriteMsg *pWrite); + int32_t (*fpDestored)(); pthread_mutex_t mutex; } SSdbTable; @@ -83,33 +83,33 @@ typedef struct { typedef struct { pthread_t thread; int32_t workerId; -} SSdbWriteWorker; +} SSWriteWorker; typedef struct { int32_t num; - SSdbWriteWorker *writeWorker; -} SSdbWriteWorkerPool; + SSWriteWorker *worker; +} SSWriteWorkerPool; extern void * tsMnodeTmr; -static void * tsUpdateSyncTmr; +static void * tsSdbTmr; static SSdbObject tsSdbObj = {0}; -static taos_qset tsSdbWriteQset; -static taos_qall tsSdbWriteQall; -static taos_queue tsSdbWriteQueue; -static SSdbWriteWorkerPool tsSdbPool; +static taos_qset tsSdbWQset; +static taos_qall tsSdbWQall; +static taos_queue tsSdbWQueue; +static SSWriteWorkerPool tsSdbPool; -static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg); +static int32_t sdbWrite(void *wparam, void *data, int32_t type, void *pMsg); static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg); static void * sdbWorkerFp(void *param); static int32_t sdbInitWriteWorker(); static void sdbCleanupWriteWorker(); static int32_t sdbAllocWriteQueue(); static void sdbFreeWritequeue(); -static int32_t sdbUpdateRowImp(SSdbOper *pOper); -static int32_t sdbDeleteRowImp(SSdbOper *pOper); -static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper); -static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper); -static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper); +static int32_t sdbUpdateRowImp(SSWriteMsg *pWrite); +static int32_t sdbDeleteRowImp(SSWriteMsg *pWrite); +static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite); +static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite); +static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite); int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; @@ -199,8 +199,8 @@ static void sdbRestoreTables() { for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; - if (pTable->restoredFp) { - (*pTable->restoredFp)(); + if (pTable->fpDestored) { + (*pTable->fpDestored)(); } totalRows += pTable->numOfRows; @@ -255,11 +255,11 @@ static void sdbNotifyRole(void *ahandle, int8_t role) { FORCE_INLINE static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { assert(param); - SSdbOper * pOper = param; - SMnodeMsg *pMsg = pOper->pMsg; - if (code <= 0) pOper->retCode = code; + SSWriteMsg * pWrite = param; + SMnodeMsg *pMsg = pWrite->pMsg; + if (code <= 0) pWrite->retCode = code; - int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1); + int32_t processedCount = atomic_add_fetch_32(&pWrite->processedCount, 1); if (processedCount <= 1) { if (pMsg != NULL) { sdbDebug("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, processedCount, code); @@ -272,40 +272,40 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { } // failed to forward, need revert insert - if (pOper->retCode != TSDB_CODE_SUCCESS) { - SWalHead *pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; + if (pWrite->retCode != TSDB_CODE_SUCCESS) { + SWalHead *pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; int32_t action = pHead->msgType % 10; - sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pOper->pObj, - sdbGetKeyStr(pOper->table, pHead->cont), pHead->version, action, tstrerror(pOper->retCode)); + sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pWrite->pObj, + sdbGetKeyStr(pWrite->table, pHead->cont), pHead->version, action, tstrerror(pWrite->retCode)); if (action == SDB_ACTION_INSERT) { // It's better to create a table in two stages, create it first and then set it success - //sdbDeleteHash(pOper->table, pOper); - SSdbOper oper = { + //sdbDeleteHash(pWrite->table, pWrite); + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, - .table = pOper->table, - .pObj = pOper->pObj + .table = pWrite->table, + .pObj = pWrite->pObj }; sdbDeleteRow(&oper); } } - if (pOper->writeCb != NULL) { - pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode); + if (pWrite->writeCb != NULL) { + pWrite->retCode = (*pWrite->writeCb)(pMsg, pWrite->retCode); } - dnodeSendRpcMWriteRsp(pMsg, pOper->retCode); + dnodeSendRpcMWriteRsp(pMsg, pWrite->retCode); // if ahandle, means this func is called by sdb write if (ahandle == NULL) { - sdbDecRef(pOper->table, pOper->pObj); + sdbDecRef(pWrite->table, pWrite->pObj); } - taosFreeQitem(pOper); + taosFreeQitem(pWrite); } static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); } void sdbUpdateAsync() { - taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsUpdateSyncTmr); + taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsSdbTmr); } void sdbUpdateSync(void *pMnodes) { @@ -462,8 +462,8 @@ void sdbDecRef(void *handle, void *pObj) { int32_t *updateEnd = pObj + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); - SSdbOper oper = {.pObj = pObj}; - (*pTable->destroyFp)(&oper); + SSWriteMsg oper = {.pObj = pObj}; + (*pTable->fpDestroy)(&oper); } } @@ -500,8 +500,8 @@ static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { return sdbGetRow(pTable, sdbGetObjKey(pTable, key)); } -static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { - void * key = sdbGetObjKey(pTable, pOper->pObj); +static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { + void * key = sdbGetObjKey(pTable, pWrite->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { @@ -509,43 +509,43 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { } pthread_mutex_lock(&pTable->mutex); - taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(int64_t)); + taosHashPut(pTable->iHandle, key, keySize, &pWrite->pObj, sizeof(int64_t)); pthread_mutex_unlock(&pTable->mutex); - sdbIncRef(pTable, pOper->pObj); + sdbIncRef(pTable, pWrite->pObj); atomic_add_fetch_32(&pTable->numOfRows, 1); if (pTable->keyType == SDB_KEY_AUTO) { - pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj)); + pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pWrite->pObj)); } else { atomic_add_fetch_32(&pTable->autoIndex, 1); } sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, pOper->pMsg); + sdbGetObjStr(pTable, pWrite->pObj), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); - int32_t code = (*pTable->insertFp)(pOper); + int32_t code = (*pTable->fpInsert)(pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj)); - sdbDeleteHash(pTable, pOper); + sdbGetObjStr(pTable, pWrite->pObj)); + sdbDeleteHash(pTable, pWrite); } return TSDB_CODE_SUCCESS; } -static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { - int32_t *updateEnd = pOper->pObj + pTable->refCountPos - 4; +static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { + int32_t *updateEnd = pWrite->pObj + pTable->refCountPos - 4; bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj)); + sdbGetObjStr(pTable, pWrite->pObj)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - (*pTable->deleteFp)(pOper); + (*pTable->fpDelete)(pWrite); - void * key = sdbGetObjKey(pTable, pOper->pObj); + void * key = sdbGetObjKey(pTable, pWrite->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); @@ -558,23 +558,23 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { atomic_sub_fetch_32(&pTable->numOfRows, 1); sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); + sdbGetObjStr(pTable, pWrite->pObj), pTable->numOfRows, pWrite->pMsg); - sdbDecRef(pTable, pOper->pObj); + sdbDecRef(pTable, pWrite->pObj); return TSDB_CODE_SUCCESS; } -static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { +static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); + sdbGetObjStr(pTable, pWrite->pObj), pTable->numOfRows, pWrite->pMsg); - (*pTable->updateFp)(pOper); + (*pTable->fpUpdate)(pWrite); return TSDB_CODE_SUCCESS; } static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { - SSdbOper *pOper = param; + SSWriteMsg *pWrite = param; SWalHead *pHead = data; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -614,21 +614,21 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { pthread_mutex_unlock(&tsSdbObj.mutex); // from app, oper is created - if (pOper != NULL) { + if (pWrite != NULL) { // forward to peers - pOper->processedCount = 0; - int32_t syncCode = syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC); - if (syncCode <= 0) pOper->processedCount = 1; + pWrite->processedCount = 0; + int32_t syncCode = syncForwardToPeer(tsSdbObj.sync, pHead, pWrite, TAOS_QTYPE_RPC); + if (syncCode <= 0) pWrite->processedCount = 1; if (syncCode < 0) { sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); + tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else if (syncCode > 0) { sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); + sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else { sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg); + sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } return syncCode; } @@ -637,12 +637,12 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync - syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC); + syncForwardToPeer(tsSdbObj.sync, pHead, pWrite, TAOS_QTYPE_RPC); // from wal or forward msg, oper not created, should add into hash if (action == SDB_ACTION_INSERT) { - SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; - code = (*pTable->decodeFp)(&oper); + SSWriteMsg oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; + code = (*pTable->fpDecode)(&oper); return sdbInsertHash(pTable, &oper); } else if (action == SDB_ACTION_DELETE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); @@ -651,7 +651,7 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSdbOper oper = {.table = pTable, .pObj = pRow}; + SSWriteMsg oper = {.table = pTable, .pObj = pRow}; return sdbDeleteHash(pTable, &oper); } else if (action == SDB_ACTION_UPDATE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); @@ -660,77 +660,77 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; - code = (*pTable->decodeFp)(&oper); + SSWriteMsg oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; + code = (*pTable->fpDecode)(&oper); return sdbUpdateHash(pTable, &oper); } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } } -int32_t sdbInsertRow(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbInsertRow(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - if (sdbGetRowFromObj(pTable, pOper->pObj)) { + if (sdbGetRowFromObj(pTable, pWrite->pObj)) { sdbError("vgId:1, sdb:%s, failed to insert key:%s, already exist", pTable->tableName, - sdbGetObjStr(pTable, pOper->pObj)); - sdbDecRef(pTable, pOper->pObj); + sdbGetObjStr(pTable, pWrite->pObj)); + sdbDecRef(pTable, pWrite->pObj); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } if (pTable->keyType == SDB_KEY_AUTO) { - *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pWrite->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); // let vgId increase from 2 if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { - *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pWrite->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); } } - int32_t code = sdbInsertHash(pTable, pOper); + int32_t code = sdbInsertHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to insert into hash", pTable->tableName); return code; } // just insert data into memory - if (pOper->type != SDB_OPER_GLOBAL) { + if (pWrite->type != SDB_OPER_GLOBAL) { return TSDB_CODE_SUCCESS; } - if (pOper->reqFp) { - return (*pOper->reqFp)(pOper->pMsg); + if (pWrite->reqFp) { + return (*pWrite->reqFp)(pWrite->pMsg); } else { - return sdbInsertRowImp(pOper); + return sdbInsertRowImp(pWrite); } } -int32_t sdbInsertRowImp(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSdbOper *pNewOper = taosAllocateQitem(size); + int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; + SSWriteMsg *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; + SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; pHead->version = 0; - pHead->len = pOper->rowSize; + pHead->len = pWrite->rowSize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; - pOper->rowData = pHead->cont; - (*pTable->encodeFp)(pOper); - pHead->len = pOper->rowSize; + pWrite->rowData = pHead->cont; + (*pTable->fpEncode)(pWrite); + pHead->len = pWrite->rowSize; - memcpy(pNewOper, pOper, sizeof(SSdbOper)); + memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); } sdbIncRef(pNewOper->table, pNewOper->pObj); - taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -743,117 +743,117 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } -int32_t sdbDeleteRow(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbDeleteRow(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); + void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); if (pRow == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - sdbIncRef(pTable, pOper->pObj); + sdbIncRef(pTable, pWrite->pObj); - int32_t code = sdbDeleteHash(pTable, pOper); + int32_t code = sdbDeleteHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->tableName); - sdbDecRef(pTable, pOper->pObj); + sdbDecRef(pTable, pWrite->pObj); return code; } // just delete data from memory - if (pOper->type != SDB_OPER_GLOBAL) { - sdbDecRef(pTable, pOper->pObj); + if (pWrite->type != SDB_OPER_GLOBAL) { + sdbDecRef(pTable, pWrite->pObj); return TSDB_CODE_SUCCESS; } - if (pOper->reqFp) { - return (*pOper->reqFp)(pOper->pMsg); + if (pWrite->reqFp) { + return (*pWrite->reqFp)(pWrite->pMsg); } else { - return sdbDeleteRowImp(pOper); + return sdbDeleteRowImp(pWrite); } } -int32_t sdbDeleteRowImp(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSdbOper *pNewOper = taosAllocateQitem(size); + int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; + SSWriteMsg *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; + SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; - pOper->rowData = pHead->cont; - (*pTable->encodeFp)(pOper); - pHead->len = pOper->rowSize; + pWrite->rowData = pHead->cont; + (*pTable->fpEncode)(pWrite); + pHead->len = pWrite->rowSize; - memcpy(pNewOper, pOper, sizeof(SSdbOper)); + memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); } - taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -int32_t sdbUpdateRow(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbUpdateRow(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); + void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); if (pRow == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - int32_t code = sdbUpdateHash(pTable, pOper); + int32_t code = sdbUpdateHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to update hash", pTable->tableName); return code; } // just update data in memory - if (pOper->type != SDB_OPER_GLOBAL) { + if (pWrite->type != SDB_OPER_GLOBAL) { return TSDB_CODE_SUCCESS; } - if (pOper->reqFp) { - return (*pOper->reqFp)(pOper->pMsg); + if (pWrite->reqFp) { + return (*pWrite->reqFp)(pWrite->pMsg); } else { - return sdbUpdateRowImp(pOper); + return sdbUpdateRowImp(pWrite); } } -int32_t sdbUpdateRowImp(SSdbOper *pOper) { - SSdbTable *pTable = (SSdbTable *)pOper->table; +int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { + SSdbTable *pTable = (SSdbTable *)pWrite->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSdbOper *pNewOper = taosAllocateQitem(size); + int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; + SSWriteMsg *pNewOper = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK; + SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; - pOper->rowData = pHead->cont; - (*pTable->encodeFp)(pOper); - pHead->len = pOper->rowSize; + pWrite->rowData = pHead->cont; + (*pTable->fpEncode)(pWrite); + pHead->len = pWrite->rowSize; - memcpy(pNewOper, pOper, sizeof(SSdbOper)); + memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetObjStr(pTable, pOper->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); } sdbIncRef(pNewOper->table, pNewOper->pObj); - taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -903,13 +903,13 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->hashSessions = pDesc->hashSessions; pTable->maxRowSize = pDesc->maxRowSize; pTable->refCountPos = pDesc->refCountPos; - pTable->insertFp = pDesc->insertFp; - pTable->deleteFp = pDesc->deleteFp; - pTable->updateFp = pDesc->updateFp; - pTable->encodeFp = pDesc->encodeFp; - pTable->decodeFp = pDesc->decodeFp; - pTable->destroyFp = pDesc->destroyFp; - pTable->restoredFp = pDesc->restoredFp; + pTable->fpInsert = pDesc->fpInsert; + pTable->fpDelete = pDesc->fpDelete; + pTable->fpUpdate = pDesc->fpUpdate; + pTable->fpEncode = pDesc->fpEncode; + pTable->fpDecode = pDesc->fpDecode; + pTable->fpDestroy = pDesc->fpDestroy; + pTable->fpDestored = pDesc->fpDestored; _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { @@ -934,12 +934,12 @@ void sdbCloseTable(void *handle) { void **ppRow = taosHashIterGet(pIter); if (ppRow == NULL) continue; - SSdbOper oper = { + SSWriteMsg oper = { .pObj = *ppRow, .table = pTable, }; - (*pTable->destroyFp)(&oper); + (*pTable->fpDestroy)(&oper); } taosHashDestroyIter(pIter); @@ -952,11 +952,11 @@ void sdbCloseTable(void *handle) { int32_t sdbInitWriteWorker() { tsSdbPool.num = 1; - tsSdbPool.writeWorker = (SSdbWriteWorker *)calloc(sizeof(SSdbWriteWorker), tsSdbPool.num); + tsSdbPool.worker = (SSWriteWorker *)calloc(sizeof(SSWriteWorker), tsSdbPool.num); - if (tsSdbPool.writeWorker == NULL) return -1; + if (tsSdbPool.worker == NULL) return -1; for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + SSWriteWorker *pWorker = tsSdbPool.worker + i; pWorker->workerId = i; } @@ -968,45 +968,45 @@ int32_t sdbInitWriteWorker() { void sdbCleanupWriteWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + SSWriteWorker *pWorker = tsSdbPool.worker + i; if (pWorker->thread) { - taosQsetThreadResume(tsSdbWriteQset); + taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + SSWriteWorker *pWorker = tsSdbPool.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); } } sdbFreeWritequeue(); - tfree(tsSdbPool.writeWorker); + tfree(tsSdbPool.worker); mInfo("vgId:1, sdb write is closed"); } int32_t sdbAllocWriteQueue() { - tsSdbWriteQueue = taosOpenQueue(); - if (tsSdbWriteQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; + tsSdbWQueue = taosOpenQueue(); + if (tsSdbWQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - tsSdbWriteQset = taosOpenQset(); - if (tsSdbWriteQset == NULL) { - taosCloseQueue(tsSdbWriteQueue); + tsSdbWQset = taosOpenQset(); + if (tsSdbWQset == NULL) { + taosCloseQueue(tsSdbWQueue); return TSDB_CODE_MND_OUT_OF_MEMORY; } - taosAddIntoQset(tsSdbWriteQset, tsSdbWriteQueue, NULL); + taosAddIntoQset(tsSdbWQset, tsSdbWQueue, NULL); - tsSdbWriteQall = taosAllocateQall(); - if (tsSdbWriteQall == NULL) { - taosCloseQset(tsSdbWriteQset); - taosCloseQueue(tsSdbWriteQueue); + tsSdbWQall = taosAllocateQall(); + if (tsSdbWQall == NULL) { + taosCloseQset(tsSdbWQset); + taosCloseQueue(tsSdbWQueue); return TSDB_CODE_MND_OUT_OF_MEMORY; } for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i; + SSWriteWorker *pWorker = tsSdbPool.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; @@ -1015,9 +1015,9 @@ int32_t sdbAllocWriteQueue() { if (pthread_create(&pWorker->thread, &thAttr, sdbWorkerFp, pWorker) != 0) { mError("failed to create thread to process sdb write queue, reason:%s", strerror(errno)); - taosFreeQall(tsSdbWriteQall); - taosCloseQset(tsSdbWriteQset); - taosCloseQueue(tsSdbWriteQueue); + taosFreeQall(tsSdbWQall); + taosCloseQset(tsSdbWQset); + taosCloseQueue(tsSdbWQueue); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1025,17 +1025,17 @@ int32_t sdbAllocWriteQueue() { mDebug("sdb write worker:%d is launched, total:%d", pWorker->workerId, tsSdbPool.num); } - mDebug("sdb write queue:%p is allocated", tsSdbWriteQueue); + mDebug("sdb write queue:%p is allocated", tsSdbWQueue); return TSDB_CODE_SUCCESS; } void sdbFreeWritequeue() { - taosCloseQueue(tsSdbWriteQueue); - taosFreeQall(tsSdbWriteQall); - taosCloseQset(tsSdbWriteQset); - tsSdbWriteQall = NULL; - tsSdbWriteQset = NULL; - tsSdbWriteQueue = NULL; + taosCloseQueue(tsSdbWQueue); + taosFreeQall(tsSdbWQall); + taosCloseQset(tsSdbWQset); + tsSdbWQall = NULL; + tsSdbWQset = NULL; + tsSdbWQueue = NULL; } int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { @@ -1044,45 +1044,45 @@ int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { SWalHead *pWal = taosAllocateQitem(size); memcpy(pWal, pHead, size); - taosWriteQitem(tsSdbWriteQueue, qtype, pWal); + taosWriteQitem(tsSdbWQueue, qtype, pWal); return 0; } static void *sdbWorkerFp(void *param) { SWalHead *pHead; - SSdbOper *pOper; + SSWriteMsg *pWrite; int32_t type; int32_t numOfMsgs; void * item; void * unUsed; while (1) { - numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed); + numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { - sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWriteQset); + sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWQset); break; } for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWriteQall, &type, &item); + taosGetQitem(tsSdbWQall, &type, &item); if (type == TAOS_QTYPE_RPC) { - pOper = (SSdbOper *)item; - pOper->processedCount = 1; - pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; - if (pOper->pMsg != NULL) { + pWrite = (SSWriteMsg *)item; + pWrite->processedCount = 1; + pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; + if (pWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", - pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, ((SSdbTable *)pOper->table)->tableName, pOper->pObj, - sdbGetKeyStr(pOper->table, pHead->cont), pHead->version); + pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, ((SSdbTable *)pWrite->table)->tableName, pWrite->pObj, + sdbGetKeyStr(pWrite->table, pHead->cont), pHead->version); } } else { pHead = (SWalHead *)item; - pOper = NULL; + pWrite = NULL; } - int32_t code = sdbWrite(pOper, pHead, type, NULL); + int32_t code = sdbWrite(pWrite, pHead, type, NULL); if (code > 0) code = 0; - if (pOper) { - pOper->retCode = code; + if (pWrite) { + pWrite->retCode = code; } else { pHead->len = code; // hackway } @@ -1091,13 +1091,13 @@ static void *sdbWorkerFp(void *param) { walFsync(tsSdbObj.wal, true); // browse all items, and process them one by one - taosResetQitems(tsSdbWriteQall); + taosResetQitems(tsSdbWQall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWriteQall, &type, &item); + taosGetQitem(tsSdbWQall, &type, &item); if (type == TAOS_QTYPE_RPC) { - pOper = (SSdbOper *)item; - sdbConfirmForward(NULL, pOper, pOper->retCode); + pWrite = (SSWriteMsg *)item; + sdbConfirmForward(NULL, pWrite, pWrite->retCode); } else if (type == TAOS_QTYPE_FWD) { pHead = (SWalHead *)item; syncConfirmForward(tsSdbObj.sync, pHead->version, pHead->len); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2b5e6455c0..d3ba149f39 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -99,12 +99,12 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) { tfree(pTable); } -static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { +static int32_t mnodeChildTableActionDestroy(SSWriteMsg *pOper) { mnodeDestroyChildTable(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { +static int32_t mnodeChildTableActionInsert(SSWriteMsg *pOper) { SCTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); @@ -153,7 +153,7 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { +static int32_t mnodeChildTableActionDelete(SSWriteMsg *pOper) { SCTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; @@ -189,7 +189,7 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { +static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pOper) { SCTableObj *pNew = pOper->pObj; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { @@ -216,7 +216,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { +static int32_t mnodeChildTableActionEncode(SSWriteMsg *pOper) { SCTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); @@ -246,7 +246,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { +static int32_t mnodeChildTableActionDecode(SSWriteMsg *pOper) { assert(pOper->rowData != NULL); SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -297,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() { SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); - SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); @@ -309,7 +309,7 @@ static int32_t mnodeChildTableActionRestored() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid); pTable->vgId = 0; - SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -320,7 +320,7 @@ static int32_t mnodeChildTableActionRestored() { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid); pTable->vgId = 0; - SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -331,7 +331,7 @@ static int32_t mnodeChildTableActionRestored() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; - SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -358,13 +358,13 @@ static int32_t mnodeInitChildTables() { .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, - .insertFp = mnodeChildTableActionInsert, - .deleteFp = mnodeChildTableActionDelete, - .updateFp = mnodeChildTableActionUpdate, - .encodeFp = mnodeChildTableActionEncode, - .decodeFp = mnodeChildTableActionDecode, - .destroyFp = mnodeChildTableActionDestroy, - .restoredFp = mnodeChildTableActionRestored + .fpInsert = mnodeChildTableActionInsert, + .fpDelete = mnodeChildTableActionDelete, + .fpUpdate = mnodeChildTableActionUpdate, + .fpEncode = mnodeChildTableActionEncode, + .fpDecode = mnodeChildTableActionDecode, + .fpDestroy = mnodeChildTableActionDestroy, + .fpDestored = mnodeChildTableActionRestored }; tsChildTableSdb = sdbOpenTable(&tableDesc); @@ -430,12 +430,12 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) { tfree(pStable); } -static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionDestroy(SSWriteMsg *pOper) { mnodeDestroySuperTable(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pOper) { SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { @@ -446,7 +446,7 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pOper) { SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { @@ -458,7 +458,7 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pOper) { SSTableObj *pNew = pOper->pObj; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { @@ -483,7 +483,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionEncode(SSWriteMsg *pOper) { SSTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); @@ -506,7 +506,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) { +static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pOper) { assert(pOper->rowData != NULL); SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -551,13 +551,13 @@ static int32_t mnodeInitSuperTables() { .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, - .insertFp = mnodeSuperTableActionInsert, - .deleteFp = mnodeSuperTableActionDelete, - .updateFp = mnodeSuperTableActionUpdate, - .encodeFp = mnodeSuperTableActionEncode, - .decodeFp = mnodeSuperTableActionDecode, - .destroyFp = mnodeSuperTableActionDestroy, - .restoredFp = mnodeSuperTableActionRestored + .fpInsert = mnodeSuperTableActionInsert, + .fpDelete = mnodeSuperTableActionDelete, + .fpUpdate = mnodeSuperTableActionUpdate, + .fpEncode = mnodeSuperTableActionEncode, + .fpDecode = mnodeSuperTableActionDecode, + .fpDestroy = mnodeSuperTableActionDestroy, + .fpDestored = mnodeSuperTableActionRestored }; tsSuperTableSdb = sdbOpenTable(&tableDesc); @@ -828,7 +828,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); - SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb}; sdbDeleteRow(&desc); } @@ -878,7 +878,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pMsg->pTable = (STableObj *)pStable; mnodeIncTableRef(pMsg->pTable); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -937,7 +937,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mnodeDropAllChildTablesInStable(pStable); } - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1010,7 +1010,7 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t mInfo("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, schema[0].name); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1044,7 +1044,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1088,7 +1088,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c mInfo("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldTagName, newTagName); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1162,7 +1162,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1207,7 +1207,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1251,7 +1251,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldName, newName); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, @@ -1417,7 +1417,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsSuperTableSdb, .pObj = pTable, @@ -1694,7 +1694,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); - SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); return code; } @@ -1780,7 +1780,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { pMsg->pTable = (STableObj *)pTable; mnodeIncTableRef(pMsg->pTable); - SSdbOper desc = { + SSWriteMsg desc = { .type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb, @@ -1901,7 +1901,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_APP_ERROR; } - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2005,7 +2005,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2038,7 +2038,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2075,7 +2075,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, oldName, newName); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2218,7 +2218,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { if (pTable == NULL) break; if (pTable->vgId == pVgroup->vgId) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2251,7 +2251,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2280,7 +2280,7 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { if (pTable == NULL) break; if (pTable->superTable == pStable) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, @@ -2410,7 +2410,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { } if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - SSdbOper desc = { + SSWriteMsg desc = { .type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb, @@ -2440,7 +2440,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); - SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; + SSWriteMsg oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&oper); if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index cd02d5a935..edd0839b0c 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -42,12 +42,12 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); -static int32_t mnodeUserActionDestroy(SSdbOper *pOper) { +static int32_t mnodeUserActionDestroy(SSWriteMsg *pOper) { tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionInsert(SSdbOper *pOper) { +static int32_t mnodeUserActionInsert(SSWriteMsg *pOper) { SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); @@ -62,7 +62,7 @@ static int32_t mnodeUserActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDelete(SSdbOper *pOper) { +static int32_t mnodeUserActionDelete(SSWriteMsg *pOper) { SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); @@ -74,7 +74,7 @@ static int32_t mnodeUserActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionUpdate(SSdbOper *pOper) { +static int32_t mnodeUserActionUpdate(SSWriteMsg *pOper) { SUserObj *pUser = pOper->pObj; SUserObj *pSaved = mnodeGetUser(pUser->user); if (pUser != pSaved) { @@ -85,14 +85,14 @@ static int32_t mnodeUserActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionEncode(SSdbOper *pOper) { +static int32_t mnodeUserActionEncode(SSWriteMsg *pOper) { SUserObj *pUser = pOper->pObj; memcpy(pOper->rowData, pUser, tsUserUpdateSize); pOper->rowSize = tsUserUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDecode(SSdbOper *pOper) { +static int32_t mnodeUserActionDecode(SSWriteMsg *pOper) { SUserObj *pUser = (SUserObj *)calloc(1, sizeof(SUserObj)); if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -157,13 +157,13 @@ int32_t mnodeInitUsers() { .maxRowSize = tsUserUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_STRING, - .insertFp = mnodeUserActionInsert, - .deleteFp = mnodeUserActionDelete, - .updateFp = mnodeUserActionUpdate, - .encodeFp = mnodeUserActionEncode, - .decodeFp = mnodeUserActionDecode, - .destroyFp = mnodeUserActionDestroy, - .restoredFp = mnodeUserActionRestored + .fpInsert = mnodeUserActionInsert, + .fpDelete = mnodeUserActionDelete, + .fpUpdate = mnodeUserActionUpdate, + .fpEncode = mnodeUserActionEncode, + .fpDecode = mnodeUserActionDecode, + .fpDestroy = mnodeUserActionDestroy, + .fpDestored = mnodeUserActionRestored }; tsUserSdb = sdbOpenTable(&tableDesc); @@ -205,7 +205,7 @@ void mnodeDecUserRef(SUserObj *pUser) { } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, @@ -259,7 +259,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { pUser->superAuth = 1; } - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, @@ -279,7 +279,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { } static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, @@ -562,7 +562,7 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { if (pUser == NULL) break; if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsUserSdb, .pObj = pUser, diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9e398e94f1..da0ed1cd36 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -72,12 +72,12 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { tfree(pVgroup); } -static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { +static int32_t mnodeVgroupActionDestroy(SSWriteMsg *pOper) { mnodeDestroyVgroup(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { +static int32_t mnodeVgroupActionInsert(SSWriteMsg *pOper) { SVgObj *pVgroup = pOper->pObj; // refer to db @@ -115,7 +115,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) { +static int32_t mnodeVgroupActionDelete(SSWriteMsg *pOper) { SVgObj *pVgroup = pOper->pObj; if (pVgroup->pDb == NULL) { @@ -137,7 +137,7 @@ static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { +static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pOper) { SVgObj *pNew = pOper->pObj; SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); @@ -176,7 +176,7 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionEncode(SSdbOper *pOper) { +static int32_t mnodeVgroupActionEncode(SSWriteMsg *pOper) { SVgObj *pVgroup = pOper->pObj; memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); SVgObj *pTmpVgroup = pOper->rowData; @@ -189,7 +189,7 @@ static int32_t mnodeVgroupActionEncode(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDecode(SSdbOper *pOper) { +static int32_t mnodeVgroupActionDecode(SSWriteMsg *pOper) { SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); if (pVgroup == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -213,13 +213,13 @@ int32_t mnodeInitVgroups() { .maxRowSize = tsVgUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_AUTO, - .insertFp = mnodeVgroupActionInsert, - .deleteFp = mnodeVgroupActionDelete, - .updateFp = mnodeVgroupActionUpdate, - .encodeFp = mnodeVgroupActionEncode, - .decodeFp = mnodeVgroupActionDecode, - .destroyFp = mnodeVgroupActionDestroy, - .restoredFp = mnodeVgroupActionRestored, + .fpInsert = mnodeVgroupActionInsert, + .fpDelete = mnodeVgroupActionDelete, + .fpUpdate = mnodeVgroupActionUpdate, + .fpEncode = mnodeVgroupActionEncode, + .fpDecode = mnodeVgroupActionDecode, + .fpDestroy = mnodeVgroupActionDestroy, + .fpDestored = mnodeVgroupActionRestored, }; tsVgroupSdb = sdbOpenTable(&tableDesc); @@ -253,7 +253,7 @@ SVgObj *mnodeGetVgroup(int32_t vgId) { } void mnodeUpdateVgroup(SVgObj *pVgroup) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup @@ -519,14 +519,14 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, tstrerror(code)); - SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; sdbDeleteRow(&desc); return code; } else { mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); pVgroup->status = TAOS_VG_STATUS_READY; - SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; (void)sdbUpdateRow(&desc); dnodeReprocessMWriteMsg(pMsg); @@ -535,7 +535,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, // pDb->name, pVgroup->numOfVnodes); // pVgroup->status = TAOS_VG_STATUS_READY; - // SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -571,7 +571,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { pMsg->pVgroup = pVgroup; mnodeIncVgroupRef(pVgroup); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, @@ -595,7 +595,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mDebug("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mnodeSendDropVgroupMsg(pVgroup, NULL); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup @@ -957,7 +957,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; if (mnodeMsg->received == mnodeMsg->successed) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, @@ -973,7 +973,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup @@ -1031,7 +1031,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup @@ -1084,7 +1084,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) { mnodeDropAllChildTablesInVgroups(pVgroup); - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsVgroupSdb, .pObj = pVgroup, @@ -1135,7 +1135,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { if (pVgroup == NULL) break; if (pVgroup->pDb == pDropDb) { - SSdbOper oper = { + SSWriteMsg oper = { .type = SDB_OPER_LOCAL, .table = tsVgroupSdb, .pObj = pVgroup, diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index affc0e838e..ebdfabf310 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -118,7 +118,7 @@ typedef struct { typedef struct { char *module; - bool (*decodeFp)(struct HttpContext *pContext); + bool (*fpDecode)(struct HttpContext *pContext); } HttpDecodeMethod; typedef struct { diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index b50217cfc4..7c56507514 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -21,11 +21,11 @@ #include "httpHandle.h" bool httpDecodeRequest(HttpContext* pContext) { - if (pContext->decodeMethod->decodeFp == NULL) { + if (pContext->decodeMethod->fpDecode == NULL) { return false; } - return (*pContext->decodeMethod->decodeFp)(pContext); + return (*pContext->decodeMethod->fpDecode)(pContext); } /** From 97e41cdf8db6ea95a94cb13ae3fbd610fdf01bb1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 15:39:57 +0800 Subject: [PATCH 05/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 52 +++--- src/mnode/src/mnodeAcct.c | 42 ++--- src/mnode/src/mnodeCluster.c | 34 ++-- src/mnode/src/mnodeDb.c | 100 +++++------ src/mnode/src/mnodeDnode.c | 66 +++---- src/mnode/src/mnodeMnode.c | 58 +++--- src/mnode/src/mnodeSdb.c | 199 ++++++++++----------- src/mnode/src/mnodeTable.c | 334 +++++++++++++++++------------------ src/mnode/src/mnodeUser.c | 78 ++++---- src/mnode/src/mnodeVgroup.c | 130 +++++++------- tests/script/sh/deploy.sh | 28 +-- 11 files changed, 562 insertions(+), 559 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 938a8a40fd..0b0933c08a 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -20,7 +20,9 @@ extern "C" { #endif -struct SMnodeMsg; +#include "mnode.h" + +struct SSdbTable; typedef enum { SDB_TABLE_CLUSTER = 0, @@ -36,35 +38,35 @@ typedef enum { } ESdbTable; typedef enum { - SDB_KEY_STRING, - SDB_KEY_INT, - SDB_KEY_AUTO, - SDB_KEY_VAR_STRING, + SDB_KEY_STRING = 0, + SDB_KEY_INT = 1, + SDB_KEY_AUTO = 2, + SDB_KEY_VAR_STRING = 3, } ESdbKey; typedef enum { - SDB_OPER_GLOBAL, - SDB_OPER_LOCAL + SDB_OPER_GLOBAL = 0, + SDB_OPER_LOCAL = 1 } ESdbOper; typedef struct SSWriteMsg { ESdbOper type; + int32_t processedCount; // for sync fwd callback + int32_t retCode; // for callback in sdb queue int32_t rowSize; - int32_t retCode; // for callback in sdb queue - int32_t processedCount; // for sync fwd callback - int32_t (*reqFp)(struct SMnodeMsg *pMsg); - int32_t (*writeCb)(struct SMnodeMsg *pMsg, int32_t code); - void * table; - void * pObj; void * rowData; - struct SMnodeMsg *pMsg; + int32_t (*fpReq)(SMnodeMsg *pMsg); + int32_t (*fpWrite)(SMnodeMsg *pMsg, int32_t code); + void * pObj; + SMnodeMsg *pMsg; + struct SSdbTable *pTable; } SSWriteMsg; typedef struct { - char *tableName; - int32_t hashSessions; - int32_t maxRowSize; - int32_t refCountPos; + char * tableName; + int32_t hashSessions; + int32_t maxRowSize; + int32_t refCountPos; ESdbTable tableId; ESdbKey keyType; int32_t (*fpInsert)(SSWriteMsg *pWrite); @@ -89,15 +91,15 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite); int32_t sdbUpdateRow(SSWriteMsg *pWrite); int32_t sdbInsertRowImp(SSWriteMsg *pWrite); -void *sdbGetRow(void *handle, void *key); -void *sdbFetchRow(void *handle, void *pIter, void **ppRow); +void *sdbGetRow(void *pTable, void *key); +void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); void sdbFreeIter(void *pIter); -void sdbIncRef(void *thandle, void *pRow); -void sdbDecRef(void *thandle, void *pRow); -int64_t sdbGetNumOfRows(void *handle); -int32_t sdbGetId(void *handle); +void sdbIncRef(void *pTable, void *pRow); +void sdbDecRef(void *pTable, void *pRow); +int64_t sdbGetNumOfRows(void *pTable); +int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); -bool sdbCheckRowDeleted(void *thandle, void *pRow); +bool sdbCheckRowDeleted(void *pTable, void *pRow); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index c6d366ebb3..697ff1d5a1 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -31,30 +31,30 @@ void * tsAcctSdb = NULL; static int32_t tsAcctUpdateSize; static int32_t mnodeCreateRootAcct(); -static int32_t mnodeAcctActionDestroy(SSWriteMsg *pOper) { - SAcctObj *pAcct = pOper->pObj; +static int32_t mnodeAcctActionDestroy(SSWriteMsg *pWMsg) { + SAcctObj *pAcct = pWMsg->pObj; pthread_mutex_destroy(&pAcct->mutex); - tfree(pOper->pObj); + tfree(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionInsert(SSWriteMsg *pOper) { - SAcctObj *pAcct = pOper->pObj; +static int32_t mnodeAcctActionInsert(SSWriteMsg *pWMsg) { + SAcctObj *pAcct = pWMsg->pObj; memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo)); pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS; pthread_mutex_init(&pAcct->mutex, NULL); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDelete(SSWriteMsg *pOper) { - SAcctObj *pAcct = pOper->pObj; +static int32_t mnodeAcctActionDelete(SSWriteMsg *pWMsg) { + SAcctObj *pAcct = pWMsg->pObj; mnodeDropAllUsers(pAcct); mnodeDropAllDbs(pAcct); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionUpdate(SSWriteMsg *pOper) { - SAcctObj *pAcct = pOper->pObj; +static int32_t mnodeAcctActionUpdate(SSWriteMsg *pWMsg) { + SAcctObj *pAcct = pWMsg->pObj; SAcctObj *pSaved = mnodeGetAcct(pAcct->user); if (pAcct != pSaved) { memcpy(pSaved, pAcct, tsAcctUpdateSize); @@ -64,19 +64,19 @@ static int32_t mnodeAcctActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionEncode(SSWriteMsg *pOper) { - SAcctObj *pAcct = pOper->pObj; - memcpy(pOper->rowData, pAcct, tsAcctUpdateSize); - pOper->rowSize = tsAcctUpdateSize; +static int32_t mnodeAcctActionEncode(SSWriteMsg *pWMsg) { + SAcctObj *pAcct = pWMsg->pObj; + memcpy(pWMsg->rowData, pAcct, tsAcctUpdateSize); + pWMsg->rowSize = tsAcctUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeAcctActionDecode(SSWriteMsg *pWMsg) { SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj)); if (pAcct == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pAcct, pOper->rowData, tsAcctUpdateSize); - pOper->pObj = pAcct; + memcpy(pAcct, pWMsg->rowData, tsAcctUpdateSize); + pWMsg->pObj = pAcct; return TSDB_CODE_SUCCESS; } @@ -226,13 +226,13 @@ static int32_t mnodeCreateRootAcct() { pAcct->acctId = sdbGetId(tsAcctSdb); pAcct->createdTime = taosGetTimestampMs(); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsAcctSdb, - .pObj = pAcct, + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, }; - return sdbInsertRow(&oper); + return sdbInsertRow(&wmsg); } #ifndef _ACCT diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 3b2b668a1f..c94c833270 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -32,36 +32,36 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mnodeClusterActionDestroy(SSWriteMsg *pOper) { - tfree(pOper->pObj); +static int32_t mnodeClusterActionDestroy(SSWriteMsg *pWMsg) { + tfree(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionInsert(SSWriteMsg *pOper) { +static int32_t mnodeClusterActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDelete(SSWriteMsg *pOper) { +static int32_t mnodeClusterActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionUpdate(SSWriteMsg *pOper) { +static int32_t mnodeClusterActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionEncode(SSWriteMsg *pOper) { - SClusterObj *pCluster = pOper->pObj; - memcpy(pOper->rowData, pCluster, tsClusterUpdateSize); - pOper->rowSize = tsClusterUpdateSize; +static int32_t mnodeClusterActionEncode(SSWriteMsg *pWMsg) { + SClusterObj *pCluster = pWMsg->pObj; + memcpy(pWMsg->rowData, pCluster, tsClusterUpdateSize); + pWMsg->rowSize = tsClusterUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeClusterActionDecode(SSWriteMsg *pWMsg) { SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj)); if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pCluster, pOper->rowData, tsClusterUpdateSize); - pOper->pObj = pCluster; + memcpy(pCluster, pWMsg->rowData, tsClusterUpdateSize); + pWMsg->pObj = pCluster; return TSDB_CODE_SUCCESS; } @@ -145,13 +145,13 @@ static int32_t mnodeCreateCluster() { mDebug("uid is %s", pCluster->uid); } - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsClusterSdb, - .pObj = pCluster, + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, }; - return sdbInsertRow(&oper); + return sdbInsertRow(&wmsg); } const char* mnodeGetClusterId() { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 403213c22f..3c2bfbb834 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -56,8 +56,8 @@ static void mnodeDestroyDb(SDbObj *pDb) { tfree(pDb); } -static int32_t mnodeDbActionDestroy(SSWriteMsg *pOper) { - mnodeDestroyDb(pOper->pObj); +static int32_t mnodeDbActionDestroy(SSWriteMsg *pWMsg) { + mnodeDestroyDb(pWMsg->pObj); return TSDB_CODE_SUCCESS; } @@ -65,8 +65,8 @@ int64_t mnodeGetDbNum() { return sdbGetNumOfRows(tsDbSdb); } -static int32_t mnodeDbActionInsert(SSWriteMsg *pOper) { - SDbObj *pDb = pOper->pObj; +static int32_t mnodeDbActionInsert(SSWriteMsg *pWMsg) { + SDbObj *pDb = pWMsg->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); pthread_mutex_init(&pDb->mutex, NULL); @@ -91,8 +91,8 @@ static int32_t mnodeDbActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDelete(SSWriteMsg *pOper) { - SDbObj *pDb = pOper->pObj; +static int32_t mnodeDbActionDelete(SSWriteMsg *pWMsg) { + SDbObj *pDb = pWMsg->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); mnodeDropAllChildTables(pDb); @@ -107,11 +107,11 @@ static int32_t mnodeDbActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionUpdate(SSWriteMsg *pOper) { - SDbObj *pNew = pOper->pObj; +static int32_t mnodeDbActionUpdate(SSWriteMsg *pWMsg) { + SDbObj *pNew = pWMsg->pObj; SDbObj *pDb = mnodeGetDb(pNew->name); if (pDb != NULL && pNew != pDb) { - memcpy(pDb, pNew, pOper->rowSize); + memcpy(pDb, pNew, pWMsg->rowSize); free(pNew->vgList); free(pNew); } @@ -120,19 +120,19 @@ static int32_t mnodeDbActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionEncode(SSWriteMsg *pOper) { - SDbObj *pDb = pOper->pObj; - memcpy(pOper->rowData, pDb, tsDbUpdateSize); - pOper->rowSize = tsDbUpdateSize; +static int32_t mnodeDbActionEncode(SSWriteMsg *pWMsg) { + SDbObj *pDb = pWMsg->pObj; + memcpy(pWMsg->rowData, pDb, tsDbUpdateSize); + pWMsg->rowSize = tsDbUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeDbActionDecode(SSWriteMsg *pWMsg) { SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); if (pDb == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pDb, pOper->rowData, tsDbUpdateSize); - pOper->pObj = pDb; + memcpy(pDb, pWMsg->rowData, tsDbUpdateSize); + pWMsg->pObj = pDb; return TSDB_CODE_SUCCESS; } @@ -412,16 +412,16 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * pMsg->pDb = pDb; mnodeIncDbRef(pDb); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, - .rowSize = sizeof(SDbObj), - .pMsg = pMsg, - .writeCb = mnodeCreateDbCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + .pMsg = pMsg, + .fpWrite = mnodeCreateDbCb }; - code = sdbInsertRow(&oper); + code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); pMsg->pDb = NULL; @@ -440,8 +440,8 @@ bool mnodeCheckIsMonitorDB(char *db, char *monitordb) { } #if 0 -void mnodePrintVgroups(SDbObj *pDb, char *oper) { - mInfo("db:%s, vgroup link from head, oper:%s", pDb->name, oper); +void mnodePrintVgroups(SDbObj *pDb, char *wmsg) { + mInfo("db:%s, vgroup link from head, wmsg:%s", pDb->name, wmsg); SVgObj *pVgroup = pDb->pHead; while (pVgroup != NULL) { mInfo("vgId:%d", pVgroup->vgId); @@ -807,13 +807,13 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { if (pDb->status) return TSDB_CODE_SUCCESS; pDb->status = true; - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb }; - int32_t code = sdbUpdateRow(&oper); + int32_t code = sdbUpdateRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to set dropping state, reason:%s", pDb->name, tstrerror(code)); } @@ -1019,15 +1019,15 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; pDb->cfgVersion++; - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, - .pMsg = pMsg, - .writeCb = mnodeAlterDbCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .pMsg = pMsg, + .fpWrite = mnodeAlterDbCb }; - code = sdbUpdateRow(&oper); + code = sdbUpdateRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter, reason:%s", pDb->name, tstrerror(code)); } @@ -1071,15 +1071,15 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { SDbObj *pDb = pMsg->pDb; mInfo("db:%s, drop db from sdb", pDb->name); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, - .pMsg = pMsg, - .writeCb = mnodeDropDbCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .pMsg = pMsg, + .fpWrite = mnodeDropDbCb }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to drop, reason:%s", pDb->name, tstrerror(code)); } @@ -1134,13 +1134,13 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { if (pDb->pAcct == pAcct) { mInfo("db:%s, drop db from sdb for acct:%s is dropped", pDb->name, pAcct->user); - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsDbSdb, - .pObj = pDb + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsDbSdb, + .pObj = pDb }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfDbs++; } mnodeDecDbRef(pDb); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 0dd43fef89..617499967e 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -87,13 +87,13 @@ static char* offlineReason[] = { "unknown", }; -static int32_t mnodeDnodeActionDestroy(SSWriteMsg *pOper) { - tfree(pOper->pObj); +static int32_t mnodeDnodeActionDestroy(SSWriteMsg *pWMsg) { + tfree(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionInsert(SSWriteMsg *pOper) { - SDnodeObj *pDnode = pOper->pObj; +static int32_t mnodeDnodeActionInsert(SSWriteMsg *pWMsg) { + SDnodeObj *pDnode = pWMsg->pObj; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->lastAccess = tsAccessSquence; @@ -107,8 +107,8 @@ static int32_t mnodeDnodeActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDelete(SSWriteMsg *pOper) { - SDnodeObj *pDnode = pOper->pObj; +static int32_t mnodeDnodeActionDelete(SSWriteMsg *pWMsg) { + SDnodeObj *pDnode = pWMsg->pObj; #ifndef _SYNC mnodeDropAllDnodeVgroups(pDnode); @@ -121,11 +121,11 @@ static int32_t mnodeDnodeActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pOper) { - SDnodeObj *pNew = pOper->pObj; +static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pWMsg) { + SDnodeObj *pNew = pWMsg->pObj; SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId); if (pDnode != NULL && pNew != pDnode) { - memcpy(pDnode, pNew, pOper->rowSize); + memcpy(pDnode, pNew, pWMsg->rowSize); free(pNew); } mnodeDecDnodeRef(pDnode); @@ -134,19 +134,19 @@ static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionEncode(SSWriteMsg *pOper) { - SDnodeObj *pDnode = pOper->pObj; - memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize); - pOper->rowSize = tsDnodeUpdateSize; +static int32_t mnodeDnodeActionEncode(SSWriteMsg *pWMsg) { + SDnodeObj *pDnode = pWMsg->pObj; + memcpy(pWMsg->rowData, pDnode, tsDnodeUpdateSize); + pWMsg->rowSize = tsDnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeDnodeActionDecode(SSWriteMsg *pWMsg) { SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); if (pDnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pDnode, pOper->rowData, tsDnodeUpdateSize); - pOper->pObj = pDnode; + memcpy(pDnode, pWMsg->rowData, tsDnodeUpdateSize); + pWMsg->pObj = pDnode; return TSDB_CODE_SUCCESS; } @@ -296,13 +296,13 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode) { } void mnodeUpdateDnode(SDnodeObj *pDnode) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDnodeSdb, - .pObj = pDnode + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode }; - int32_t code = sdbUpdateRow(&oper); + int32_t code = sdbUpdateRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnodeId:%d, failed update", pDnode->dnodeId); } @@ -644,15 +644,15 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDnodeSdb, - .pObj = pDnode, + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, .rowSize = sizeof(SDnodeObj), - .pMsg = pMsg + .pMsg = pMsg }; - int32_t code = sdbInsertRow(&oper); + int32_t code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { int dnodeId = pDnode->dnodeId; tfree(pDnode); @@ -665,14 +665,14 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { } int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDnodeSdb, - .pObj = pDnode, - .pMsg = pMsg + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop from cluster, result:%s", pDnode->dnodeId, tstrerror(code)); } else { diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index bf9033c5cd..37915319e5 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -58,13 +58,13 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #define mnodeMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock) #endif -static int32_t mnodeMnodeActionDestroy(SSWriteMsg *pOper) { - tfree(pOper->pObj); +static int32_t mnodeMnodeActionDestroy(SSWriteMsg *pWMsg) { + tfree(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionInsert(SSWriteMsg *pOper) { - SMnodeObj *pMnode = pOper->pObj; +static int32_t mnodeMnodeActionInsert(SSWriteMsg *pWMsg) { + SMnodeObj *pMnode = pWMsg->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -76,8 +76,8 @@ static int32_t mnodeMnodeActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDelete(SSWriteMsg *pOper) { - SMnodeObj *pMnode = pOper->pObj; +static int32_t mnodeMnodeActionDelete(SSWriteMsg *pWMsg) { + SMnodeObj *pMnode = pWMsg->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -88,30 +88,30 @@ static int32_t mnodeMnodeActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pOper) { - SMnodeObj *pMnode = pOper->pObj; +static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pWMsg) { + SMnodeObj *pMnode = pWMsg->pObj; SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId); if (pMnode != pSaved) { - memcpy(pSaved, pMnode, pOper->rowSize); + memcpy(pSaved, pMnode, pWMsg->rowSize); free(pMnode); } mnodeDecMnodeRef(pSaved); return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionEncode(SSWriteMsg *pOper) { - SMnodeObj *pMnode = pOper->pObj; - memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize); - pOper->rowSize = tsMnodeUpdateSize; +static int32_t mnodeMnodeActionEncode(SSWriteMsg *pWMsg) { + SMnodeObj *pMnode = pWMsg->pObj; + memcpy(pWMsg->rowData, pMnode, tsMnodeUpdateSize); + pWMsg->rowSize = tsMnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeMnodeActionDecode(SSWriteMsg *pWMsg) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); if (pMnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pMnode, pOper->rowData, tsMnodeUpdateSize); - pOper->pObj = pMnode; + memcpy(pMnode, pWMsg->rowData, tsMnodeUpdateSize); + pWMsg->pObj = pMnode; return TSDB_CODE_SUCCESS; } @@ -329,11 +329,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { pMnode->mnodeId = dnodeId; pMnode->createdTime = taosGetTimestampMs(); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsMnodeSdb, - .pObj = pMnode, - .writeCb = mnodeCreateMnodeCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .fpWrite = mnodeCreateMnodeCb }; int32_t code = TSDB_CODE_SUCCESS; @@ -346,7 +346,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { return; } - code = sdbInsertRow(&oper); + code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code)); tfree(pMnode); @@ -356,8 +356,8 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { void mnodeDropMnodeLocal(int32_t dnodeId) { SMnodeObj *pMnode = mnodeGetMnode(dnodeId); if (pMnode != NULL) { - SSWriteMsg oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode}; - sdbDeleteRow(&oper); + SSWriteMsg wmsg = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pObj = pMnode}; + sdbDeleteRow(&wmsg); mnodeDecMnodeRef(pMnode); } @@ -371,13 +371,13 @@ int32_t mnodeDropMnode(int32_t dnodeId) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsMnodeSdb, - .pObj = pMnode + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); sdbDecRef(tsMnodeSdb, pMnode); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 59812068d4..034ff870c5 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -83,12 +83,12 @@ typedef struct { typedef struct { pthread_t thread; int32_t workerId; -} SSWriteWorker; +} SSdbWorker; typedef struct { int32_t num; - SSWriteWorker *worker; -} SSWriteWorkerPool; + SSdbWorker *worker; +} SSdbWorkerPool; extern void * tsMnodeTmr; static void * tsSdbTmr; @@ -96,27 +96,28 @@ static SSdbObject tsSdbObj = {0}; static taos_qset tsSdbWQset; static taos_qall tsSdbWQall; static taos_queue tsSdbWQueue; -static SSWriteWorkerPool tsSdbPool; +static SSdbWorkerPool tsSdbPool; -static int32_t sdbWrite(void *wparam, void *data, int32_t type, void *pMsg); -static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg); -static void * sdbWorkerFp(void *param); -static int32_t sdbInitWriteWorker(); -static void sdbCleanupWriteWorker(); -static int32_t sdbAllocWriteQueue(); -static void sdbFreeWritequeue(); +static int32_t sdbWrite(void *pWrite, void *pHead, int32_t qtype, void *unused); +static int32_t sdbWriteToQueue(void *pWrite, void *pHead, int32_t qtype, void *unused); +static void * sdbWorkerFp(void *pWorker); +static int32_t sdbInitWorker(); +static void sdbCleanupWorker(); +static int32_t sdbAllocQueue(); +static void sdbFreeQueue(); +extern int32_t sdbInsertRowImp(SSWriteMsg *pWrite); static int32_t sdbUpdateRowImp(SSWriteMsg *pWrite); static int32_t sdbDeleteRowImp(SSWriteMsg *pWrite); static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite); static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite); static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite); -int32_t sdbGetId(void *handle) { - return ((SSdbTable *)handle)->autoIndex; +int32_t sdbGetId(void *pTable) { + return ((SSdbTable *)pTable)->autoIndex; } -int64_t sdbGetNumOfRows(void *handle) { - return ((SSdbTable *)handle)->numOfRows; +int64_t sdbGetNumOfRows(void *pTable) { + return ((SSdbTable *)pTable)->numOfRows; } uint64_t sdbGetVersion() { @@ -276,27 +277,27 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { SWalHead *pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; int32_t action = pHead->msgType % 10; sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pWrite->pObj, - sdbGetKeyStr(pWrite->table, pHead->cont), pHead->version, action, tstrerror(pWrite->retCode)); + sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, action, tstrerror(pWrite->retCode)); if (action == SDB_ACTION_INSERT) { // It's better to create a table in two stages, create it first and then set it success - //sdbDeleteHash(pWrite->table, pWrite); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = pWrite->table, - .pObj = pWrite->pObj + //sdbDeleteHash(pWrite->pTable, pWrite); + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = pWrite->pTable, + .pObj = pWrite->pObj }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); } } - if (pWrite->writeCb != NULL) { - pWrite->retCode = (*pWrite->writeCb)(pMsg, pWrite->retCode); + if (pWrite->fpWrite != NULL) { + pWrite->retCode = (*pWrite->fpWrite)(pMsg, pWrite->retCode); } dnodeSendRpcMWriteRsp(pMsg, pWrite->retCode); // if ahandle, means this func is called by sdb write if (ahandle == NULL) { - sdbDecRef(pWrite->table, pWrite->pObj); + sdbDecRef(pWrite->pTable, pWrite->pObj); } taosFreeQitem(pWrite); @@ -403,7 +404,7 @@ void sdbUpdateSync(void *pMnodes) { int32_t sdbInit() { pthread_mutex_init(&tsSdbObj.mutex, NULL); - if (sdbInitWriteWorker() != 0) { + if (sdbInitWorker() != 0) { return -1; } @@ -426,7 +427,7 @@ void sdbCleanUp() { tsSdbObj.status = SDB_STATUS_CLOSING; - sdbCleanupWriteWorker(); + sdbCleanupWorker(); sdbDebug("vgId:1, sdb will be closed, mver:%" PRIu64, tsSdbObj.version); if (tsSdbObj.sync) { @@ -442,19 +443,19 @@ void sdbCleanUp() { pthread_mutex_destroy(&tsSdbObj.mutex); } -void sdbIncRef(void *handle, void *pObj) { - if (pObj == NULL || handle == NULL) return; +void sdbIncRef(void *tparam, void *pObj) { + if (pObj == NULL || tparam == NULL) return; - SSdbTable *pTable = handle; + SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, inc ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); } -void sdbDecRef(void *handle, void *pObj) { - if (pObj == NULL || handle == NULL) return; +void sdbDecRef(void *tparam, void *pObj) { + if (pObj == NULL || tparam == NULL) return; - SSdbTable *pTable = handle; + SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, dec ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); @@ -462,8 +463,8 @@ void sdbDecRef(void *handle, void *pObj) { int32_t *updateEnd = pObj + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); - SSWriteMsg oper = {.pObj = pObj}; - (*pTable->fpDestroy)(&oper); + SSWriteMsg wmsg = {.pObj = pObj}; + (*pTable->fpDestroy)(&wmsg); } } @@ -485,12 +486,12 @@ static void *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) { return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key)); } -void *sdbGetRow(void *handle, void *key) { - SSdbTable *pTable = handle; +void *sdbGetRow(void *tparam, void *key) { + SSdbTable *pTable = tparam; pthread_mutex_lock(&pTable->mutex); - void *pRow = sdbGetRowMeta(handle, key); - if (pRow) sdbIncRef(handle, pRow); + void *pRow = sdbGetRowMeta(pTable, key); + if (pRow) sdbIncRef(pTable, pRow); pthread_mutex_unlock(&pTable->mutex); return pRow; @@ -573,9 +574,9 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { return TSDB_CODE_SUCCESS; } -static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { - SSWriteMsg *pWrite = param; - SWalHead *pHead = data; +static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { + SSWriteMsg *pWrite = wparam; + SWalHead *pHead = hparam; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -593,12 +594,12 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { if (pHead->version <= tsSdbObj.version) { pthread_mutex_unlock(&tsSdbObj.mutex); sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version); + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbObj.version); return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbObj.version + 1) { pthread_mutex_unlock(&tsSdbObj.mutex); sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version); + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbObj.version); return TSDB_CODE_SYN_INVALID_VERSION; } else { tsSdbObj.version = pHead->version; @@ -613,7 +614,7 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { pthread_mutex_unlock(&tsSdbObj.mutex); - // from app, oper is created + // from app, wmsg is created if (pWrite != NULL) { // forward to peers pWrite->processedCount = 0; @@ -639,11 +640,11 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { // even it is WAL/FWD, it shall be called to update version in sync syncForwardToPeer(tsSdbObj.sync, pHead, pWrite, TAOS_QTYPE_RPC); - // from wal or forward msg, oper not created, should add into hash + // from wal or forward msg, wmsg not created, should add into hash if (action == SDB_ACTION_INSERT) { - SSWriteMsg oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; - code = (*pTable->fpDecode)(&oper); - return sdbInsertHash(pTable, &oper); + SSWriteMsg wmsg = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + code = (*pTable->fpDecode)(&wmsg); + return sdbInsertHash(pTable, &wmsg); } else if (action == SDB_ACTION_DELETE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { @@ -651,8 +652,8 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSWriteMsg oper = {.table = pTable, .pObj = pRow}; - return sdbDeleteHash(pTable, &oper); + SSWriteMsg wmsg = {.pTable = pTable, .pObj = pRow}; + return sdbDeleteHash(pTable, &wmsg); } else if (action == SDB_ACTION_UPDATE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { @@ -660,16 +661,16 @@ static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSWriteMsg oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; - code = (*pTable->fpDecode)(&oper); - return sdbUpdateHash(pTable, &oper); + SSWriteMsg wmsg = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + code = (*pTable->fpDecode)(&wmsg); + return sdbUpdateHash(pTable, &wmsg); } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } } int32_t sdbInsertRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (sdbGetRowFromObj(pTable, pWrite->pObj)) { @@ -699,15 +700,15 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { return TSDB_CODE_SUCCESS; } - if (pWrite->reqFp) { - return (*pWrite->reqFp)(pWrite->pMsg); + if (pWrite->fpReq) { + return (*pWrite->fpReq)(pWrite->pMsg); } else { return sdbInsertRowImp(pWrite); } } int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; @@ -729,14 +730,14 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); } - sdbIncRef(pNewOper->table, pNewOper->pObj); + sdbIncRef(pNewOper->pTable, pNewOper->pObj); taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { - SSdbTable *pTable = pTableInput; +bool sdbCheckRowDeleted(void *tparam, void *pRow) { + SSdbTable *pTable = tparam; if (pTable == NULL) return false; int32_t *updateEnd = pRow + pTable->refCountPos - 4; @@ -744,7 +745,7 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { } int32_t sdbDeleteRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); @@ -768,15 +769,15 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite) { return TSDB_CODE_SUCCESS; } - if (pWrite->reqFp) { - return (*pWrite->reqFp)(pWrite->pMsg); + if (pWrite->fpReq) { + return (*pWrite->fpReq)(pWrite->pMsg); } else { return sdbDeleteRowImp(pWrite); } } int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; @@ -803,7 +804,7 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { } int32_t sdbUpdateRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); @@ -823,15 +824,15 @@ int32_t sdbUpdateRow(SSWriteMsg *pWrite) { return TSDB_CODE_SUCCESS; } - if (pWrite->reqFp) { - return (*pWrite->reqFp)(pWrite->pMsg); + if (pWrite->fpReq) { + return (*pWrite->fpReq)(pWrite->pMsg); } else { return sdbUpdateRowImp(pWrite); } } int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = (SSdbTable *)pWrite->table; + SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; @@ -852,14 +853,14 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); } - sdbIncRef(pNewOper->table, pNewOper->pObj); + sdbIncRef(pNewOper->pTable, pNewOper->pObj); taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { - SSdbTable *pTable = (SSdbTable *)handle; +void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { + SSdbTable *pTable = tparam; *ppRow = NULL; if (pTable == NULL) return NULL; @@ -880,7 +881,7 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { } *ppRow = *ppMetaRow; - sdbIncRef(handle, *ppMetaRow); + sdbIncRef(pTable, *ppMetaRow); return pIter; } @@ -934,12 +935,12 @@ void sdbCloseTable(void *handle) { void **ppRow = taosHashIterGet(pIter); if (ppRow == NULL) continue; - SSWriteMsg oper = { + SSWriteMsg wmsg = { .pObj = *ppRow, - .table = pTable, + .pTable = pTable, }; - (*pTable->fpDestroy)(&oper); + (*pTable->fpDestroy)(&wmsg); } taosHashDestroyIter(pIter); @@ -950,44 +951,44 @@ void sdbCloseTable(void *handle) { free(pTable); } -int32_t sdbInitWriteWorker() { +int32_t sdbInitWorker() { tsSdbPool.num = 1; - tsSdbPool.worker = (SSWriteWorker *)calloc(sizeof(SSWriteWorker), tsSdbPool.num); + tsSdbPool.worker = calloc(sizeof(SSdbWorker), tsSdbPool.num); if (tsSdbPool.worker == NULL) return -1; for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSWriteWorker *pWorker = tsSdbPool.worker + i; + SSdbWorker *pWorker = tsSdbPool.worker + i; pWorker->workerId = i; } - sdbAllocWriteQueue(); + sdbAllocQueue(); mInfo("vgId:1, sdb write is opened"); return 0; } -void sdbCleanupWriteWorker() { +void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSWriteWorker *pWorker = tsSdbPool.worker + i; + SSdbWorker *pWorker = tsSdbPool.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSWriteWorker *pWorker = tsSdbPool.worker + i; + SSdbWorker *pWorker = tsSdbPool.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); } } - sdbFreeWritequeue(); + sdbFreeQueue(); tfree(tsSdbPool.worker); mInfo("vgId:1, sdb write is closed"); } -int32_t sdbAllocWriteQueue() { +int32_t sdbAllocQueue() { tsSdbWQueue = taosOpenQueue(); if (tsSdbWQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1006,7 +1007,7 @@ int32_t sdbAllocWriteQueue() { } for (int32_t i = 0; i < tsSdbPool.num; ++i) { - SSWriteWorker *pWorker = tsSdbPool.worker + i; + SSdbWorker *pWorker = tsSdbPool.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; @@ -1029,7 +1030,7 @@ int32_t sdbAllocWriteQueue() { return TSDB_CODE_SUCCESS; } -void sdbFreeWritequeue() { +void sdbFreeQueue() { taosCloseQueue(tsSdbWQueue); taosFreeQall(tsSdbWQall); taosCloseQset(tsSdbWQset); @@ -1038,8 +1039,8 @@ void sdbFreeWritequeue() { tsSdbWQueue = NULL; } -int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { - SWalHead *pHead = data; +int32_t sdbWriteToQueue(void *wparam, void *hparam, int32_t qtype, void *unsed) { + SWalHead *pHead = hparam; int32_t size = sizeof(SWalHead) + pHead->len; SWalHead *pWal = taosAllocateQitem(size); memcpy(pWal, pHead, size); @@ -1048,10 +1049,10 @@ int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { return 0; } -static void *sdbWorkerFp(void *param) { +static void *sdbWorkerFp(void *pWorker) { SWalHead *pHead; SSWriteMsg *pWrite; - int32_t type; + int32_t qtype; int32_t numOfMsgs; void * item; void * unUsed; @@ -1064,22 +1065,22 @@ static void *sdbWorkerFp(void *param) { } for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &type, &item); - if (type == TAOS_QTYPE_RPC) { + taosGetQitem(tsSdbWQall, &qtype, &item); + if (qtype == TAOS_QTYPE_RPC) { pWrite = (SSWriteMsg *)item; pWrite->processedCount = 1; pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; if (pWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", - pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, ((SSdbTable *)pWrite->table)->tableName, pWrite->pObj, - sdbGetKeyStr(pWrite->table, pHead->cont), pHead->version); + pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->tableName, pWrite->pObj, + sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version); } } else { pHead = (SWalHead *)item; pWrite = NULL; } - int32_t code = sdbWrite(pWrite, pHead, type, NULL); + int32_t code = sdbWrite(pWrite, pHead, qtype, NULL); if (code > 0) code = 0; if (pWrite) { pWrite->retCode = code; @@ -1093,12 +1094,12 @@ static void *sdbWorkerFp(void *param) { // browse all items, and process them one by one taosResetQitems(tsSdbWQall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &type, &item); + taosGetQitem(tsSdbWQall, &qtype, &item); - if (type == TAOS_QTYPE_RPC) { + if (qtype == TAOS_QTYPE_RPC) { pWrite = (SSWriteMsg *)item; sdbConfirmForward(NULL, pWrite, pWrite->retCode); - } else if (type == TAOS_QTYPE_FWD) { + } else if (qtype == TAOS_QTYPE_FWD) { pHead = (SWalHead *)item; syncConfirmForward(tsSdbObj.sync, pHead->version, pHead->len); taosFreeQitem(item); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index d3ba149f39..1ba476cf2a 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -99,13 +99,13 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) { tfree(pTable); } -static int32_t mnodeChildTableActionDestroy(SSWriteMsg *pOper) { - mnodeDestroyChildTable(pOper->pObj); +static int32_t mnodeChildTableActionDestroy(SSWriteMsg *pWMsg) { + mnodeDestroyChildTable(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionInsert(SSWriteMsg *pOper) { - SCTableObj *pTable = pOper->pObj; +static int32_t mnodeChildTableActionInsert(SSWriteMsg *pWMsg) { + SCTableObj *pTable = pWMsg->pObj; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -153,8 +153,8 @@ static int32_t mnodeChildTableActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDelete(SSWriteMsg *pOper) { - SCTableObj *pTable = pOper->pObj; +static int32_t mnodeChildTableActionDelete(SSWriteMsg *pWMsg) { + SCTableObj *pTable = pWMsg->pObj; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -189,8 +189,8 @@ static int32_t mnodeChildTableActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pOper) { - SCTableObj *pNew = pOper->pObj; +static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pWMsg) { + SCTableObj *pNew = pWMsg->pObj; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -216,50 +216,50 @@ static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionEncode(SSWriteMsg *pOper) { - SCTableObj *pTable = pOper->pObj; - assert(pTable != NULL && pOper->rowData != NULL); +static int32_t mnodeChildTableActionEncode(SSWriteMsg *pWMsg) { + SCTableObj *pTable = pWMsg->pObj; + assert(pTable != NULL && pWMsg->rowData != NULL); int32_t len = strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; - memcpy(pOper->rowData, pTable->info.tableId, len); - memset(pOper->rowData + len, 0, 1); + memcpy(pWMsg->rowData, pTable->info.tableId, len); + memset(pWMsg->rowData + len, 0, 1); len++; - memcpy(pOper->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy(pWMsg->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pOper->rowData + len, pTable->schema, schemaSize); + memcpy(pWMsg->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pOper->rowData + len, pTable->sql, pTable->sqlLen); + memcpy(pWMsg->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } - pOper->rowSize = len; + pWMsg->rowSize = len; return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDecode(SSWriteMsg *pOper) { - assert(pOper->rowData != NULL); +static int32_t mnodeChildTableActionDecode(SSWriteMsg *pWMsg) { + assert(pWMsg->rowData != NULL); SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pOper->rowData); + int32_t len = strlen(pWMsg->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; } - pTable->info.tableId = strdup(pOper->rowData); + pTable->info.tableId = strdup(pWMsg->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pOper->rowData + len, tsChildTableUpdateSize); + memcpy((char*)pTable + sizeof(char *), pWMsg->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -269,7 +269,7 @@ static int32_t mnodeChildTableActionDecode(SSWriteMsg *pOper) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pOper->rowData + len, schemaSize); + memcpy(pTable->schema, pWMsg->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -278,11 +278,11 @@ static int32_t mnodeChildTableActionDecode(SSWriteMsg *pOper) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pOper->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, pWMsg->rowData + len, pTable->sqlLen); } } - pOper->pObj = pTable; + pWMsg->pObj = pTable; return TSDB_CODE_SUCCESS; } @@ -297,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() { SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); @@ -309,7 +309,7 @@ static int32_t mnodeChildTableActionRestored() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -320,7 +320,7 @@ static int32_t mnodeChildTableActionRestored() { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -331,7 +331,7 @@ static int32_t mnodeChildTableActionRestored() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -430,13 +430,13 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) { tfree(pStable); } -static int32_t mnodeSuperTableActionDestroy(SSWriteMsg *pOper) { - mnodeDestroySuperTable(pOper->pObj); +static int32_t mnodeSuperTableActionDestroy(SSWriteMsg *pWMsg) { + mnodeDestroySuperTable(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pOper) { - SSTableObj *pStable = pOper->pObj; +static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pWMsg) { + SSTableObj *pStable = pWMsg->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); @@ -446,8 +446,8 @@ static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pOper) { - SSTableObj *pStable = pOper->pObj; +static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pWMsg) { + SSTableObj *pStable = pWMsg->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); @@ -458,8 +458,8 @@ static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pOper) { - SSTableObj *pNew = pOper->pObj; +static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pWMsg) { + SSTableObj *pNew = pWMsg->pObj; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -483,43 +483,43 @@ static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionEncode(SSWriteMsg *pOper) { - SSTableObj *pStable = pOper->pObj; - assert(pOper->pObj != NULL && pOper->rowData != NULL); +static int32_t mnodeSuperTableActionEncode(SSWriteMsg *pWMsg) { + SSTableObj *pStable = pWMsg->pObj; + assert(pWMsg->pObj != NULL && pWMsg->rowData != NULL); int32_t len = strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; - memcpy(pOper->rowData, pStable->info.tableId, len); - memset(pOper->rowData + len, 0, 1); + memcpy(pWMsg->rowData, pStable->info.tableId, len); + memset(pWMsg->rowData + len, 0, 1); len++; - memcpy(pOper->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy(pWMsg->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pOper->rowData + len, pStable->schema, schemaSize); + memcpy(pWMsg->rowData + len, pStable->schema, schemaSize); len += schemaSize; - pOper->rowSize = len; + pWMsg->rowSize = len; return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pOper) { - assert(pOper->rowData != NULL); +static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pWMsg) { + assert(pWMsg->rowData != NULL); SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pOper->rowData); + int32_t len = strlen(pWMsg->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; } - pStable->info.tableId = strdup(pOper->rowData); + pStable->info.tableId = strdup(pWMsg->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pOper->rowData + len, tsSuperTableUpdateSize); + memcpy((char*)pStable + sizeof(char *), pWMsg->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -529,9 +529,9 @@ static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pOper) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pOper->rowData + len, schemaSize); + memcpy(pStable->schema, pWMsg->rowData + len, schemaSize); - pOper->pObj = pStable; + pWMsg->pObj = pStable; return TSDB_CODE_SUCCESS; } @@ -828,7 +828,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsSuperTableSdb}; sdbDeleteRow(&desc); } @@ -878,16 +878,16 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pMsg->pTable = (STableObj *)pStable; mnodeIncTableRef(pMsg->pTable); - SSWriteMsg oper = { + SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, + .pTable = tsSuperTableSdb, .pObj = pStable, .rowSize = sizeof(SSTableObj) + schemaSize, .pMsg = pMsg, - .writeCb = mnodeCreateSuperTableCb + .fpWrite = mnodeCreateSuperTableCb }; - int32_t code = sdbInsertRow(&oper); + int32_t code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroySuperTable(pStable); pMsg->pTable = NULL; @@ -937,15 +937,15 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mnodeDropAllChildTablesInStable(pStable); } - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeDropSuperTableCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeDropSuperTableCb }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("app:%p:%p, table:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); @@ -1010,15 +1010,15 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t mInfo("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, schema[0].name); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeAddSuperTableTagCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeAddSuperTableTagCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { @@ -1044,15 +1044,15 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeDropSuperTableTagCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeDropSuperTableTagCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { @@ -1088,15 +1088,15 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c mInfo("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldTagName, newTagName); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeModifySuperTableTagNameCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeModifySuperTableTagNameCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) { @@ -1162,15 +1162,15 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeAddSuperTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeAddSuperTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { @@ -1207,15 +1207,15 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeDropSuperTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeDropSuperTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { @@ -1251,15 +1251,15 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldName, newName); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsSuperTableSdb, - .pObj = pStable, - .pMsg = pMsg, - .writeCb = mnodeChangeSuperTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .fpWrite = mnodeChangeSuperTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } // show super tables @@ -1417,12 +1417,12 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsSuperTableSdb, - .pObj = pTable, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfTables ++; } @@ -1694,7 +1694,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); return code; } @@ -1781,11 +1781,11 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { mnodeIncTableRef(pMsg->pTable); SSWriteMsg desc = { - .type = SDB_OPER_GLOBAL, - .pObj = pTable, - .table = tsChildTableSdb, - .pMsg = pMsg, - .reqFp = mnodeDoCreateChildTableFp + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + .pMsg = pMsg, + .fpReq = mnodeDoCreateChildTableFp }; int32_t code = sdbInsertRow(&desc); @@ -1901,15 +1901,15 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_APP_ERROR; } - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable, - .pMsg = pMsg, - .writeCb = mnodeDropChildTableCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .fpWrite = mnodeDropChildTableCb }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("app:%p:%p, ctable:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); @@ -2005,15 +2005,15 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable, - .pMsg = pMsg, - .writeCb = mnodeAlterNormalTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .fpWrite = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { @@ -2038,15 +2038,15 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable, - .pMsg = pMsg, - .writeCb = mnodeAlterNormalTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .fpWrite = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { @@ -2075,15 +2075,15 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, oldName, newName); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable, - .pMsg = pMsg, - .writeCb = mnodeAlterNormalTableColumnCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .fpWrite = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&oper); + return sdbUpdateRow(&wmsg); } static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) { @@ -2218,12 +2218,12 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { if (pTable == NULL) break; if (pTable->vgId == pVgroup->vgId) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsChildTableSdb, - .pObj = pTable, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsChildTableSdb, + .pObj = pTable, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfTables++; } mnodeDecTableRef(pTable); @@ -2251,12 +2251,12 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsChildTableSdb, - .pObj = pTable, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsChildTableSdb, + .pObj = pTable, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfTables++; } mnodeDecTableRef(pTable); @@ -2280,12 +2280,12 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { if (pTable == NULL) break; if (pTable->superTable == pStable) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsChildTableSdb, - .pObj = pTable, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsChildTableSdb, + .pObj = pTable, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfTables++; } @@ -2411,11 +2411,11 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { SSWriteMsg desc = { - .type = SDB_OPER_GLOBAL, - .pObj = pTable, - .table = tsChildTableSdb, - .pMsg = mnodeMsg, - .writeCb = mnodeDoCreateChildTableCb + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + .pMsg = mnodeMsg, + .fpWrite = mnodeDoCreateChildTableCb }; int32_t code = sdbInsertRowImp(&desc); @@ -2440,8 +2440,8 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); - SSWriteMsg oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; - sdbDeleteRow(&oper); + SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pObj = pTable}; + sdbDeleteRow(&wmsg); if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { //Avoid retry again in client diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index edd0839b0c..95d5befa5a 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -42,13 +42,13 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); -static int32_t mnodeUserActionDestroy(SSWriteMsg *pOper) { - tfree(pOper->pObj); +static int32_t mnodeUserActionDestroy(SSWriteMsg *pWMsg) { + tfree(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionInsert(SSWriteMsg *pOper) { - SUserObj *pUser = pOper->pObj; +static int32_t mnodeUserActionInsert(SSWriteMsg *pWMsg) { + SUserObj *pUser = pWMsg->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -62,8 +62,8 @@ static int32_t mnodeUserActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDelete(SSWriteMsg *pOper) { - SUserObj *pUser = pOper->pObj; +static int32_t mnodeUserActionDelete(SSWriteMsg *pWMsg) { + SUserObj *pUser = pWMsg->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -74,8 +74,8 @@ static int32_t mnodeUserActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionUpdate(SSWriteMsg *pOper) { - SUserObj *pUser = pOper->pObj; +static int32_t mnodeUserActionUpdate(SSWriteMsg *pWMsg) { + SUserObj *pUser = pWMsg->pObj; SUserObj *pSaved = mnodeGetUser(pUser->user); if (pUser != pSaved) { memcpy(pSaved, pUser, tsUserUpdateSize); @@ -85,19 +85,19 @@ static int32_t mnodeUserActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionEncode(SSWriteMsg *pOper) { - SUserObj *pUser = pOper->pObj; - memcpy(pOper->rowData, pUser, tsUserUpdateSize); - pOper->rowSize = tsUserUpdateSize; +static int32_t mnodeUserActionEncode(SSWriteMsg *pWMsg) { + SUserObj *pUser = pWMsg->pObj; + memcpy(pWMsg->rowData, pUser, tsUserUpdateSize); + pWMsg->rowSize = tsUserUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeUserActionDecode(SSWriteMsg *pWMsg) { SUserObj *pUser = (SUserObj *)calloc(1, sizeof(SUserObj)); if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pUser, pOper->rowData, tsUserUpdateSize); - pOper->pObj = pUser; + memcpy(pUser, pWMsg->rowData, tsUserUpdateSize); + pWMsg->pObj = pUser; return TSDB_CODE_SUCCESS; } @@ -205,14 +205,14 @@ void mnodeDecUserRef(SUserObj *pUser) { } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsUserSdb, - .pObj = pUser, - .pMsg = pMsg + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .pMsg = pMsg }; - int32_t code = sdbUpdateRow(&oper); + int32_t code = sdbUpdateRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); } else { @@ -259,15 +259,15 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { pUser->superAuth = 1; } - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsUserSdb, - .pObj = pUser, - .rowSize = sizeof(SUserObj), - .pMsg = pMsg + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + .pMsg = pMsg }; - code = sdbInsertRow(&oper); + code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); tfree(pUser); @@ -279,14 +279,14 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { } static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsUserSdb, - .pObj = pUser, - .pMsg = pMsg + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); } else { @@ -562,12 +562,12 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { if (pUser == NULL) break; if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsUserSdb, - .pObj = pUser, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsUserSdb, + .pObj = pUser, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfUsers++; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index da0ed1cd36..5a99dfbfa3 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -72,13 +72,13 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { tfree(pVgroup); } -static int32_t mnodeVgroupActionDestroy(SSWriteMsg *pOper) { - mnodeDestroyVgroup(pOper->pObj); +static int32_t mnodeVgroupActionDestroy(SSWriteMsg *pWMsg) { + mnodeDestroyVgroup(pWMsg->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionInsert(SSWriteMsg *pOper) { - SVgObj *pVgroup = pOper->pObj; +static int32_t mnodeVgroupActionInsert(SSWriteMsg *pWMsg) { + SVgObj *pVgroup = pWMsg->pObj; // refer to db SDbObj *pDb = mnodeGetDb(pVgroup->dbName); @@ -115,8 +115,8 @@ static int32_t mnodeVgroupActionInsert(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDelete(SSWriteMsg *pOper) { - SVgObj *pVgroup = pOper->pObj; +static int32_t mnodeVgroupActionDelete(SSWriteMsg *pWMsg) { + SVgObj *pVgroup = pWMsg->pObj; if (pVgroup->pDb == NULL) { mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName); @@ -137,8 +137,8 @@ static int32_t mnodeVgroupActionDelete(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pOper) { - SVgObj *pNew = pOper->pObj; +static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pWMsg) { + SVgObj *pNew = pWMsg->pObj; SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); if (pVgroup != pNew) { @@ -176,25 +176,25 @@ static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionEncode(SSWriteMsg *pOper) { - SVgObj *pVgroup = pOper->pObj; - memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); - SVgObj *pTmpVgroup = pOper->rowData; +static int32_t mnodeVgroupActionEncode(SSWriteMsg *pWMsg) { + SVgObj *pVgroup = pWMsg->pObj; + memcpy(pWMsg->rowData, pVgroup, tsVgUpdateSize); + SVgObj *pTmpVgroup = pWMsg->rowData; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; } - pOper->rowSize = tsVgUpdateSize; + pWMsg->rowSize = tsVgUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDecode(SSWriteMsg *pOper) { +static int32_t mnodeVgroupActionDecode(SSWriteMsg *pWMsg) { SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); if (pVgroup == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); - pOper->pObj = pVgroup; + memcpy(pVgroup, pWMsg->rowData, tsVgUpdateSize); + pWMsg->pObj = pVgroup; return TSDB_CODE_SUCCESS; } @@ -253,13 +253,13 @@ SVgObj *mnodeGetVgroup(int32_t vgId) { } void mnodeUpdateVgroup(SVgObj *pVgroup) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup }; - int32_t code = sdbUpdateRow(&oper); + int32_t code = sdbUpdateRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("vgId:%d, failed to update vgroup", pVgroup->vgId); } @@ -519,14 +519,14 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); return code; } else { mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); pVgroup->status = TAOS_VG_STATUS_READY; - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; (void)sdbUpdateRow(&desc); dnodeReprocessMWriteMsg(pMsg); @@ -535,7 +535,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, // pDb->name, pVgroup->numOfVnodes); // pVgroup->status = TAOS_VG_STATUS_READY; - // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; + // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -571,16 +571,16 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { pMsg->pVgroup = pVgroup; mnodeIncVgroupRef(pVgroup); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup, - .rowSize = sizeof(SVgObj), - .pMsg = pMsg, - .reqFp = mnodeCreateVgroupFp + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + .pMsg = pMsg, + .fpReq = mnodeCreateVgroupFp }; - code = sdbInsertRow(&oper); + code = sdbInsertRow(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); @@ -595,12 +595,12 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mDebug("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mnodeSendDropVgroupMsg(pVgroup, NULL); - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); } } @@ -957,28 +957,28 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; if (mnodeMsg->received == mnodeMsg->successed) { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup, - .rowSize = sizeof(SVgObj), - .pMsg = mnodeMsg, - .writeCb = mnodeCreateVgroupCb + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + .pMsg = mnodeMsg, + .fpWrite = mnodeCreateVgroupCb }; - int32_t code = sdbInsertRowImp(&oper); + int32_t code = sdbInsertRowImp(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1031,12 +1031,12 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; - SSWriteMsg oper = { - .type = SDB_OPER_GLOBAL, - .table = tsVgroupSdb, - .pObj = pVgroup + SSWriteMsg wmsg = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup }; - int32_t code = sdbDeleteRow(&oper); + int32_t code = sdbDeleteRow(&wmsg); if (code != 0) { code = TSDB_CODE_MND_SDB_ERROR; } @@ -1084,12 +1084,12 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) { mnodeDropAllChildTablesInVgroups(pVgroup); - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsVgroupSdb, - .pObj = pVgroup, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfVgroups++; } mnodeDecVgroupRef(pVgroup); @@ -1135,12 +1135,12 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { if (pVgroup == NULL) break; if (pVgroup->pDb == pDropDb) { - SSWriteMsg oper = { - .type = SDB_OPER_LOCAL, - .table = tsVgroupSdb, - .pObj = pVgroup, + SSWriteMsg wmsg = { + .type = SDB_OPER_LOCAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, }; - sdbDeleteRow(&oper); + sdbDeleteRow(&wmsg); numOfVgroups++; } diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index e26778e86b..591d7749ea 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,22 +113,22 @@ echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG echo "mDebugFlag 143" >> $TAOS_CFG echo "sdbDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 143" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG -echo "tsdbDebugFlag 143" >> $TAOS_CFG -echo "cDebugFlag 143" >> $TAOS_CFG -echo "jnidebugFlag 143" >> $TAOS_CFG -echo "odbcdebugFlag 143" >> $TAOS_CFG -echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monitorDebugFlag 143" >> $TAOS_CFG -echo "mqttDebugFlag 143" >> $TAOS_CFG -echo "qdebugFlag 143" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 131" >> $TAOS_CFG +echo "vDebugFlag 131" >> $TAOS_CFG +echo "tsdbDebugFlag 131" >> $TAOS_CFG +echo "cDebugFlag 131" >> $TAOS_CFG +echo "jnidebugFlag 131" >> $TAOS_CFG +echo "odbcdebugFlag 131" >> $TAOS_CFG +echo "httpDebugFlag 131" >> $TAOS_CFG +echo "monitorDebugFlag 131" >> $TAOS_CFG +echo "mqttDebugFlag 131" >> $TAOS_CFG +echo "qdebugFlag 131" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG +echo "udebugFlag 131" >> $TAOS_CFG echo "sdebugFlag 143" >> $TAOS_CFG echo "wdebugFlag 143" >> $TAOS_CFG -echo "cqdebugFlag 143" >> $TAOS_CFG +echo "cqdebugFlag 131" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG @@ -140,7 +140,7 @@ echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 4" >> $TAOS_CFG +echo "maxVgroupsPerDb 10" >> $TAOS_CFG echo "minTablesPerVnode 4" >> $TAOS_CFG echo "maxTablesPerVnode 1000" >> $TAOS_CFG echo "tableIncStepPerVnode 10000" >> $TAOS_CFG From 98ec34b40dcf563ab5c3797dd232562332d29f4d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 16:06:06 +0800 Subject: [PATCH 06/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 4 +- src/mnode/src/mnodeAcct.c | 18 +-- src/mnode/src/mnodeCluster.c | 10 +- src/mnode/src/mnodeDb.c | 24 ++-- src/mnode/src/mnodeDnode.c | 22 ++-- src/mnode/src/mnodeMnode.c | 26 ++-- src/mnode/src/mnodeSdb.c | 228 +++++++++++++++++------------------ src/mnode/src/mnodeTable.c | 80 ++++++------ src/mnode/src/mnodeUser.c | 22 ++-- src/mnode/src/mnodeVgroup.c | 38 +++--- 10 files changed, 234 insertions(+), 238 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 0b0933c08a..7169e14f03 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -57,7 +57,7 @@ typedef struct SSWriteMsg { void * rowData; int32_t (*fpReq)(SMnodeMsg *pMsg); int32_t (*fpWrite)(SMnodeMsg *pMsg, int32_t code); - void * pObj; + void * pRow; SMnodeMsg *pMsg; struct SSdbTable *pTable; } SSWriteMsg; @@ -75,7 +75,7 @@ typedef struct { int32_t (*fpEncode)(SSWriteMsg *pWrite); int32_t (*fpDecode)(SSWriteMsg *pWrite); int32_t (*fpDestroy)(SSWriteMsg *pWrite); - int32_t (*fpDestored)(); + int32_t (*fpRestored)(); } SSdbTableDesc; int32_t sdbInit(); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 697ff1d5a1..b12c200507 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -32,14 +32,14 @@ static int32_t tsAcctUpdateSize; static int32_t mnodeCreateRootAcct(); static int32_t mnodeAcctActionDestroy(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pObj; + SAcctObj *pAcct = pWMsg->pRow; pthread_mutex_destroy(&pAcct->mutex); - tfree(pWMsg->pObj); + tfree(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeAcctActionInsert(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pObj; + SAcctObj *pAcct = pWMsg->pRow; memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo)); pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS; pthread_mutex_init(&pAcct->mutex, NULL); @@ -47,14 +47,14 @@ static int32_t mnodeAcctActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeAcctActionDelete(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pObj; + SAcctObj *pAcct = pWMsg->pRow; mnodeDropAllUsers(pAcct); mnodeDropAllDbs(pAcct); return TSDB_CODE_SUCCESS; } static int32_t mnodeAcctActionUpdate(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pObj; + SAcctObj *pAcct = pWMsg->pRow; SAcctObj *pSaved = mnodeGetAcct(pAcct->user); if (pAcct != pSaved) { memcpy(pSaved, pAcct, tsAcctUpdateSize); @@ -65,7 +65,7 @@ static int32_t mnodeAcctActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeAcctActionEncode(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pObj; + SAcctObj *pAcct = pWMsg->pRow; memcpy(pWMsg->rowData, pAcct, tsAcctUpdateSize); pWMsg->rowSize = tsAcctUpdateSize; return TSDB_CODE_SUCCESS; @@ -76,7 +76,7 @@ static int32_t mnodeAcctActionDecode(SSWriteMsg *pWMsg) { if (pAcct == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pAcct, pWMsg->rowData, tsAcctUpdateSize); - pWMsg->pObj = pAcct; + pWMsg->pRow = pAcct; return TSDB_CODE_SUCCESS; } @@ -112,7 +112,7 @@ int32_t mnodeInitAccts() { .fpEncode = mnodeAcctActionEncode, .fpDecode = mnodeAcctActionDecode, .fpDestroy = mnodeAcctActionDestroy, - .fpDestored = mnodeAcctActionRestored + .fpRestored = mnodeAcctActionRestored }; tsAcctSdb = sdbOpenTable(&tableDesc); @@ -229,7 +229,7 @@ static int32_t mnodeCreateRootAcct() { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsAcctSdb, - .pObj = pAcct, + .pRow = pAcct, }; return sdbInsertRow(&wmsg); diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index c94c833270..f9182704b0 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeClusterActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pObj); + tfree(pWMsg->pRow); return TSDB_CODE_SUCCESS; } @@ -50,7 +50,7 @@ static int32_t mnodeClusterActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeClusterActionEncode(SSWriteMsg *pWMsg) { - SClusterObj *pCluster = pWMsg->pObj; + SClusterObj *pCluster = pWMsg->pRow; memcpy(pWMsg->rowData, pCluster, tsClusterUpdateSize); pWMsg->rowSize = tsClusterUpdateSize; return TSDB_CODE_SUCCESS; @@ -61,7 +61,7 @@ static int32_t mnodeClusterActionDecode(SSWriteMsg *pWMsg) { if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pCluster, pWMsg->rowData, tsClusterUpdateSize); - pWMsg->pObj = pCluster; + pWMsg->pRow = pCluster; return TSDB_CODE_SUCCESS; } @@ -97,7 +97,7 @@ int32_t mnodeInitCluster() { .fpEncode = mnodeClusterActionEncode, .fpDecode = mnodeClusterActionDecode, .fpDestroy = mnodeClusterActionDestroy, - .fpDestored = mnodeClusterActionRestored + .fpRestored = mnodeClusterActionRestored }; tsClusterSdb = sdbOpenTable(&tableDesc); @@ -148,7 +148,7 @@ static int32_t mnodeCreateCluster() { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsClusterSdb, - .pObj = pCluster, + .pRow = pCluster, }; return sdbInsertRow(&wmsg); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 3c2bfbb834..9696df155a 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -57,7 +57,7 @@ static void mnodeDestroyDb(SDbObj *pDb) { } static int32_t mnodeDbActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyDb(pWMsg->pObj); + mnodeDestroyDb(pWMsg->pRow); return TSDB_CODE_SUCCESS; } @@ -66,7 +66,7 @@ int64_t mnodeGetDbNum() { } static int32_t mnodeDbActionInsert(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pObj; + SDbObj *pDb = pWMsg->pRow; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); pthread_mutex_init(&pDb->mutex, NULL); @@ -92,7 +92,7 @@ static int32_t mnodeDbActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeDbActionDelete(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pObj; + SDbObj *pDb = pWMsg->pRow; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); mnodeDropAllChildTables(pDb); @@ -108,7 +108,7 @@ static int32_t mnodeDbActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeDbActionUpdate(SSWriteMsg *pWMsg) { - SDbObj *pNew = pWMsg->pObj; + SDbObj *pNew = pWMsg->pRow; SDbObj *pDb = mnodeGetDb(pNew->name); if (pDb != NULL && pNew != pDb) { memcpy(pDb, pNew, pWMsg->rowSize); @@ -121,7 +121,7 @@ static int32_t mnodeDbActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeDbActionEncode(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pObj; + SDbObj *pDb = pWMsg->pRow; memcpy(pWMsg->rowData, pDb, tsDbUpdateSize); pWMsg->rowSize = tsDbUpdateSize; return TSDB_CODE_SUCCESS; @@ -132,7 +132,7 @@ static int32_t mnodeDbActionDecode(SSWriteMsg *pWMsg) { if (pDb == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pDb, pWMsg->rowData, tsDbUpdateSize); - pWMsg->pObj = pDb; + pWMsg->pRow = pDb; return TSDB_CODE_SUCCESS; } @@ -157,7 +157,7 @@ int32_t mnodeInitDbs() { .fpEncode = mnodeDbActionEncode, .fpDecode = mnodeDbActionDecode, .fpDestroy = mnodeDbActionDestroy, - .fpDestored = mnodeDbActionRestored + .fpRestored = mnodeDbActionRestored }; tsDbSdb = sdbOpenTable(&tableDesc); @@ -415,7 +415,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pObj = pDb, + .pRow = pDb, .rowSize = sizeof(SDbObj), .pMsg = pMsg, .fpWrite = mnodeCreateDbCb @@ -810,7 +810,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pObj = pDb + .pRow = pDb }; int32_t code = sdbUpdateRow(&wmsg); @@ -1022,7 +1022,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pObj = pDb, + .pRow = pDb, .pMsg = pMsg, .fpWrite = mnodeAlterDbCb }; @@ -1074,7 +1074,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pObj = pDb, + .pRow = pDb, .pMsg = pMsg, .fpWrite = mnodeDropDbCb }; @@ -1137,7 +1137,7 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsDbSdb, - .pObj = pDb + .pRow = pDb }; sdbDeleteRow(&wmsg); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 617499967e..99cc44ba7d 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -88,12 +88,12 @@ static char* offlineReason[] = { }; static int32_t mnodeDnodeActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pObj); + tfree(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeDnodeActionInsert(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pObj; + SDnodeObj *pDnode = pWMsg->pRow; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->lastAccess = tsAccessSquence; @@ -108,7 +108,7 @@ static int32_t mnodeDnodeActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeDnodeActionDelete(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pObj; + SDnodeObj *pDnode = pWMsg->pRow; #ifndef _SYNC mnodeDropAllDnodeVgroups(pDnode); @@ -122,7 +122,7 @@ static int32_t mnodeDnodeActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pWMsg) { - SDnodeObj *pNew = pWMsg->pObj; + SDnodeObj *pNew = pWMsg->pRow; SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId); if (pDnode != NULL && pNew != pDnode) { memcpy(pDnode, pNew, pWMsg->rowSize); @@ -135,7 +135,7 @@ static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeDnodeActionEncode(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pObj; + SDnodeObj *pDnode = pWMsg->pRow; memcpy(pWMsg->rowData, pDnode, tsDnodeUpdateSize); pWMsg->rowSize = tsDnodeUpdateSize; return TSDB_CODE_SUCCESS; @@ -146,7 +146,7 @@ static int32_t mnodeDnodeActionDecode(SSWriteMsg *pWMsg) { if (pDnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pDnode, pWMsg->rowData, tsDnodeUpdateSize); - pWMsg->pObj = pDnode; + pWMsg->pRow = pDnode; return TSDB_CODE_SUCCESS; } @@ -184,7 +184,7 @@ int32_t mnodeInitDnodes() { .fpEncode = mnodeDnodeActionEncode, .fpDecode = mnodeDnodeActionDecode, .fpDestroy = mnodeDnodeActionDestroy, - .fpDestored = mnodeDnodeActionRestored + .fpRestored = mnodeDnodeActionRestored }; tsDnodeSdb = sdbOpenTable(&tableDesc); @@ -299,7 +299,7 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pObj = pDnode + .pRow = pDnode }; int32_t code = sdbUpdateRow(&wmsg); @@ -647,7 +647,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pObj = pDnode, + .pRow = pDnode, .rowSize = sizeof(SDnodeObj), .pMsg = pMsg }; @@ -668,7 +668,7 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pObj = pDnode, + .pRow = pDnode, .pMsg = pMsg }; @@ -1141,7 +1141,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mnodeGetMnodeRoleStr(pVgid->role)); + strcpy(pWrite, syncRole[pVgid->role]); cols++; } } diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 37915319e5..33925960be 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -59,12 +59,12 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #endif static int32_t mnodeMnodeActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pObj); + tfree(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeMnodeActionInsert(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pObj; + SMnodeObj *pMnode = pWMsg->pRow; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -77,7 +77,7 @@ static int32_t mnodeMnodeActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeMnodeActionDelete(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pObj; + SMnodeObj *pMnode = pWMsg->pRow; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -89,7 +89,7 @@ static int32_t mnodeMnodeActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pObj; + SMnodeObj *pMnode = pWMsg->pRow; SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId); if (pMnode != pSaved) { memcpy(pSaved, pMnode, pWMsg->rowSize); @@ -100,7 +100,7 @@ static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeMnodeActionEncode(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pObj; + SMnodeObj *pMnode = pWMsg->pRow; memcpy(pWMsg->rowData, pMnode, tsMnodeUpdateSize); pWMsg->rowSize = tsMnodeUpdateSize; return TSDB_CODE_SUCCESS; @@ -111,7 +111,7 @@ static int32_t mnodeMnodeActionDecode(SSWriteMsg *pWMsg) { if (pMnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pMnode, pWMsg->rowData, tsMnodeUpdateSize); - pWMsg->pObj = pMnode; + pWMsg->pRow = pMnode; return TSDB_CODE_SUCCESS; } @@ -150,7 +150,7 @@ int32_t mnodeInitMnodes() { .fpEncode = mnodeMnodeActionEncode, .fpDecode = mnodeMnodeActionDecode, .fpDestroy = mnodeMnodeActionDestroy, - .fpDestored = mnodeMnodeActionRestored + .fpRestored = mnodeMnodeActionRestored }; tsMnodeSdb = sdbOpenTable(&tableDesc); @@ -192,10 +192,6 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); } -char *mnodeGetMnodeRoleStr(int32_t role) { - return syncRole[role]; -} - void mnodeUpdateMnodeEpSet() { mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); @@ -332,7 +328,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsMnodeSdb, - .pObj = pMnode, + .pRow = pMnode, .fpWrite = mnodeCreateMnodeCb }; @@ -356,7 +352,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { void mnodeDropMnodeLocal(int32_t dnodeId) { SMnodeObj *pMnode = mnodeGetMnode(dnodeId); if (pMnode != NULL) { - SSWriteMsg wmsg = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pObj = pMnode}; + SSWriteMsg wmsg = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pRow = pMnode}; sdbDeleteRow(&wmsg); mnodeDecMnodeRef(pMnode); } @@ -374,7 +370,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsMnodeSdb, - .pObj = pMnode + .pRow = pMnode }; int32_t code = sdbDeleteRow(&wmsg); @@ -469,7 +465,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* roles = mnodeGetMnodeRoleStr(pMnode->role); + char* roles = syncRole[pMnode->role]; STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]); cols++; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 034ff870c5..8056833cdb 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -37,15 +37,15 @@ #define SDB_SYNC_HACK 16 typedef enum { - SDB_ACTION_INSERT, - SDB_ACTION_DELETE, - SDB_ACTION_UPDATE + SDB_ACTION_INSERT = 0, + SDB_ACTION_DELETE = 1, + SDB_ACTION_UPDATE = 2 } ESdbAction; typedef enum { - SDB_STATUS_OFFLINE, - SDB_STATUS_SERVING, - SDB_STATUS_CLOSING + SDB_STATUS_OFFLINE = 0, + SDB_STATUS_SERVING = 1, + SDB_STATUS_CLOSING = 2 } ESdbStatus; typedef struct SSdbTable { @@ -64,7 +64,7 @@ typedef struct SSdbTable { int32_t (*fpDecode)(SSWriteMsg *pWrite); int32_t (*fpEncode)(SSWriteMsg *pWrite); int32_t (*fpDestroy)(SSWriteMsg *pWrite); - int32_t (*fpDestored)(); + int32_t (*fpRestored)(); pthread_mutex_t mutex; } SSdbTable; @@ -78,7 +78,7 @@ typedef struct { int32_t numOfTables; SSdbTable *tableList[SDB_TABLE_MAX]; pthread_mutex_t mutex; -} SSdbObject; +} SSdbMgmt; typedef struct { pthread_t thread; @@ -92,7 +92,7 @@ typedef struct { extern void * tsMnodeTmr; static void * tsSdbTmr; -static SSdbObject tsSdbObj = {0}; +static SSdbMgmt tsSdbMgmt = {0}; static taos_qset tsSdbWQset; static taos_qall tsSdbWQall; static taos_queue tsSdbWQueue; @@ -121,15 +121,15 @@ int64_t sdbGetNumOfRows(void *pTable) { } uint64_t sdbGetVersion() { - return tsSdbObj.version; + return tsSdbMgmt.version; } bool sdbIsMaster() { - return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER; + return tsSdbMgmt.role == TAOS_SYNC_ROLE_MASTER; } bool sdbIsServing() { - return tsSdbObj.status == SDB_STATUS_SERVING; + return tsSdbMgmt.status == SDB_STATUS_SERVING; } static void *sdbGetObjKey(SSdbTable *pTable, void *key) { @@ -172,21 +172,21 @@ static char *sdbGetObjStr(SSdbTable *pTable, void *key) { } static void *sdbGetTableFromId(int32_t tableId) { - return tsSdbObj.tableList[tableId]; + return tsSdbMgmt.tableList[tableId]; } static int32_t sdbInitWal() { SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; char temp[TSDB_FILENAME_LEN] = {0}; sprintf(temp, "%s/wal", tsMnodeDir); - tsSdbObj.wal = walOpen(temp, &walCfg); - if (tsSdbObj.wal == NULL) { + tsSdbMgmt.wal = walOpen(temp, &walCfg); + if (tsSdbMgmt.wal == NULL) { sdbError("vgId:1, failed to open wal in %s", tsMnodeDir); return -1; } sdbInfo("vgId:1, open wal for restore"); - int code = walRestore(tsSdbObj.wal, NULL, sdbWrite); + int code = walRestore(tsSdbMgmt.wal, NULL, sdbWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code)); return -1; @@ -200,8 +200,8 @@ static void sdbRestoreTables() { for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; - if (pTable->fpDestored) { - (*pTable->fpDestored)(); + if (pTable->fpRestored) { + (*pTable->fpRestored)(); } totalRows += pTable->numOfRows; @@ -209,22 +209,22 @@ static void sdbRestoreTables() { sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->tableName, pTable->numOfRows); } - sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbObj.version, totalRows, numOfTables); + sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables); } void sdbUpdateMnodeRoles() { - if (tsSdbObj.sync <= 0) return; + if (tsSdbMgmt.sync <= 0) return; SNodesRole roles = {0}; - syncGetNodesRole(tsSdbObj.sync, &roles); + syncGetNodesRole(tsSdbMgmt.sync, &roles); - sdbInfo("vgId:1, update mnodes roles, replica:%d", tsSdbObj.cfg.replica); - for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) { + sdbInfo("vgId:1, update mnodes role, replica:%d", tsSdbMgmt.cfg.replica); + for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]); if (pMnode != NULL) { pMnode->role = roles.role[i]; - sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, mnodeGetMnodeRoleStr(pMnode->role)); - if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbObj.role = pMnode->role; + sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); + if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; mnodeDecMnodeRef(pMnode); } } @@ -239,16 +239,16 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3 } static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { - return walGetWalFile(tsSdbObj.wal, fileName, fileId); + return walGetWalFile(tsSdbMgmt.wal, fileName, fileId); } static void sdbNotifyRole(void *ahandle, int8_t role) { - sdbInfo("vgId:1, mnode role changed from %s to %s", mnodeGetMnodeRoleStr(tsSdbObj.role), mnodeGetMnodeRoleStr(role)); + sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]); - if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) { + if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) { balanceReset(); } - tsSdbObj.role = role; + tsSdbMgmt.role = role; sdbUpdateMnodeRoles(); } @@ -276,7 +276,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { if (pWrite->retCode != TSDB_CODE_SUCCESS) { SWalHead *pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; int32_t action = pHead->msgType % 10; - sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pWrite->pObj, + sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pWrite->pRow, sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, action, tstrerror(pWrite->retCode)); if (action == SDB_ACTION_INSERT) { // It's better to create a table in two stages, create it first and then set it success @@ -284,7 +284,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = pWrite->pTable, - .pObj = pWrite->pObj + .pRow = pWrite->pRow }; sdbDeleteRow(&wmsg); } @@ -297,7 +297,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { // if ahandle, means this func is called by sdb write if (ahandle == NULL) { - sdbDecRef(pWrite->pTable, pWrite->pObj); + sdbDecRef(pWrite->pTable, pWrite->pRow); } taosFreeQitem(pWrite); @@ -369,7 +369,7 @@ void sdbUpdateSync(void *pMnodes) { return; } - if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) { + if (memcmp(&syncCfg, &tsSdbMgmt.cfg, sizeof(SSyncCfg)) == 0) { sdbDebug("vgId:1, update sync config, info not changed"); return; } @@ -391,18 +391,18 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.writeToCache = sdbWriteToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; - tsSdbObj.cfg = syncCfg; + tsSdbMgmt.cfg = syncCfg; - if (tsSdbObj.sync) { - syncReconfig(tsSdbObj.sync, &syncCfg); + if (tsSdbMgmt.sync) { + syncReconfig(tsSdbMgmt.sync, &syncCfg); } else { - tsSdbObj.sync = syncStart(&syncInfo); + tsSdbMgmt.sync = syncStart(&syncInfo); } sdbUpdateMnodeRoles(); } int32_t sdbInit() { - pthread_mutex_init(&tsSdbObj.mutex, NULL); + pthread_mutex_init(&tsSdbMgmt.mutex, NULL); if (sdbInitWorker() != 0) { return -1; @@ -415,55 +415,55 @@ int32_t sdbInit() { sdbRestoreTables(); if (mnodeGetMnodesNum() == 1) { - tsSdbObj.role = TAOS_SYNC_ROLE_MASTER; + tsSdbMgmt.role = TAOS_SYNC_ROLE_MASTER; } - tsSdbObj.status = SDB_STATUS_SERVING; + tsSdbMgmt.status = SDB_STATUS_SERVING; return TSDB_CODE_SUCCESS; } void sdbCleanUp() { - if (tsSdbObj.status != SDB_STATUS_SERVING) return; + if (tsSdbMgmt.status != SDB_STATUS_SERVING) return; - tsSdbObj.status = SDB_STATUS_CLOSING; + tsSdbMgmt.status = SDB_STATUS_CLOSING; sdbCleanupWorker(); - sdbDebug("vgId:1, sdb will be closed, mver:%" PRIu64, tsSdbObj.version); + sdbDebug("vgId:1, sdb will be closed, mver:%" PRIu64, tsSdbMgmt.version); - if (tsSdbObj.sync) { - syncStop(tsSdbObj.sync); - tsSdbObj.sync = -1; + if (tsSdbMgmt.sync) { + syncStop(tsSdbMgmt.sync); + tsSdbMgmt.sync = -1; } - if (tsSdbObj.wal) { - walClose(tsSdbObj.wal); - tsSdbObj.wal = NULL; + if (tsSdbMgmt.wal) { + walClose(tsSdbMgmt.wal); + tsSdbMgmt.wal = NULL; } - pthread_mutex_destroy(&tsSdbObj.mutex); + pthread_mutex_destroy(&tsSdbMgmt.mutex); } -void sdbIncRef(void *tparam, void *pObj) { - if (pObj == NULL || tparam == NULL) return; +void sdbIncRef(void *tparam, void *pRow) { + if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, inc ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); + sdbTrace("vgId:1, sdb:%s, inc ref to key:%p:%s:%d", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); } -void sdbDecRef(void *tparam, void *pObj) { - if (pObj == NULL || tparam == NULL) return; +void sdbDecRef(void *tparam, void *pRow) { + if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, dec ref to key:%p:%s:%d", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); + sdbTrace("vgId:1, sdb:%s, dec ref to key:%p:%s:%d", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); - int32_t *updateEnd = pObj + pTable->refCountPos - 4; + int32_t *updateEnd = pRow + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { - sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pObj, sdbGetObjStr(pTable, pObj), refCount); - SSWriteMsg wmsg = {.pObj = pObj}; + sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); + SSWriteMsg wmsg = {.pRow = pRow}; (*pTable->fpDestroy)(&wmsg); } } @@ -502,7 +502,7 @@ static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { } static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { - void * key = sdbGetObjKey(pTable, pWrite->pObj); + void * key = sdbGetObjKey(pTable, pWrite->pRow); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { @@ -510,25 +510,25 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { } pthread_mutex_lock(&pTable->mutex); - taosHashPut(pTable->iHandle, key, keySize, &pWrite->pObj, sizeof(int64_t)); + taosHashPut(pTable->iHandle, key, keySize, &pWrite->pRow, sizeof(int64_t)); pthread_mutex_unlock(&pTable->mutex); - sdbIncRef(pTable, pWrite->pObj); + sdbIncRef(pTable, pWrite->pRow); atomic_add_fetch_32(&pTable->numOfRows, 1); if (pTable->keyType == SDB_KEY_AUTO) { - pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pWrite->pObj)); + pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pWrite->pRow)); } else { atomic_add_fetch_32(&pTable->autoIndex, 1); } sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); + sdbGetObjStr(pTable, pWrite->pRow), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); int32_t code = (*pTable->fpInsert)(pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj)); + sdbGetObjStr(pTable, pWrite->pRow)); sdbDeleteHash(pTable, pWrite); } @@ -536,17 +536,17 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { - int32_t *updateEnd = pWrite->pObj + pTable->refCountPos - 4; + int32_t *updateEnd = pWrite->pRow + pTable->refCountPos - 4; bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj)); + sdbGetObjStr(pTable, pWrite->pRow)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } (*pTable->fpDelete)(pWrite); - void * key = sdbGetObjKey(pTable, pWrite->pObj); + void * key = sdbGetObjKey(pTable, pWrite->pRow); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); @@ -559,16 +559,16 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { atomic_sub_fetch_32(&pTable->numOfRows, 1); sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj), pTable->numOfRows, pWrite->pMsg); + sdbGetObjStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); - sdbDecRef(pTable, pWrite->pObj); + sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_SUCCESS; } static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj), pTable->numOfRows, pWrite->pMsg); + sdbGetObjStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); (*pTable->fpUpdate)(pWrite); return TSDB_CODE_SUCCESS; @@ -583,42 +583,42 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); - pthread_mutex_lock(&tsSdbObj.mutex); + pthread_mutex_lock(&tsSdbMgmt.mutex); if (pHead->version == 0) { // assign version - tsSdbObj.version++; - pHead->version = tsSdbObj.version; + tsSdbMgmt.version++; + pHead->version = tsSdbMgmt.version; } else { // for data from WAL or forward, version may be smaller - if (pHead->version <= tsSdbObj.version) { - pthread_mutex_unlock(&tsSdbObj.mutex); + if (pHead->version <= tsSdbMgmt.version) { + pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbObj.version); + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SUCCESS; - } else if (pHead->version != tsSdbObj.version + 1) { - pthread_mutex_unlock(&tsSdbObj.mutex); + } else if (pHead->version != tsSdbMgmt.version + 1) { + pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbObj.version); + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SYN_INVALID_VERSION; } else { - tsSdbObj.version = pHead->version; + tsSdbMgmt.version = pHead->version; } } - int32_t code = walWrite(tsSdbObj.wal, pHead); + int32_t code = walWrite(tsSdbMgmt.wal, pHead); if (code < 0) { - pthread_mutex_unlock(&tsSdbObj.mutex); + pthread_mutex_unlock(&tsSdbMgmt.mutex); return code; } - pthread_mutex_unlock(&tsSdbObj.mutex); + pthread_mutex_unlock(&tsSdbMgmt.mutex); // from app, wmsg is created if (pWrite != NULL) { // forward to peers pWrite->processedCount = 0; - int32_t syncCode = syncForwardToPeer(tsSdbObj.sync, pHead, pWrite, TAOS_QTYPE_RPC); + int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pWrite, TAOS_QTYPE_RPC); if (syncCode <= 0) pWrite->processedCount = 1; if (syncCode < 0) { @@ -638,7 +638,7 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync - syncForwardToPeer(tsSdbObj.sync, pHead, pWrite, TAOS_QTYPE_RPC); + syncForwardToPeer(tsSdbMgmt.sync, pHead, pWrite, TAOS_QTYPE_RPC); // from wal or forward msg, wmsg not created, should add into hash if (action == SDB_ACTION_INSERT) { @@ -652,7 +652,7 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSWriteMsg wmsg = {.pTable = pTable, .pObj = pRow}; + SSWriteMsg wmsg = {.pTable = pTable, .pRow = pRow}; return sdbDeleteHash(pTable, &wmsg); } else if (action == SDB_ACTION_UPDATE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); @@ -673,19 +673,19 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - if (sdbGetRowFromObj(pTable, pWrite->pObj)) { + if (sdbGetRowFromObj(pTable, pWrite->pRow)) { sdbError("vgId:1, sdb:%s, failed to insert key:%s, already exist", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pObj)); - sdbDecRef(pTable, pWrite->pObj); + sdbGetObjStr(pTable, pWrite->pRow)); + sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } if (pTable->keyType == SDB_KEY_AUTO) { - *((uint32_t *)pWrite->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); // let vgId increase from 2 if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { - *((uint32_t *)pWrite->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); } } @@ -727,10 +727,10 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); } - sdbIncRef(pNewOper->pTable, pNewOper->pObj); + sdbIncRef(pNewOper->pTable, pNewOper->pRow); taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -748,24 +748,24 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite) { SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); + void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); if (pRow == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - sdbIncRef(pTable, pWrite->pObj); + sdbIncRef(pTable, pWrite->pRow); int32_t code = sdbDeleteHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->tableName); - sdbDecRef(pTable, pWrite->pObj); + sdbDecRef(pTable, pWrite->pRow); return code; } // just delete data from memory if (pWrite->type != SDB_OPER_GLOBAL) { - sdbDecRef(pTable, pWrite->pObj); + sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_SUCCESS; } @@ -795,7 +795,7 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); } taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); @@ -807,7 +807,7 @@ int32_t sdbUpdateRow(SSWriteMsg *pWrite) { SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pObj); + void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); if (pRow == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; @@ -850,10 +850,10 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { if (pNewOper->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pObj, sdbGetObjStr(pTable, pWrite->pObj)); + pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); } - sdbIncRef(pNewOper->pTable, pNewOper->pObj); + sdbIncRef(pNewOper->pTable, pNewOper->pRow); taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -910,7 +910,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->fpEncode = pDesc->fpEncode; pTable->fpDecode = pDesc->fpDecode; pTable->fpDestroy = pDesc->fpDestroy; - pTable->fpDestored = pDesc->fpDestored; + pTable->fpRestored = pDesc->fpRestored; _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { @@ -918,8 +918,8 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { } pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); - tsSdbObj.numOfTables++; - tsSdbObj.tableList[pTable->tableId] = pTable; + tsSdbMgmt.numOfTables++; + tsSdbMgmt.tableList[pTable->tableId] = pTable; return pTable; } @@ -927,8 +927,8 @@ void sdbCloseTable(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; if (pTable == NULL) return; - tsSdbObj.numOfTables--; - tsSdbObj.tableList[pTable->tableId] = NULL; + tsSdbMgmt.numOfTables--; + tsSdbMgmt.tableList[pTable->tableId] = NULL; SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); while (taosHashIterNext(pIter)) { @@ -936,7 +936,7 @@ void sdbCloseTable(void *handle) { if (ppRow == NULL) continue; SSWriteMsg wmsg = { - .pObj = *ppRow, + .pRow = *ppRow, .pTable = pTable, }; @@ -947,7 +947,7 @@ void sdbCloseTable(void *handle) { taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); - sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables); + sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbMgmt.numOfTables); free(pTable); } @@ -1072,7 +1072,7 @@ static void *sdbWorkerFp(void *pWorker) { pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; if (pWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", - pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->tableName, pWrite->pObj, + pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->tableName, pWrite->pRow, sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version); } } else { @@ -1089,7 +1089,7 @@ static void *sdbWorkerFp(void *pWorker) { } } - walFsync(tsSdbObj.wal, true); + walFsync(tsSdbMgmt.wal, true); // browse all items, and process them one by one taosResetQitems(tsSdbWQall); @@ -1101,7 +1101,7 @@ static void *sdbWorkerFp(void *pWorker) { sdbConfirmForward(NULL, pWrite, pWrite->retCode); } else if (qtype == TAOS_QTYPE_FWD) { pHead = (SWalHead *)item; - syncConfirmForward(tsSdbObj.sync, pHead->version, pHead->len); + syncConfirmForward(tsSdbMgmt.sync, pHead->version, pHead->len); taosFreeQitem(item); } else { taosFreeQitem(item); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 1ba476cf2a..fc441bf524 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -100,12 +100,12 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) { } static int32_t mnodeChildTableActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyChildTable(pWMsg->pObj); + mnodeDestroyChildTable(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeChildTableActionInsert(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pObj; + SCTableObj *pTable = pWMsg->pRow; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -154,7 +154,7 @@ static int32_t mnodeChildTableActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeChildTableActionDelete(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pObj; + SCTableObj *pTable = pWMsg->pRow; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -190,7 +190,7 @@ static int32_t mnodeChildTableActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pWMsg) { - SCTableObj *pNew = pWMsg->pObj; + SCTableObj *pNew = pWMsg->pRow; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -217,7 +217,7 @@ static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeChildTableActionEncode(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pObj; + SCTableObj *pTable = pWMsg->pRow; assert(pTable != NULL && pWMsg->rowData != NULL); int32_t len = strlen(pTable->info.tableId); @@ -282,7 +282,7 @@ static int32_t mnodeChildTableActionDecode(SSWriteMsg *pWMsg) { } } - pWMsg->pObj = pTable; + pWMsg->pRow = pTable; return TSDB_CODE_SUCCESS; } @@ -297,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() { SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); @@ -309,7 +309,7 @@ static int32_t mnodeChildTableActionRestored() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -320,7 +320,7 @@ static int32_t mnodeChildTableActionRestored() { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -331,7 +331,7 @@ static int32_t mnodeChildTableActionRestored() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -364,7 +364,7 @@ static int32_t mnodeInitChildTables() { .fpEncode = mnodeChildTableActionEncode, .fpDecode = mnodeChildTableActionDecode, .fpDestroy = mnodeChildTableActionDestroy, - .fpDestored = mnodeChildTableActionRestored + .fpRestored = mnodeChildTableActionRestored }; tsChildTableSdb = sdbOpenTable(&tableDesc); @@ -431,12 +431,12 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) { } static int32_t mnodeSuperTableActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroySuperTable(pWMsg->pObj); + mnodeDestroySuperTable(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pObj; + SSTableObj *pStable = pWMsg->pRow; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); @@ -447,7 +447,7 @@ static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pObj; + SSTableObj *pStable = pWMsg->pRow; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); @@ -459,7 +459,7 @@ static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pWMsg) { - SSTableObj *pNew = pWMsg->pObj; + SSTableObj *pNew = pWMsg->pRow; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -484,8 +484,8 @@ static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeSuperTableActionEncode(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pObj; - assert(pWMsg->pObj != NULL && pWMsg->rowData != NULL); + SSTableObj *pStable = pWMsg->pRow; + assert(pWMsg->pRow != NULL && pWMsg->rowData != NULL); int32_t len = strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; @@ -531,7 +531,7 @@ static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pWMsg) { memcpy(pStable->schema, pWMsg->rowData + len, schemaSize); - pWMsg->pObj = pStable; + pWMsg->pRow = pStable; return TSDB_CODE_SUCCESS; } @@ -557,7 +557,7 @@ static int32_t mnodeInitSuperTables() { .fpEncode = mnodeSuperTableActionEncode, .fpDecode = mnodeSuperTableActionDecode, .fpDestroy = mnodeSuperTableActionDestroy, - .fpDestored = mnodeSuperTableActionRestored + .fpRestored = mnodeSuperTableActionRestored }; tsSuperTableSdb = sdbOpenTable(&tableDesc); @@ -828,7 +828,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsSuperTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pTable, .pTable = tsSuperTableSdb}; sdbDeleteRow(&desc); } @@ -881,7 +881,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .rowSize = sizeof(SSTableObj) + schemaSize, .pMsg = pMsg, .fpWrite = mnodeCreateSuperTableCb @@ -940,7 +940,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeDropSuperTableCb }; @@ -1013,7 +1013,7 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeAddSuperTableTagCb }; @@ -1047,7 +1047,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeDropSuperTableTagCb }; @@ -1091,7 +1091,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeModifySuperTableTagNameCb }; @@ -1165,7 +1165,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeAddSuperTableColumnCb }; @@ -1210,7 +1210,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeDropSuperTableColumnCb }; @@ -1254,7 +1254,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pObj = pStable, + .pRow = pStable, .pMsg = pMsg, .fpWrite = mnodeChangeSuperTableColumnCb }; @@ -1420,7 +1420,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsSuperTableSdb, - .pObj = pTable, + .pRow = pTable, }; sdbDeleteRow(&wmsg); numOfTables ++; @@ -1694,7 +1694,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); return code; } @@ -1782,7 +1782,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SSWriteMsg desc = { .type = SDB_OPER_GLOBAL, - .pObj = pTable, + .pRow = pTable, .pTable = tsChildTableSdb, .pMsg = pMsg, .fpReq = mnodeDoCreateChildTableFp @@ -1904,7 +1904,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, .pMsg = pMsg, .fpWrite = mnodeDropChildTableCb }; @@ -2008,7 +2008,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, .pMsg = pMsg, .fpWrite = mnodeAlterNormalTableColumnCb }; @@ -2041,7 +2041,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, .pMsg = pMsg, .fpWrite = mnodeAlterNormalTableColumnCb }; @@ -2078,7 +2078,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, .pMsg = pMsg, .fpWrite = mnodeAlterNormalTableColumnCb }; @@ -2221,7 +2221,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, }; sdbDeleteRow(&wmsg); numOfTables++; @@ -2254,7 +2254,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, }; sdbDeleteRow(&wmsg); numOfTables++; @@ -2283,7 +2283,7 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pObj = pTable, + .pRow = pTable, }; sdbDeleteRow(&wmsg); numOfTables++; @@ -2412,7 +2412,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { SSWriteMsg desc = { .type = SDB_OPER_GLOBAL, - .pObj = pTable, + .pRow = pTable, .pTable = tsChildTableSdb, .pMsg = mnodeMsg, .fpWrite = mnodeDoCreateChildTableCb @@ -2440,7 +2440,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); - SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pObj = pTable}; + SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pRow = pTable}; sdbDeleteRow(&wmsg); if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 95d5befa5a..edb15fb778 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -43,12 +43,12 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); static int32_t mnodeUserActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pObj); + tfree(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeUserActionInsert(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pObj; + SUserObj *pUser = pWMsg->pRow; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -63,7 +63,7 @@ static int32_t mnodeUserActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeUserActionDelete(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pObj; + SUserObj *pUser = pWMsg->pRow; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -75,7 +75,7 @@ static int32_t mnodeUserActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeUserActionUpdate(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pObj; + SUserObj *pUser = pWMsg->pRow; SUserObj *pSaved = mnodeGetUser(pUser->user); if (pUser != pSaved) { memcpy(pSaved, pUser, tsUserUpdateSize); @@ -86,7 +86,7 @@ static int32_t mnodeUserActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeUserActionEncode(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pObj; + SUserObj *pUser = pWMsg->pRow; memcpy(pWMsg->rowData, pUser, tsUserUpdateSize); pWMsg->rowSize = tsUserUpdateSize; return TSDB_CODE_SUCCESS; @@ -97,7 +97,7 @@ static int32_t mnodeUserActionDecode(SSWriteMsg *pWMsg) { if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pUser, pWMsg->rowData, tsUserUpdateSize); - pWMsg->pObj = pUser; + pWMsg->pRow = pUser; return TSDB_CODE_SUCCESS; } @@ -163,7 +163,7 @@ int32_t mnodeInitUsers() { .fpEncode = mnodeUserActionEncode, .fpDecode = mnodeUserActionDecode, .fpDestroy = mnodeUserActionDestroy, - .fpDestored = mnodeUserActionRestored + .fpRestored = mnodeUserActionRestored }; tsUserSdb = sdbOpenTable(&tableDesc); @@ -208,7 +208,7 @@ static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pObj = pUser, + .pRow = pUser, .pMsg = pMsg }; @@ -262,7 +262,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pObj = pUser, + .pRow = pUser, .rowSize = sizeof(SUserObj), .pMsg = pMsg }; @@ -282,7 +282,7 @@ static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pObj = pUser, + .pRow = pUser, .pMsg = pMsg }; @@ -565,7 +565,7 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsUserSdb, - .pObj = pUser, + .pRow = pUser, }; sdbDeleteRow(&wmsg); numOfUsers++; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5a99dfbfa3..9da96a8acf 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -73,12 +73,12 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { } static int32_t mnodeVgroupActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyVgroup(pWMsg->pObj); + mnodeDestroyVgroup(pWMsg->pRow); return TSDB_CODE_SUCCESS; } static int32_t mnodeVgroupActionInsert(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pObj; + SVgObj *pVgroup = pWMsg->pRow; // refer to db SDbObj *pDb = mnodeGetDb(pVgroup->dbName); @@ -116,7 +116,7 @@ static int32_t mnodeVgroupActionInsert(SSWriteMsg *pWMsg) { } static int32_t mnodeVgroupActionDelete(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pObj; + SVgObj *pVgroup = pWMsg->pRow; if (pVgroup->pDb == NULL) { mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName); @@ -138,7 +138,7 @@ static int32_t mnodeVgroupActionDelete(SSWriteMsg *pWMsg) { } static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pWMsg) { - SVgObj *pNew = pWMsg->pObj; + SVgObj *pNew = pWMsg->pRow; SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); if (pVgroup != pNew) { @@ -177,7 +177,7 @@ static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pWMsg) { } static int32_t mnodeVgroupActionEncode(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pObj; + SVgObj *pVgroup = pWMsg->pRow; memcpy(pWMsg->rowData, pVgroup, tsVgUpdateSize); SVgObj *pTmpVgroup = pWMsg->rowData; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { @@ -194,7 +194,7 @@ static int32_t mnodeVgroupActionDecode(SSWriteMsg *pWMsg) { if (pVgroup == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; memcpy(pVgroup, pWMsg->rowData, tsVgUpdateSize); - pWMsg->pObj = pVgroup; + pWMsg->pRow = pVgroup; return TSDB_CODE_SUCCESS; } @@ -219,7 +219,7 @@ int32_t mnodeInitVgroups() { .fpEncode = mnodeVgroupActionEncode, .fpDecode = mnodeVgroupActionDecode, .fpDestroy = mnodeVgroupActionDestroy, - .fpDestored = mnodeVgroupActionRestored, + .fpRestored = mnodeVgroupActionRestored, }; tsVgroupSdb = sdbOpenTable(&tableDesc); @@ -256,7 +256,7 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup + .pRow = pVgroup }; int32_t code = sdbUpdateRow(&wmsg); @@ -519,14 +519,14 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); return code; } else { mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); pVgroup->status = TAOS_VG_STATUS_READY; - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; + SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; (void)sdbUpdateRow(&desc); dnodeReprocessMWriteMsg(pMsg); @@ -535,7 +535,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, // pDb->name, pVgroup->numOfVnodes); // pVgroup->status = TAOS_VG_STATUS_READY; - // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; + // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -574,7 +574,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup, + .pRow = pVgroup, .rowSize = sizeof(SVgObj), .pMsg = pMsg, .fpReq = mnodeCreateVgroupFp @@ -598,7 +598,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup + .pRow = pVgroup }; sdbDeleteRow(&wmsg); } @@ -770,7 +770,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v SDnodeObj * pDnode = pVgroup->vnodeGid[i].pDnode; const char *role = "NULL"; if (pDnode != NULL) { - role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role); + role = syncRole[pVgroup->vnodeGid[i].role]; } pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -960,7 +960,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup, + .pRow = pVgroup, .rowSize = sizeof(SVgObj), .pMsg = mnodeMsg, .fpWrite = mnodeCreateVgroupCb @@ -976,7 +976,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup + .pRow = pVgroup }; sdbDeleteRow(&wmsg); dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); @@ -1034,7 +1034,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pObj = pVgroup + .pRow = pVgroup }; int32_t code = sdbDeleteRow(&wmsg); if (code != 0) { @@ -1087,7 +1087,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsVgroupSdb, - .pObj = pVgroup, + .pRow = pVgroup, }; sdbDeleteRow(&wmsg); numOfVgroups++; @@ -1138,7 +1138,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { SSWriteMsg wmsg = { .type = SDB_OPER_LOCAL, .pTable = tsVgroupSdb, - .pObj = pVgroup, + .pRow = pVgroup, }; sdbDeleteRow(&wmsg); numOfVgroups++; From c1e1dd642764c81187357f081d0c69116a42c591 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 17:05:06 +0800 Subject: [PATCH 07/75] TD-2045 --- src/mnode/inc/mnodeSdb.h | 4 +- src/mnode/src/mnodeDb.c | 6 +- src/mnode/src/mnodeMnode.c | 2 +- src/mnode/src/mnodeSdb.c | 142 +++++++++++++++++------------------- src/mnode/src/mnodeTable.c | 122 +++++++++++++++---------------- src/mnode/src/mnodeVgroup.c | 12 +-- 6 files changed, 139 insertions(+), 149 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 7169e14f03..fb23e252c0 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -52,11 +52,11 @@ typedef enum { typedef struct SSWriteMsg { ESdbOper type; int32_t processedCount; // for sync fwd callback - int32_t retCode; // for callback in sdb queue + int32_t code; // for callback in sdb queue int32_t rowSize; void * rowData; int32_t (*fpReq)(SMnodeMsg *pMsg); - int32_t (*fpWrite)(SMnodeMsg *pMsg, int32_t code); + int32_t (*fpRsp)(SMnodeMsg *pMsg, int32_t code); void * pRow; SMnodeMsg *pMsg; struct SSdbTable *pTable; diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 9696df155a..6590ff0490 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -418,7 +418,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * .pRow = pDb, .rowSize = sizeof(SDbObj), .pMsg = pMsg, - .fpWrite = mnodeCreateDbCb + .fpRsp = mnodeCreateDbCb }; code = sdbInsertRow(&wmsg); @@ -1024,7 +1024,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { .pTable = tsDbSdb, .pRow = pDb, .pMsg = pMsg, - .fpWrite = mnodeAlterDbCb + .fpRsp = mnodeAlterDbCb }; code = sdbUpdateRow(&wmsg); @@ -1076,7 +1076,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { .pTable = tsDbSdb, .pRow = pDb, .pMsg = pMsg, - .fpWrite = mnodeDropDbCb + .fpRsp = mnodeDropDbCb }; int32_t code = sdbDeleteRow(&wmsg); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 33925960be..20e70c1af7 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -329,7 +329,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { .type = SDB_OPER_GLOBAL, .pTable = tsMnodeSdb, .pRow = pMnode, - .fpWrite = mnodeCreateMnodeCb + .fpRsp = mnodeCreateMnodeCb }; int32_t code = TSDB_CODE_SUCCESS; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 8056833cdb..850da1f0c3 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -48,6 +48,13 @@ typedef enum { SDB_STATUS_CLOSING = 2 } ESdbStatus; +char *actStr[] = { + "insert", + "delete", + "update", + "invalid" +}; + typedef struct SSdbTable { char tableName[SDB_TABLE_LEN]; ESdbTable tableId; @@ -140,18 +147,6 @@ static void *sdbGetObjKey(SSdbTable *pTable, void *key) { return key; } -static char *sdbGetActionStr(int32_t action) { - switch (action) { - case SDB_ACTION_INSERT: - return "insert"; - case SDB_ACTION_DELETE: - return "delete"; - case SDB_ACTION_UPDATE: - return "update"; - } - return "invalid"; -} - static char *sdbGetKeyStr(SSdbTable *pTable, void *key) { static char str[16]; switch (pTable->keyType) { @@ -167,7 +162,7 @@ static char *sdbGetKeyStr(SSdbTable *pTable, void *key) { } } -static char *sdbGetObjStr(SSdbTable *pTable, void *key) { +static char *sdbGetRowStr(SSdbTable *pTable, void *key) { return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key)); } @@ -254,33 +249,28 @@ static void sdbNotifyRole(void *ahandle, int8_t role) { } FORCE_INLINE -static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { - assert(param); - SSWriteMsg * pWrite = param; - SMnodeMsg *pMsg = pWrite->pMsg; - if (code <= 0) pWrite->retCode = code; +static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { + if (wparam == NULL) return; + SSWriteMsg *pWrite = wparam; + SMnodeMsg * pMsg = pWrite->pMsg; - int32_t processedCount = atomic_add_fetch_32(&pWrite->processedCount, 1); - if (processedCount <= 1) { - if (pMsg != NULL) { - sdbDebug("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, processedCount, code); - } + if (code <= 0) pWrite->code = code; + int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); + if (count <= 1) { + if (pMsg != NULL) sdbTrace("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, count, code); return; - } - - if (pMsg != NULL) { - sdbDebug("vgId:1, msg:%p is confirmed, code:%x", pMsg, code); + } else { + if (pMsg != NULL) sdbTrace("vgId:1, msg:%p is confirmed, code:%x", pMsg, code); } // failed to forward, need revert insert - if (pWrite->retCode != TSDB_CODE_SUCCESS) { - SWalHead *pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; + if (pWrite->code != TSDB_CODE_SUCCESS) { + SWalHead *pHead = (SWalHead *)((char *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); int32_t action = pHead->msgType % 10; - sdbError("vgId:1, key:%p:%s hver:%" PRIu64 " action:%d, failed to foward since %s", pWrite->pRow, - sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, action, tstrerror(pWrite->retCode)); + sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pWrite->pRow, + sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pWrite->code)); if (action == SDB_ACTION_INSERT) { // It's better to create a table in two stages, create it first and then set it success - //sdbDeleteHash(pWrite->pTable, pWrite); SSWriteMsg wmsg = { .type = SDB_OPER_GLOBAL, .pTable = pWrite->pTable, @@ -290,10 +280,10 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { } } - if (pWrite->fpWrite != NULL) { - pWrite->retCode = (*pWrite->fpWrite)(pMsg, pWrite->retCode); + if (pWrite->fpRsp != NULL) { + pWrite->code = (*pWrite->fpRsp)(pMsg, pWrite->code); } - dnodeSendRpcMWriteRsp(pMsg, pWrite->retCode); + dnodeSendRpcMWriteRsp(pMsg, pWrite->code); // if ahandle, means this func is called by sdb write if (ahandle == NULL) { @@ -449,7 +439,7 @@ void sdbIncRef(void *tparam, void *pRow) { SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, inc ref to key:%p:%s:%d", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); } void sdbDecRef(void *tparam, void *pRow) { @@ -458,11 +448,11 @@ void sdbDecRef(void *tparam, void *pRow) { SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, dec ref to key:%p:%s:%d", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); int32_t *updateEnd = pRow + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { - sdbTrace("vgId:1, sdb:%s, key:%p:%s:%d destroyed", pTable->tableName, pRow, sdbGetObjStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); SSWriteMsg wmsg = {.pRow = pRow}; (*pTable->fpDestroy)(&wmsg); } @@ -523,12 +513,12 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { } sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pWrite->pRow), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); int32_t code = (*pTable->fpInsert)(pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow)); + sdbGetRowStr(pTable, pWrite->pRow)); sdbDeleteHash(pTable, pWrite); } @@ -540,7 +530,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow)); + sdbGetRowStr(pTable, pWrite->pRow)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } @@ -559,7 +549,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { atomic_sub_fetch_32(&pTable->numOfRows, 1); sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); sdbDecRef(pTable, pWrite->pRow); @@ -568,7 +558,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); (*pTable->fpUpdate)(pWrite); return TSDB_CODE_SUCCESS; @@ -594,12 +584,12 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { if (pHead->version <= tsSdbMgmt.version) { pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); + pTable->tableName, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbMgmt.version + 1) { pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); + pTable->tableName, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SYN_INVALID_VERSION; } else { tsSdbMgmt.version = pHead->version; @@ -623,19 +613,19 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { if (syncCode < 0) { sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + tstrerror(syncCode), actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else if (syncCode > 0) { sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else { sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, - sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } return syncCode; } sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->tableName, - sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version); + actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync syncForwardToPeer(tsSdbMgmt.sync, pHead, pWrite, TAOS_QTYPE_RPC); @@ -675,7 +665,7 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { if (sdbGetRowFromObj(pTable, pWrite->pRow)) { sdbError("vgId:1, sdb:%s, failed to insert key:%s, already exist", pTable->tableName, - sdbGetObjStr(pTable, pWrite->pRow)); + sdbGetRowStr(pTable, pWrite->pRow)); sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } @@ -712,9 +702,9 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewOper = taosAllocateQitem(size); + SSWriteMsg *pNewWrite = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; + SWalHead *pHead = (SWalHead *)((char *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; pHead->len = pWrite->rowSize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; @@ -723,15 +713,15 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { (*pTable->fpEncode)(pWrite); pHead->len = pWrite->rowSize; - memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); + memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - if (pNewOper->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); + if (pNewWrite->pMsg != NULL) { + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, insert action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, + pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } - sdbIncRef(pNewOper->pTable, pNewOper->pRow); - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); + sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -781,9 +771,9 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewOper = taosAllocateQitem(size); + SSWriteMsg *pNewWrite = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; + SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; @@ -791,14 +781,14 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { (*pTable->fpEncode)(pWrite); pHead->len = pWrite->rowSize; - memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); + memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - if (pNewOper->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); + if (pNewWrite->pMsg != NULL) { + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, delete action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, + pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -836,9 +826,9 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewOper = taosAllocateQitem(size); + SSWriteMsg *pNewWrite = taosAllocateQitem(size); - SWalHead *pHead = (void *)pNewOper + sizeof(SSWriteMsg) + SDB_SYNC_HACK; + SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; @@ -846,15 +836,15 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { (*pTable->fpEncode)(pWrite); pHead->len = pWrite->rowSize; - memcpy(pNewOper, pWrite, sizeof(SSWriteMsg)); + memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - if (pNewOper->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, - pNewOper->pMsg, pTable->tableName, pWrite->pRow, sdbGetObjStr(pTable, pWrite->pRow)); + if (pNewWrite->pMsg != NULL) { + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, update action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, + pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } - sdbIncRef(pNewOper->pTable, pNewOper->pRow); - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewOper); + sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); + taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1071,7 +1061,7 @@ static void *sdbWorkerFp(void *pWorker) { pWrite->processedCount = 1; pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; if (pWrite->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s key:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", + sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->tableName, pWrite->pRow, sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version); } @@ -1083,7 +1073,7 @@ static void *sdbWorkerFp(void *pWorker) { int32_t code = sdbWrite(pWrite, pHead, qtype, NULL); if (code > 0) code = 0; if (pWrite) { - pWrite->retCode = code; + pWrite->code = code; } else { pHead->len = code; // hackway } @@ -1098,7 +1088,7 @@ static void *sdbWorkerFp(void *pWorker) { if (qtype == TAOS_QTYPE_RPC) { pWrite = (SSWriteMsg *)item; - sdbConfirmForward(NULL, pWrite, pWrite->retCode); + sdbConfirmForward(NULL, pWrite, pWrite->code); } else if (qtype == TAOS_QTYPE_FWD) { pHead = (SWalHead *)item; syncConfirmForward(tsSdbMgmt.sync, pHead->version, pHead->len); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index fc441bf524..ce9f652330 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -879,12 +879,12 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { mnodeIncTableRef(pMsg->pTable); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, .rowSize = sizeof(SSTableObj) + schemaSize, - .pMsg = pMsg, - .fpWrite = mnodeCreateSuperTableCb + .pMsg = pMsg, + .fpRsp = mnodeCreateSuperTableCb }; int32_t code = sdbInsertRow(&wmsg); @@ -942,7 +942,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { .pTable = tsSuperTableSdb, .pRow = pStable, .pMsg = pMsg, - .fpWrite = mnodeDropSuperTableCb + .fpRsp = mnodeDropSuperTableCb }; int32_t code = sdbDeleteRow(&wmsg); @@ -1011,11 +1011,11 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t schema[0].name); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeAddSuperTableTagCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeAddSuperTableTagCb }; return sdbUpdateRow(&wmsg); @@ -1045,11 +1045,11 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeDropSuperTableTagCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeDropSuperTableTagCb }; return sdbUpdateRow(&wmsg); @@ -1089,11 +1089,11 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c oldTagName, newTagName); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeModifySuperTableTagNameCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeModifySuperTableTagNameCb }; return sdbUpdateRow(&wmsg); @@ -1163,11 +1163,11 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeAddSuperTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeAddSuperTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -1208,11 +1208,11 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeDropSuperTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeDropSuperTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -1252,11 +1252,11 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char oldName, newName); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsSuperTableSdb, - .pRow = pStable, - .pMsg = pMsg, - .fpWrite = mnodeChangeSuperTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pRow = pStable, + .pMsg = pMsg, + .fpRsp = mnodeChangeSuperTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -1902,11 +1902,11 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { } SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsChildTableSdb, - .pRow = pTable, - .pMsg = pMsg, - .fpWrite = mnodeDropChildTableCb + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pRow = pTable, + .pMsg = pMsg, + .fpRsp = mnodeDropChildTableCb }; int32_t code = sdbDeleteRow(&wmsg); @@ -2006,11 +2006,11 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsChildTableSdb, - .pRow = pTable, - .pMsg = pMsg, - .fpWrite = mnodeAlterNormalTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pRow = pTable, + .pMsg = pMsg, + .fpRsp = mnodeAlterNormalTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -2039,11 +2039,11 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsChildTableSdb, - .pRow = pTable, - .pMsg = pMsg, - .fpWrite = mnodeAlterNormalTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pRow = pTable, + .pMsg = pMsg, + .fpRsp = mnodeAlterNormalTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -2076,11 +2076,11 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char oldName, newName); SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsChildTableSdb, - .pRow = pTable, - .pMsg = pMsg, - .fpWrite = mnodeAlterNormalTableColumnCb + .type = SDB_OPER_GLOBAL, + .pTable = tsChildTableSdb, + .pRow = pTable, + .pMsg = pMsg, + .fpRsp = mnodeAlterNormalTableColumnCb }; return sdbUpdateRow(&wmsg); @@ -2411,11 +2411,11 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { SSWriteMsg desc = { - .type = SDB_OPER_GLOBAL, - .pRow = pTable, - .pTable = tsChildTableSdb, - .pMsg = mnodeMsg, - .fpWrite = mnodeDoCreateChildTableCb + .type = SDB_OPER_GLOBAL, + .pRow = pTable, + .pTable = tsChildTableSdb, + .pMsg = mnodeMsg, + .fpRsp = mnodeDoCreateChildTableCb }; int32_t code = sdbInsertRowImp(&desc); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9da96a8acf..c12d2057f8 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -958,12 +958,12 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received == mnodeMsg->successed) { SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = tsVgroupSdb, - .pRow = pVgroup, - .rowSize = sizeof(SVgObj), - .pMsg = mnodeMsg, - .fpWrite = mnodeCreateVgroupCb + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pRow = pVgroup, + .rowSize = sizeof(SVgObj), + .pMsg = mnodeMsg, + .fpRsp = mnodeCreateVgroupCb }; int32_t code = sdbInsertRowImp(&wmsg); From 4c11e46075d65d1bc29a116ed9359dd36c80d421 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 18 Nov 2020 17:40:00 +0800 Subject: [PATCH 08/75] change --- .../com/taosdata/jdbc/rs/RestfulDriver.java | 3 +- .../taosdata/jdbc/rs/RestfulDriverTest.java | 35 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index c267f660de..a07d6eb2b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -86,6 +86,7 @@ public class RestfulDriver extends AbstractTaosDriver { @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { - return null; + //TODO SQLFeatureNotSupportedException + throw new SQLFeatureNotSupportedException(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index a91d1c2d6b..768eceab8e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -31,10 +31,43 @@ public class RestfulDriverTest { } @Test - public void testAcceptUrl() throws SQLException { + public void connect() { + + } + + @Test + public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041"); Assert.assertTrue(isAccept); + isAccept = driver.acceptsURL("jdbc:TAOS://master:6041"); + Assert.assertFalse(isAccept); } + @Test + public void getPropertyInfo() throws SQLException { + Driver driver = new RestfulDriver(); + final String url = ""; + DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); + } + + @Test + public void getMajorVersion() { + Assert.assertEquals(2, new RestfulDriver().getMajorVersion()); + } + + @Test + public void getMinorVersion() { + Assert.assertEquals(0, new RestfulDriver().getMinorVersion()); + } + + @Test + public void jdbcCompliant() { + Assert.assertFalse(new RestfulDriver().jdbcCompliant()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getParentLogger() throws SQLFeatureNotSupportedException { + new RestfulDriver().getParentLogger(); + } } From cf5bcc4b585c4971f39629e663ac9c3d7837cd56 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Nov 2020 17:55:57 +0800 Subject: [PATCH 09/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 4 +- src/mnode/src/mnodeAcct.c | 12 ++-- src/mnode/src/mnodeCluster.c | 12 ++-- src/mnode/src/mnodeDb.c | 8 +-- src/mnode/src/mnodeDnode.c | 8 +-- src/mnode/src/mnodeMnode.c | 8 +-- src/mnode/src/mnodeSdb.c | 109 ++++++++++++++++++----------------- src/mnode/src/mnodeTable.c | 8 +-- src/mnode/src/mnodeUser.c | 12 ++-- src/mnode/src/mnodeVgroup.c | 8 +-- 10 files changed, 95 insertions(+), 94 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index fb23e252c0..04fee5f6d5 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -63,11 +63,11 @@ typedef struct SSWriteMsg { } SSWriteMsg; typedef struct { - char * tableName; + char * name; int32_t hashSessions; int32_t maxRowSize; int32_t refCountPos; - ESdbTable tableId; + ESdbTable id; ESdbKey keyType; int32_t (*fpInsert)(SSWriteMsg *pWrite); int32_t (*fpDelete)(SSWriteMsg *pWrite); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index b12c200507..48af408c99 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -99,9 +99,9 @@ int32_t mnodeInitAccts() { SAcctObj tObj; tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_ACCOUNT, - .tableName = "accounts", + SSdbTableDesc desc = { + .id = SDB_TABLE_ACCOUNT, + .name = "accounts", .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE, .maxRowSize = tsAcctUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -115,13 +115,13 @@ int32_t mnodeInitAccts() { .fpRestored = mnodeAcctActionRestored }; - tsAcctSdb = sdbOpenTable(&tableDesc); + tsAcctSdb = sdbOpenTable(&desc); if (tsAcctSdb == NULL) { - mError("table:%s, failed to create hash", tableDesc.tableName); + mError("table:%s, failed to create hash", desc.name); return -1; } - mDebug("table:%s, hash is created", tableDesc.tableName); + mDebug("table:%s, hash is created", desc.name); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index f9182704b0..ff84b7ac3f 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -84,9 +84,9 @@ int32_t mnodeInitCluster() { SClusterObj tObj; tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_CLUSTER, - .tableName = "cluster", + SSdbTableDesc desc = { + .id = SDB_TABLE_CLUSTER, + .name = "cluster", .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, .maxRowSize = tsClusterUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -100,16 +100,16 @@ int32_t mnodeInitCluster() { .fpRestored = mnodeClusterActionRestored }; - tsClusterSdb = sdbOpenTable(&tableDesc); + tsClusterSdb = sdbOpenTable(&desc); if (tsClusterSdb == NULL) { - mError("table:%s, failed to create hash", tableDesc.tableName); + mError("table:%s, failed to create hash", desc.name); return -1; } mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); - mDebug("table:%s, hash is created", tableDesc.tableName); + mDebug("table:%s, hash is created", desc.name); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 6590ff0490..96658ac73c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -144,9 +144,9 @@ int32_t mnodeInitDbs() { SDbObj tObj; tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_DB, - .tableName = "dbs", + SSdbTableDesc desc = { + .id = SDB_TABLE_DB, + .name = "dbs", .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE, .maxRowSize = tsDbUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -160,7 +160,7 @@ int32_t mnodeInitDbs() { .fpRestored = mnodeDbActionRestored }; - tsDbSdb = sdbOpenTable(&tableDesc); + tsDbSdb = sdbOpenTable(&desc); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 99cc44ba7d..7b52747951 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -171,9 +171,9 @@ int32_t mnodeInitDnodes() { tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; pthread_mutex_init(&tsDnodeEpsMutex, NULL); - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_DNODE, - .tableName = "dnodes", + SSdbTableDesc desc = { + .id = SDB_TABLE_DNODE, + .name = "dnodes", .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE, .maxRowSize = tsDnodeUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -187,7 +187,7 @@ int32_t mnodeInitDnodes() { .fpRestored = mnodeDnodeActionRestored }; - tsDnodeSdb = sdbOpenTable(&tableDesc); + tsDnodeSdb = sdbOpenTable(&desc); if (tsDnodeSdb == NULL) { mError("failed to init dnodes data"); return -1; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 20e70c1af7..a97aef61e9 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -137,9 +137,9 @@ int32_t mnodeInitMnodes() { SMnodeObj tObj; tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_MNODE, - .tableName = "mnodes", + SSdbTableDesc desc = { + .id = SDB_TABLE_MNODE, + .name = "mnodes", .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE, .maxRowSize = tsMnodeUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -153,7 +153,7 @@ int32_t mnodeInitMnodes() { .fpRestored = mnodeMnodeActionRestored }; - tsMnodeSdb = sdbOpenTable(&tableDesc); + tsMnodeSdb = sdbOpenTable(&desc); if (tsMnodeSdb == NULL) { mError("failed to init mnodes data"); return -1; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 850da1f0c3..12cf5757b3 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -56,8 +56,8 @@ char *actStr[] = { }; typedef struct SSdbTable { - char tableName[SDB_TABLE_LEN]; - ESdbTable tableId; + char name[SDB_TABLE_LEN]; + ESdbTable id; ESdbKey keyType; int32_t hashSessions; int32_t maxRowSize; @@ -201,7 +201,7 @@ static void sdbRestoreTables() { totalRows += pTable->numOfRows; numOfTables++; - sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->tableName, pTable->numOfRows); + sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->name, pTable->numOfRows); } sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables); @@ -248,6 +248,21 @@ static void sdbNotifyRole(void *ahandle, int8_t role) { sdbUpdateMnodeRoles(); } +// failed to forward, need revert insert +static void sdbHandleFailedConfirm(SSWriteMsg *pWrite) { + SWalHead *pHead = (SWalHead *)((char *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); + int32_t action = pHead->msgType % 10; + + sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pWrite->pRow, + sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pWrite->code)); + + // It's better to create a table in two stages, create it first and then set it success + if (action == SDB_ACTION_INSERT) { + SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = pWrite->pTable, .pRow = pWrite->pRow}; + sdbDeleteRow(&wmsg); + } +} + FORCE_INLINE static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { if (wparam == NULL) return; @@ -263,26 +278,12 @@ static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { if (pMsg != NULL) sdbTrace("vgId:1, msg:%p is confirmed, code:%x", pMsg, code); } - // failed to forward, need revert insert - if (pWrite->code != TSDB_CODE_SUCCESS) { - SWalHead *pHead = (SWalHead *)((char *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); - int32_t action = pHead->msgType % 10; - sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pWrite->pRow, - sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pWrite->code)); - if (action == SDB_ACTION_INSERT) { - // It's better to create a table in two stages, create it first and then set it success - SSWriteMsg wmsg = { - .type = SDB_OPER_GLOBAL, - .pTable = pWrite->pTable, - .pRow = pWrite->pRow - }; - sdbDeleteRow(&wmsg); - } - } + if (pWrite->code != TSDB_CODE_SUCCESS) sdbHandleFailedConfirm(pWrite); if (pWrite->fpRsp != NULL) { pWrite->code = (*pWrite->fpRsp)(pMsg, pWrite->code); } + dnodeSendRpcMWriteRsp(pMsg, pWrite->code); // if ahandle, means this func is called by sdb write @@ -439,7 +440,7 @@ void sdbIncRef(void *tparam, void *pRow) { SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); } void sdbDecRef(void *tparam, void *pRow) { @@ -448,11 +449,11 @@ void sdbDecRef(void *tparam, void *pRow) { SSdbTable *pTable = tparam; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); int32_t *updateEnd = pRow + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { - sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->tableName, pRow, sdbGetRowStr(pTable, pRow), refCount); + sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); SSWriteMsg wmsg = {.pRow = pRow}; (*pTable->fpDestroy)(&wmsg); } @@ -512,12 +513,12 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { atomic_add_fetch_32(&pTable->autoIndex, 1); } - sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pWrite->pRow), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); int32_t code = (*pTable->fpInsert)(pWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->tableName, + sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); sdbDeleteHash(pTable, pWrite); } @@ -529,7 +530,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { int32_t *updateEnd = pWrite->pRow + pTable->refCountPos - 4; bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { - sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->tableName, + sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } @@ -548,7 +549,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { atomic_sub_fetch_32(&pTable->numOfRows, 1); - sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); sdbDecRef(pTable, pWrite->pRow); @@ -557,7 +558,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { } static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { - sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); (*pTable->fpUpdate)(pWrite); @@ -584,12 +585,12 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { if (pHead->version <= tsSdbMgmt.version) { pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); + pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbMgmt.version + 1) { pthread_mutex_unlock(&tsSdbMgmt.mutex); sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64, - pTable->tableName, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); + pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version); return TSDB_CODE_SYN_INVALID_VERSION; } else { tsSdbMgmt.version = pHead->version; @@ -612,19 +613,19 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { if (syncCode <= 0) pWrite->processedCount = 1; if (syncCode < 0) { - sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, + sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, tstrerror(syncCode), actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else if (syncCode > 0) { - sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } else { - sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->tableName, + sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); } return syncCode; } - sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->tableName, + sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync @@ -638,7 +639,7 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { } else if (action == SDB_ACTION_DELETE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } @@ -647,7 +648,7 @@ static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { } else if (action == SDB_ACTION_UPDATE) { void *pRow = sdbGetRowMeta(pTable, pHead->cont); if (pRow == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->tableName, + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } @@ -664,7 +665,7 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (sdbGetRowFromObj(pTable, pWrite->pRow)) { - sdbError("vgId:1, sdb:%s, failed to insert key:%s, already exist", pTable->tableName, + sdbError("vgId:1, sdb:%s, failed to insert key:%s since it already exist", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; @@ -674,14 +675,14 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); // let vgId increase from 2 - if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { + if (pTable->autoIndex == 1 && strcmp(pTable->name, "vgroups") == 0) { *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); } } int32_t code = sdbInsertHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to insert into hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to insert into hash", pTable->name); return code; } @@ -707,7 +708,7 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { SWalHead *pHead = (SWalHead *)((char *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; pHead->len = pWrite->rowSize; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; + pHead->msgType = pTable->id * 10 + SDB_ACTION_INSERT; pWrite->rowData = pHead->cont; (*pTable->fpEncode)(pWrite); @@ -717,7 +718,7 @@ int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { if (pNewWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, insert action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); + pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); @@ -740,7 +741,7 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite) { void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); if (pRow == NULL) { - sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->tableName); + sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->name); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } @@ -748,7 +749,7 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite) { int32_t code = sdbDeleteHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->name); sdbDecRef(pTable, pWrite->pRow); return code; } @@ -775,7 +776,7 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; + pHead->msgType = pTable->id * 10 + SDB_ACTION_DELETE; pWrite->rowData = pHead->cont; (*pTable->fpEncode)(pWrite); @@ -785,7 +786,7 @@ int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { if (pNewWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, delete action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); + pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); @@ -799,13 +800,13 @@ int32_t sdbUpdateRow(SSWriteMsg *pWrite) { void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); if (pRow == NULL) { - sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->tableName); + sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->name); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } int32_t code = sdbUpdateHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to update hash", pTable->tableName); + sdbError("vgId:1, sdb:%s, failed to update hash", pTable->name); return code; } @@ -830,7 +831,7 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); pHead->version = 0; - pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; + pHead->msgType = pTable->id * 10 + SDB_ACTION_UPDATE; pWrite->rowData = pHead->cont; (*pTable->fpEncode)(pWrite); @@ -840,7 +841,7 @@ int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { if (pNewWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, update action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->tableName, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); + pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); } sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); @@ -888,9 +889,9 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (pTable == NULL) return NULL; pthread_mutex_init(&pTable->mutex, NULL); - tstrncpy(pTable->tableName, pDesc->tableName, SDB_TABLE_LEN); + tstrncpy(pTable->name, pDesc->name, SDB_TABLE_LEN); pTable->keyType = pDesc->keyType; - pTable->tableId = pDesc->tableId; + pTable->id = pDesc->id; pTable->hashSessions = pDesc->hashSessions; pTable->maxRowSize = pDesc->maxRowSize; pTable->refCountPos = pDesc->refCountPos; @@ -909,7 +910,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); tsSdbMgmt.numOfTables++; - tsSdbMgmt.tableList[pTable->tableId] = pTable; + tsSdbMgmt.tableList[pTable->id] = pTable; return pTable; } @@ -918,7 +919,7 @@ void sdbCloseTable(void *handle) { if (pTable == NULL) return; tsSdbMgmt.numOfTables--; - tsSdbMgmt.tableList[pTable->tableId] = NULL; + tsSdbMgmt.tableList[pTable->id] = NULL; SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); while (taosHashIterNext(pIter)) { @@ -937,7 +938,7 @@ void sdbCloseTable(void *handle) { taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); - sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbMgmt.numOfTables); + sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->name, tsSdbMgmt.numOfTables); free(pTable); } @@ -1062,7 +1063,7 @@ static void *sdbWorkerFp(void *pWorker) { pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; if (pWrite->pMsg != NULL) { sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", - pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->tableName, pWrite->pRow, + pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->name, pWrite->pRow, sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version); } } else { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index ce9f652330..a90b573576 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -352,8 +352,8 @@ static int32_t mnodeInitChildTables() { tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_CTABLE, - .tableName = "ctables", + .id = SDB_TABLE_CTABLE, + .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -545,8 +545,8 @@ static int32_t mnodeInitSuperTables() { tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_STABLE, - .tableName = "stables", + .id = SDB_TABLE_STABLE, + .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index edb15fb778..9de14dbfd1 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -150,9 +150,9 @@ int32_t mnodeInitUsers() { SUserObj tObj; tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_USER, - .tableName = "users", + SSdbTableDesc desc = { + .id = SDB_TABLE_USER, + .name = "users", .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE, .maxRowSize = tsUserUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -166,9 +166,9 @@ int32_t mnodeInitUsers() { .fpRestored = mnodeUserActionRestored }; - tsUserSdb = sdbOpenTable(&tableDesc); + tsUserSdb = sdbOpenTable(&desc); if (tsUserSdb == NULL) { - mError("table:%s, failed to create hash", tableDesc.tableName); + mError("table:%s, failed to create hash", desc.name); return -1; } @@ -179,7 +179,7 @@ int32_t mnodeInitUsers() { mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers); mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg); - mDebug("table:%s, hash is created", tableDesc.tableName); + mDebug("table:%s, hash is created", desc.name); return 0; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index c12d2057f8..2f0caea57a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -206,9 +206,9 @@ int32_t mnodeInitVgroups() { SVgObj tObj; tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - SSdbTableDesc tableDesc = { - .tableId = SDB_TABLE_VGROUP, - .tableName = "vgroups", + SSdbTableDesc desc = { + .id = SDB_TABLE_VGROUP, + .name = "vgroups", .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE, .maxRowSize = tsVgUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, @@ -222,7 +222,7 @@ int32_t mnodeInitVgroups() { .fpRestored = mnodeVgroupActionRestored, }; - tsVgroupSdb = sdbOpenTable(&tableDesc); + tsVgroupSdb = sdbOpenTable(&desc); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; From 2171bf7a15dfcc9e96520b72df80d99aac804a8a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 Nov 2020 08:39:16 +0000 Subject: [PATCH 10/75] TD-2157 --- src/sync/src/syncMain.c | 100 ++++++++++++++++++------------------- src/sync/src/syncRestore.c | 5 +- src/sync/src/taosTcpPool.c | 2 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 895b1cd098..10ed9f6c27 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -341,12 +341,10 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_unlock(&(pNode->mutex)); - sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, - syncRole[nodeRole]); + sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); taosReleaseRef(tsSyncRefId, rid); - return 0; } @@ -378,7 +376,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { pFwdRsp->code = code; int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int32_t retLen = write(pPeer->peerFd, msg, msgLen); + int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen); if (retLen == msgLen) { sDebug("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); @@ -391,6 +389,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { taosReleaseRef(tsSyncRefId, rid); } +#if 0 void syncRecover(int64_t rid) { SSyncPeer *pPeer; @@ -417,6 +416,7 @@ void syncRecover(int64_t rid) { taosReleaseRef(tsSyncRefId, rid); } +#endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); @@ -532,7 +532,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; - sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs); + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); } @@ -566,8 +566,6 @@ static void syncChooseMaster(SSyncNode *pNode) { int32_t index = -1; int32_t replica = pNode->replica; - sDebug("vgId:%d, choose master", pNode->vgId); - for (int32_t i = 0; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; @@ -633,11 +631,11 @@ static void syncChooseMaster(SSyncNode *pNode) { static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { int32_t onlineNum = 0; - int32_t index = -1; + int32_t masterIndex = -1; int32_t replica = pNode->replica; - for (int32_t i = 0; i < pNode->replica; ++i) { - if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { + for (int32_t index = 0; index < pNode->replica; ++index) { + if (pNode->peerInfo[index]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } } @@ -652,18 +650,17 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - // pNode->peerInfo[pNode->selfIndex]->role = nodeRole; (*pNode->notifyRole)(pNode->ahandle, nodeRole); - sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int32_t i = 0; i < pNode->replica; ++i) { - SSyncPeer *pTemp = pNode->peerInfo[i]; + for (int32_t index = 0; index < pNode->replica; ++index) { + SSyncPeer *pTemp = pNode->peerInfo[index]; if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; - if (index < 0) { - index = i; + if (masterIndex < 0) { + masterIndex = index; } else { // multiple masters, it shall not happen - if (i == pNode->selfIndex) { + if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); @@ -672,77 +669,80 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { } } - SSyncPeer *pMaster = (index >= 0) ? pNode->peerInfo[index] : NULL; + SSyncPeer *pMaster = (masterIndex >= 0) ? pNode->peerInfo[masterIndex] : NULL; return pMaster; } static int32_t syncValidateMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int32_t code = 0; + int32_t code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { - sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); + sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == pNode->selfIndex) continue; - syncRestartPeer(pNode->peerInfo[i]); + for (int32_t index = 0; index < pNode->replica; ++index) { + if (index == pNode->selfIndex) continue; + syncRestartPeer(pNode->peerInfo[index]); } } return code; } -static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) { +static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t newPeerRole) { SSyncNode *pNode = pPeer->pSyncNode; - int8_t peerOldRole = pPeer->role; - int8_t selfOldRole = nodeRole; - int8_t i, syncRequired = 0; + int8_t oldPeerRole = pPeer->role; + int8_t oldSelfRole = nodeRole; + int8_t syncRequired = 0; - // pNode->peerInfo[pNode->selfIndex]->version = nodeVersion; - pPeer->role = newRole; - - sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]); + pPeer->role = newPeerRole; + sTrace("%s, peer role:%s change to %s", pPeer->id, syncRole[oldPeerRole], syncRole[newPeerRole]); SSyncPeer *pMaster = syncCheckMaster(pNode); if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sTrace("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { if (nodeVersion < pMaster->version) { + sTrace("%s, is master, sync required, self sver:%" PRIu64, pMaster->id, nodeVersion); syncRequired = 1; } else { - sInfo("%s is master, work as slave, sver:%" PRIu64, pMaster->id, pMaster->version); + sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { - // nodeVersion = pMaster->version; + sTrace("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); } } else { // master not there, if all peer's state and version are consistent, choose the master int32_t consistent = 0; - if (peersStatus) { - for (i = 0; i < pNode->replica; ++i) { - SSyncPeer *pTemp = pNode->peerInfo[i]; - if (pTemp->role != peersStatus[i].role) break; - if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[i].version)) break; + int32_t index = 0; + if (peersStatus != NULL) { + for (index = 0; index < pNode->replica; ++index) { + SSyncPeer *pTemp = pNode->peerInfo[index]; + if (pTemp->role != peersStatus[index].role) break; + if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[index].version)) break; } - if (i >= pNode->replica) consistent = 1; + if (index >= pNode->replica) consistent = 1; } else { if (pNode->replica == 2) consistent = 1; } if (consistent) { + sTrace("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); + } else { + sTrace("vgId:%d, version inconsistent, cannot choose master", pNode->vgId); } } @@ -750,7 +750,8 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne syncRecoverFromMaster(pMaster); } - if (peerOldRole != newRole || nodeRole != selfOldRole) { + if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) { + sDebug("vgId:%d, roles changed, broadcast status", pNode->vgId); syncBroadcastStatus(pNode); } @@ -760,7 +761,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne } static void syncRestartPeer(SSyncPeer *pPeer) { - sDebug("%s, restart connection", pPeer->id); + sDebug("%s, restart peer connection", pPeer->id); syncClosePeerConn(pPeer); @@ -768,6 +769,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { + sDebug("%s, check peer connection in 1000 ms", pPeer->id); taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } @@ -862,7 +864,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { firstPkt.port = tsSyncPort; taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); - if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { + if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { sError("%s, failed to send sync-req to peer", pPeer->id); } else { nodeSStatus = TAOS_SYNC_STATUS_START; @@ -1001,7 +1003,7 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; } - int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen); + int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { sDebug("%s, status msg is sent, self:%s sver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); @@ -1009,8 +1011,6 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); } - - return; } static void syncSetupPeerConnection(SSyncPeer *pPeer) { @@ -1025,7 +1025,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { - sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno)); + sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -1038,15 +1038,15 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.port = tsSyncPort; firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId - if (write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { + if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { sDebug("%s, connection to peer server is setup", pPeer->id); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); syncAddPeerRef(pPeer); } else { - sDebug("try later"); - close(connFd); + sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); + taosClose(connFd); taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 33bd96ebb3..393b6b09b1 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -291,7 +291,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); @@ -300,7 +300,8 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING); if (syncOpenRecvBuffer(pNode) < 0) { - sError("%s, failed to allocate recv buffer", pPeer->id); + sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); + syncRestartConnection(pPeer); } else { if (syncRestoreDataStepByStep(pPeer) == 0) { sInfo("%s, it is synced successfully", pPeer->id); diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 875528e66b..d1d9815f4a 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -150,7 +150,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { } void taosFreeTcpConn(void *param) { - SConnObj * pConn = (SConnObj *)param; + SConnObj * pConn = param; SThreadObj *pThread = pConn->pThread; sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); From a82d594a81b9efb54153921193f6b66e565a7250 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 Nov 2020 09:48:48 +0000 Subject: [PATCH 11/75] TD-2157 --- src/inc/tsync.h | 18 ++++++------- src/sync/inc/syncInt.h | 30 ++++++++++++--------- src/sync/src/syncMain.c | 59 +++++++++++++++++++++++++++++++---------- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index d57433eba9..967b254992 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -24,18 +24,18 @@ extern "C" { #define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF typedef enum _TAOS_SYNC_ROLE { - TAOS_SYNC_ROLE_OFFLINE, - TAOS_SYNC_ROLE_UNSYNCED, - TAOS_SYNC_ROLE_SYNCING, - TAOS_SYNC_ROLE_SLAVE, - TAOS_SYNC_ROLE_MASTER, + TAOS_SYNC_ROLE_OFFLINE = 0, + TAOS_SYNC_ROLE_UNSYNCED = 1, + TAOS_SYNC_ROLE_SYNCING = 2, + TAOS_SYNC_ROLE_SLAVE = 3, + TAOS_SYNC_ROLE_MASTER = 4 } ESyncRole; typedef enum _TAOS_SYNC_STATUS { - TAOS_SYNC_STATUS_INIT, - TAOS_SYNC_STATUS_START, - TAOS_SYNC_STATUS_FILE, - TAOS_SYNC_STATUS_CACHE, + TAOS_SYNC_STATUS_INIT = 0, + TAOS_SYNC_STATUS_START = 1, + TAOS_SYNC_STATUS_FILE = 2, + TAOS_SYNC_STATUS_CACHE = 3 } ESyncStatus; typedef struct { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 666a90c3cb..7d846ebc80 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -27,16 +27,20 @@ extern "C" { #define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} -#define TAOS_SMSG_SYNC_DATA 1 -#define TAOS_SMSG_FORWARD 2 -#define TAOS_SMSG_FORWARD_RSP 3 -#define TAOS_SMSG_SYNC_REQ 4 -#define TAOS_SMSG_SYNC_RSP 5 -#define TAOS_SMSG_SYNC_MUST 6 -#define TAOS_SMSG_STATUS 7 +typedef enum { + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_FORWARD = 2, + TAOS_SMSG_FORWARD_RSP = 3, + TAOS_SMSG_SYNC_REQ = 4, + TAOS_SMSG_SYNC_RSP = 5, + TAOS_SMSG_SYNC_MUST = 6, + TAOS_SMSG_STATUS = 7 +} ESyncMsgType; #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) +#define SYNC_FWD_TIMER 300 +#define SYNC_ROLE_TIMER 10000 #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version @@ -123,12 +127,12 @@ typedef struct SsyncPeer { int32_t nodeId; uint32_t ip; uint16_t port; + int8_t role; + int8_t sstatus; // sync status char fqdn[TSDB_FQDN_LEN]; // peer ip string char id[TSDB_EP_LEN + 32]; // peer vgId + end point - int8_t role; - int8_t sstatus; // sync status uint64_t version; - uint64_t sversion; // track the peer version in retrieve process + uint64_t sversion; // track the peer version in retrieve process int32_t syncFd; int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried @@ -138,7 +142,7 @@ typedef struct SsyncPeer { int32_t notifyFd; int32_t watchNum; int32_t *watchFd; - int8_t refCount; // reference count + int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -146,16 +150,16 @@ typedef struct SSyncNode { char path[TSDB_FILENAME_LEN]; int8_t replica; int8_t quorum; + int8_t selfIndex; uint32_t vgId; int64_t rid; void *ahandle; - int8_t selfIndex; SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator SSyncPeer *pMaster; - int8_t refCount; SRecvBuffer *pRecv; SSyncFwds *pSyncFwds; // saved forward info if quorum >1 void *pFwdTimer; + void *pRoleTimer; FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 10ed9f6c27..e7086626d6 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -59,6 +59,7 @@ static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); +static void syncMonitorNodeRole(void *param, void *tmrId); static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); @@ -79,7 +80,9 @@ typedef enum { SYNC_STATUS_SETUP_CONN, SYNC_STATUS_SETUP_CONN_RSP, SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP } ESyncStatusType; char *statusType[] = { @@ -88,7 +91,9 @@ char *statusType[] = { "setup-conn", "setup-conn-rsp", "exchange-data", - "exchange-data-rsp" + "exchange-data-rsp", + "check-role", + "check-role-rsp" }; uint16_t syncGenTranId() { @@ -233,9 +238,16 @@ int64_t syncStart(const SSyncInfo *pInfo) { return -1; } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); if (pNode->pFwdTimer == NULL) { - sError("vgId:%d, failed to allocate timer", pNode->vgId); + sError("vgId:%d, failed to allocate fwd timer", pNode->vgId); + syncStop(pNode->rid); + return -1; + } + + pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); + if (pNode->pRoleTimer == NULL) { + sError("vgId:%d, failed to allocate role timer", pNode->vgId); syncStop(pNode->rid); return -1; } @@ -262,6 +274,7 @@ void syncStop(int64_t rid) { if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); + if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer); for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; @@ -471,10 +484,10 @@ static void syncFreeNode(void *param) { tfree(pNode); } -void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } +void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); } int32_t syncDecPeerRef(SSyncPeer *pPeer) { - if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { + if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) { taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); sDebug("%s, resource is freed", pPeer->id); @@ -699,20 +712,20 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new int8_t syncRequired = 0; pPeer->role = newPeerRole; - sTrace("%s, peer role:%s change to %s", pPeer->id, syncRole[oldPeerRole], syncRole[newPeerRole]); + sDebug("%s, peer role:%s change to %s", pPeer->id, syncRole[oldPeerRole], syncRole[newPeerRole]); SSyncPeer *pMaster = syncCheckMaster(pNode); if (pMaster) { // master is there pNode->pMaster = pMaster; - sTrace("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { if (nodeVersion < pMaster->version) { - sTrace("%s, is master, sync required, self sver:%" PRIu64, pMaster->id, nodeVersion); + sDebug("%s, is master, sync required, self sver:%" PRIu64, pMaster->id, nodeVersion); syncRequired = 1; } else { sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); @@ -720,7 +733,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new (*pNode->notifyRole)(pNode->ahandle, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { - sTrace("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); + sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); } } else { // master not there, if all peer's state and version are consistent, choose the master @@ -739,10 +752,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (consistent) { - sTrace("vgId:%d, choose master", pNode->vgId); + sDebug("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); } else { - sTrace("vgId:%d, version inconsistent, cannot choose master", pNode->vgId); + sDebug("vgId:%d, version inconsistent, cannot choose master", pNode->vgId); } } @@ -1221,8 +1234,26 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } } +static void syncMonitorNodeRole(void *param, void *tmrId) { + int64_t rid = (int64_t)param; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; + + for (int32_t index = 0; index < pNode->replica; index++) { + if (index == pNode->selfIndex) continue; + + SSyncPeer *pPeer = pNode->peerInfo[index]; + if (pPeer->role <= TAOS_SYNC_ROLE_UNSYNCED || nodeRole <= TAOS_SYNC_ROLE_UNSYNCED) { + syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); + } + } + + pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); + taosReleaseRef(tsSyncRefId, rid); +} + static void syncMonitorFwdInfos(void *param, void *tmrId) { - int64_t rid = (int64_t) param; + int64_t rid = (int64_t)param; SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return; @@ -1246,7 +1277,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); } taosReleaseRef(tsSyncRefId, rid); From 5029e7da3f45b072d452f32dff8caf6cd730382c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 Nov 2020 09:53:34 +0000 Subject: [PATCH 12/75] TD-2153 --- src/sync/src/syncMain.c | 8 ++++---- src/sync/src/syncRestore.c | 10 +++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index e7086626d6..2ca77fa404 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -172,7 +172,7 @@ void syncCleanUp() { int64_t syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; - SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1); + SSyncNode *pNode = calloc(sizeof(SSyncNode), 1); if (pNode == NULL) { sError("no memory to allocate syncNode"); terrno = TAOS_SYSTEM_ERROR(errno); @@ -207,8 +207,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); if (pNode->peerInfo[i] == NULL) { - sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn, - pNodeInfo->nodePort); + sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, + pNodeInfo->nodeFqdn, pNodeInfo->nodePort); syncStop(pNode->rid); exit(1); } @@ -419,7 +419,7 @@ void syncRecover(int64_t rid) { pthread_mutex_lock(&(pNode->mutex)); for (int32_t i = 0; i < pNode->replica; ++i) { - pPeer = (SSyncPeer *)pNode->peerInfo[i]; + pPeer = pNode->peerInfo[i]; if (pPeer->peerFd >= 0) { syncRestartConnection(pPeer); } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 393b6b09b1..d3dbe3a32d 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -140,6 +140,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { if (buffer == NULL) return -1; SWalHead *pHead = (SWalHead *)buffer; + uint64_t lastVer = 0; while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); @@ -153,7 +154,14 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); if (ret < 0) break; - sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); + + if (lastVer != 0 && lastVer == pHead->version) { + sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer); + break; + } + lastVer = pHead->version; + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); } From 3b6b05691f9c2516b2d086861a133e53516ade34 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 19 Nov 2020 17:54:27 +0800 Subject: [PATCH 13/75] [TD-1355] add test case, fix queryInterval --- tests/pytest/query/queryInterval.py | 26 ++++++++++++++++++++------ tests/pytest/tools/taosdemo.py | 6 ++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index ec3255ba7f..98b8c9fbef 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -32,30 +32,44 @@ class TDTestCase: tdSql.execute("insert into t0 using st tags('beijing') values(%d, 220) (%d, 221) (%d, 225) (%d, 228) (%d, 222)" % (self.ts, self.ts + 1000000000, self.ts + 2000000000, self.ts + 3000000000, self.ts + 6000000000)) tdSql.execute("insert into t1 using st tags('shanghai') values(%d, 220) (%d, 221) (%d, 225) (%d, 228) (%d, 222)" - % (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000)) + % (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000)) + tdSql.query("select avg(voltage) from st interval(1n)") - tdSql.checkRows(4) - tdSql.checkData(0, 1, 220.0) - tdSql.checkData(1, 1, 222.33333333333334) - tdSql.checkData(2, 1, 227.0) - tdSql.checkData(3, 1, 222.0) + tdSql.checkRows(3) + tdSql.checkData(0, 0, "2020-07-01 00:00:00") + tdSql.checkData(0, 1, 221.4) + tdSql.checkData(1, 0, "2020-08-01 00:00:00") + tdSql.checkData(1, 1, 227.0) + tdSql.checkData(2, 0, "2020-09-01 00:00:00") + tdSql.checkData(2, 1, 222.0) tdSql.query("select avg(voltage) from st interval(1n, 15d)") tdSql.checkRows(4) + tdSql.checkData(0, 0, "2020-06-16 00:00:00") tdSql.checkData(0, 1, 220.333333) + tdSql.checkData(1, 0, "2020-07-16 00:00:00") tdSql.checkData(1, 1, 224.666666) + tdSql.checkData(2, 0, "2020-08-16 00:00:00") tdSql.checkData(2, 1, 225.0) + tdSql.checkData(3, 0, "2020-09-16 00:00:00") tdSql.checkData(3, 1, 222.0) tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc") tdSql.checkRows(7) + tdSql.checkData(0, 0, "2020-06-16 00:00:00") tdSql.checkData(0, 1, 220.5) + tdSql.checkData(1, 0, "2020-07-16 00:00:00") tdSql.checkData(1, 1, 226.5) + tdSql.checkData(2, 0, "2020-08-16 00:00:00") tdSql.checkData(2, 1, 222.0) + tdSql.checkData(3, 0, "2020-06-16 00:00:00") tdSql.checkData(3, 1, 220.0) + tdSql.checkData(4, 0, "2020-07-16 00:00:00") tdSql.checkData(4, 1, 221.0) + tdSql.checkData(5, 0, "2020-08-16 00:00:00") tdSql.checkData(5, 1, 226.5) + tdSql.checkData(6, 0, "2020-09-16 00:00:00") tdSql.checkData(6, 1, 222.0) def stop(self): diff --git a/tests/pytest/tools/taosdemo.py b/tests/pytest/tools/taosdemo.py index 5bf8ebaf03..1a976aef59 100644 --- a/tests/pytest/tools/taosdemo.py +++ b/tests/pytest/tools/taosdemo.py @@ -55,6 +55,12 @@ class TDTestCase: tdSql.query("select count(*) from meters") tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords) + tdSql.query("select sum(f1) from test.meters interval(1h) sliding(30m)") + tdSql.checkRows(2) + + tdSql.query("select apercentile(f1, 1) from test.meters interval(10s)") + tdSql.checkRows(11) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 976d8653bb3791923a5c6ac1ca3d827c6e2d8a12 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 Nov 2020 09:57:57 +0000 Subject: [PATCH 14/75] TD-2153 --- src/sync/src/syncRetrieve.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 52d1bded31..03f5a7bd94 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -418,7 +418,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { - sDebug("%s, wal retrieve is finished", pPeer->id); + sInfo("%s, wal retrieve is finished", pPeer->id); pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); @@ -447,7 +447,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sDebug("%s, start to retrieve file", pPeer->id); + sInfo("%s, start to retrieve file", pPeer->id); if (syncRetrieveFile(pPeer) < 0) { sError("%s, failed to retrieve file", pPeer->id); return -1; @@ -456,7 +456,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { // if no files are synced, there must be wal to sync, sversion must be larger than one if (pPeer->sversion == 0) pPeer->sversion = 1; - sDebug("%s, start to retrieve wal", pPeer->id); + sInfo("%s, start to retrieve wal", pPeer->id); if (syncRetrieveWal(pPeer) < 0) { sError("%s, failed to retrieve wal", pPeer->id); return -1; @@ -478,7 +478,7 @@ void *syncRetrieveData(void *param) { sInfo("%s, sync tcp is setup", pPeer->id); if (syncRetrieveDataStepByStep(pPeer) == 0) { - sDebug("%s, sync retrieve process is successful", pPeer->id); + sInfo("%s, sync retrieve process is successful", pPeer->id); } else { sError("%s, failed to retrieve data, restart connection", pPeer->id); syncRestartConnection(pPeer); From a8b87ce0cd79c498fc370f7c4656f335115256fd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 20 Nov 2020 01:07:33 +0000 Subject: [PATCH 15/75] change applications --- src/query/src/qResultbuf.c | 8 +- src/util/inc/hash.h | 53 ++---- src/util/src/hash.c | 332 ++++++++++++++++++------------------- src/util/src/tkvstore.c | 19 +-- src/vnode/src/vnodeMain.c | 28 ++-- 5 files changed, 191 insertions(+), 249 deletions(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 2645cff678..e9ed8dc1c3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -420,9 +420,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { unlink(pResultBuf->path); tfree(pResultBuf->path); - SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); - while(taosHashIterNext(iter)) { - SArray** p = (SArray**) taosHashIterGet(iter); + SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); + while(p) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); @@ -431,10 +430,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } taosArrayDestroy(*p); + p = taosHashIterate(pResultBuf->groupSet, p); } - taosHashDestroyIter(iter); - tdListFree(pResultBuf->lruList); taosArrayDestroy(pResultBuf->emptyDummyIdList); taosHashCleanup(pResultBuf->groupSet); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 42bc136584..aaff48f4f8 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,16 +31,17 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { -// char *key; struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key -// char *data; + uint32_t dataLen; // length of data + int8_t count; // reference count + char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) - +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -65,15 +66,6 @@ typedef struct SHashObj { SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; -typedef struct SHashMutableIterator { - SHashObj *pHashObj; - int32_t entryIndex; - SHashNode *pCur; - SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current - size_t numOfChecked; // already check number of elements in hash table - size_t numOfEntries; // number of entries while the iterator is created -} SHashMutableIterator; - /** * init the hash table * @@ -142,33 +134,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi */ void taosHashCleanup(SHashObj *pHashObj); -/** - * - * @param pHashObj - * @return - */ -SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj); - -/** - * - * @param iter - * @return - */ -bool taosHashIterNext(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void *taosHashIterGet(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void* taosHashDestroyIter(SHashMutableIterator* iter); +/* +void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); +*/ /** * @@ -179,6 +147,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); size_t taosHashGetMemSize(const SHashObj *pHashObj); +void *taosHashIterate(SHashObj *pHashObj, void *p); +void taosHashCancelIterate(SHashObj *pHashObj, void *p); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 03a7342497..cd9a9f7d0a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); + + pNode->count--; if (prev != NULL) { prev->next = pNewNode; } else { pe->next = pNewNode; } - pNewNode->next = pNode->next; + if (pNode->count <= 0) { + pNewNode->next = pNode->next; + DO_FREE_HASH_NODE(pNode); + } else { + pNewNode->next = pNode; + pe->num++; + atomic_add_fetch_64(&pHashObj->size, 1); + } + return pNewNode; } @@ -139,7 +149,6 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); * @param pIter * @return */ -static SHashNode *getNextHashNode(SHashMutableIterator *pIter); SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (capacity == 0 || fn == NULL) { @@ -244,8 +253,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pe, prev, pNode, pNewNode); - DO_FREE_HASH_NODE(pNode); + doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { DO_FREE_HASH_NODE(pNewNode); } @@ -335,22 +343,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - // no data, return directly - if (pe->num == 0) { - __rd_unlock(&pHashObj->lock, pHashObj->type); - return -1; - } - if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - // double check after locked if (pe->num == 0) { assert(pe->next == NULL); @@ -360,37 +356,36 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe return -1; } + int code = -1; SHashNode *pNode = pe->next; - SHashNode *pRes = NULL; + SHashNode *prevNode = NULL; - // remove it - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { - pe->num -= 1; - pRes = pNode; - pe->next = pNode->next; - } else { - while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { - assert((pNode->next)->hashVal == hashVal); - break; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + code = 0; // it is found + + pNode->count--; + if (pNode->count <= 0) { + if (prevNode) { + prevNode->next = pNode->next; + } else { + pe->next = pNode->next; } + + if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pNode = pNode->next; + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pNode); } - - - if (pNode->next != NULL) { - pe->num -= 1; - pRes = pNode->next; - pNode->next = pNode->next->next; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } + } if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); @@ -398,17 +393,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); - if (data != NULL && pRes != NULL) { - memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); - } - - if (pRes != NULL) { - atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pRes); - return 0; - } else { - return -1; - } + return code; } int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { @@ -531,98 +516,6 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj); } -SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { - SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator)); - if (pIter == NULL) { - return NULL; - } - - pIter->pHashObj = pHashObj; - - // keep it in local variable, in case the resize operation expand the size - pIter->numOfEntries = pHashObj->capacity; - return pIter; -} - -bool taosHashIterNext(SHashMutableIterator *pIter) { - if (pIter == NULL) { - return false; - } - - size_t size = taosHashGetSize(pIter->pHashObj); - if (size == 0) { - return false; - } - - // check the first one - if (pIter->numOfChecked == 0) { - assert(pIter->pCur == NULL && pIter->pNext == NULL); - - while (1) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - pIter->pCur = pEntry->next; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - } else { - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - pIter->pNext = getNextHashNode(pIter); - } - - break; - } - - pIter->numOfChecked++; - return true; - } else { - assert(pIter->pCur != NULL); - if (pIter->pNext) { - pIter->pCur = pIter->pNext; - } else { // no more data in the hash list - return false; - } - - pIter->numOfChecked++; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - } else { - pIter->pNext = getNextHashNode(pIter); - } - - return true; - } -} - -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } - -void *taosHashDestroyIter(SHashMutableIterator *iter) { - if (iter == NULL) { - return NULL; - } - - free(iter); - return NULL; -} - // for profile only int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { if (pHashObj == NULL || pHashObj->size == 0) { @@ -759,6 +652,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + pNewNode->dataLen = dsize; + pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -775,35 +670,6 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pEntry->num += 1; } -SHashNode *getNextHashNode(SHashMutableIterator *pIter) { - assert(pIter != NULL); - - pIter->entryIndex++; - SHashNode *p = NULL; - - while (pIter->entryIndex < pIter->numOfEntries) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - p = pEntry->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - return p; - } - - return NULL; -} - size_t taosHashGetMemSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; @@ -811,3 +677,119 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } + +// release the pNode, return next pNode, and lock the current entry +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { + + SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); + SHashNode *prevNode = NULL; + + *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[*slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + + while (pNode) { + if (pNode == pOld) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + pNode = pNode->next; + pOld->count--; + if (pOld->count <=0) { + if (prevNode) { + prevNode->next = pOld->next; + } else { + pe->next = pOld->next; + } + + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pOld); + } + } else { + uError("pNode:%p data:%p is not there!!!", pNode, p); + } + + return pNode; +} + +void *taosHashIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL) return NULL; + + int slot = 0; + char *data = NULL; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + SHashNode *pNode = NULL; + if (p) { + pNode = taosHashReleaseNode(pHashObj, p, &slot); + if (pNode == NULL) { + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + slot = slot + 1; + } + } + + if (pNode == NULL) { + for (; slot < pHashObj->capacity; ++slot) { + SHashEntry *pe = pHashObj->hashList[slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + pNode = pe->next; + if (pNode) break; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + } + + if (pNode) { + SHashEntry *pe = pHashObj->hashList[slot]; + pNode->count++; + data = GET_HASH_NODE_DATA(pNode); + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; + +} + +void taosHashCancelIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL || p == NULL) return; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int slot; + taosHashReleaseNode(pHashObj, p, &slot); + + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); +} diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 31641ac9a7..2b1d13c78b 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -529,7 +529,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { void * buf = NULL; int64_t maxBufSize = 0; SKVRecord rInfo = {0}; - SHashMutableIterator *pIter = NULL; + SKVRecord *pRecord = NULL; ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); @@ -582,16 +582,8 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - pIter = taosHashCreateIter(pStore->map); - if (pIter == NULL) { - uError("failed to create hash iter while opening KV store %s", pStore->fname); - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - while (taosHashIterNext(pIter)) { - SKVRecord *pRecord = taosHashIterGet(pIter); - + pRecord = taosHashIterate(pStore->map, NULL); + while (pRecord) { if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) { uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); @@ -613,16 +605,17 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } } + + pRecord = taosHashIterate(pStore->map, pRecord); } if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); - taosHashDestroyIter(pIter); tfree(buf); return 0; _err: - taosHashDestroyIter(pIter); + taosHashCancelIterate(pStore->map, pRecord); tfree(buf); return -1; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7447acc488..e3eb35ea32 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -517,11 +517,10 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { (*numOfVnodes)++; if (*numOfVnodes >= TSDB_MAX_VNODES) { @@ -530,25 +529,24 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { } else { vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; } - } - taosHashDestroyIter(pIter); + } + + pIter = taosHashIterate(tsVnodesHash, pIter); + } return TSDB_CODE_SUCCESS; } void vnodeBuildStatusMsg(void *param) { SStatusMsg *pStatus = param; - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { vnodeBuildVloadMsg(*pVnode, pStatus); + } } - - taosHashDestroyIter(pIter); } void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { From 566b749cc5198a07eb13eef2a0374a9aa5d84c6c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 12:32:42 +0800 Subject: [PATCH 16/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 5 +- src/mnode/src/mnodeSdb.c | 256 ++++++++++++++---------------------- src/mnode/src/mnodeTable.c | 2 +- src/mnode/src/mnodeVgroup.c | 2 +- src/sync/src/syncRestore.c | 2 +- tests/script/sh/deploy.sh | 36 ++--- 6 files changed, 125 insertions(+), 178 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 04fee5f6d5..965baf7c0d 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "mnode.h" +#include "twal.h" struct SSdbTable; @@ -60,6 +61,8 @@ typedef struct SSWriteMsg { void * pRow; SMnodeMsg *pMsg; struct SSdbTable *pTable; + char reserveForSync[16]; + SWalHead pHead[]; } SSWriteMsg; typedef struct { @@ -89,7 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbInsertRow(SSWriteMsg *pWrite); int32_t sdbDeleteRow(SSWriteMsg *pWrite); int32_t sdbUpdateRow(SSWriteMsg *pWrite); -int32_t sdbInsertRowImp(SSWriteMsg *pWrite); +int32_t sdbInsertRowToQueue(SSWriteMsg *pWrite); void *sdbGetRow(void *pTable, void *key); void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 12cf5757b3..15e638d436 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -34,7 +34,7 @@ #include "mnodeSdb.h" #define SDB_TABLE_LEN 12 -#define SDB_SYNC_HACK 16 +#define MAX_QUEUED_MSG_NUM 10000 typedef enum { SDB_ACTION_INSERT = 0, @@ -82,6 +82,7 @@ typedef struct { int64_t sync; void * wal; SSyncCfg cfg; + int32_t queuedMsg; int32_t numOfTables; SSdbTable *tableList[SDB_TABLE_MAX]; pthread_mutex_t mutex; @@ -105,16 +106,14 @@ static taos_qall tsSdbWQall; static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; -static int32_t sdbWrite(void *pWrite, void *pHead, int32_t qtype, void *unused); -static int32_t sdbWriteToQueue(void *pWrite, void *pHead, int32_t qtype, void *unused); +static int32_t sdbProcessWrite(void *pWrite, void *pHead, int32_t qtype, void *unused); +static int32_t sdbWriteWalToQueue(void *vparam, void *pHead, int32_t qtype, void *rparam); +static int32_t sdbWriteRowToQueue(SSWriteMsg *pInputWrite, int32_t action); static void * sdbWorkerFp(void *pWorker); static int32_t sdbInitWorker(); static void sdbCleanupWorker(); static int32_t sdbAllocQueue(); static void sdbFreeQueue(); -extern int32_t sdbInsertRowImp(SSWriteMsg *pWrite); -static int32_t sdbUpdateRowImp(SSWriteMsg *pWrite); -static int32_t sdbDeleteRowImp(SSWriteMsg *pWrite); static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite); static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite); static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite); @@ -181,7 +180,7 @@ static int32_t sdbInitWal() { } sdbInfo("vgId:1, open wal for restore"); - int code = walRestore(tsSdbMgmt.wal, NULL, sdbWrite); + int32_t code = walRestore(tsSdbMgmt.wal, NULL, sdbProcessWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code)); return -1; @@ -250,7 +249,7 @@ static void sdbNotifyRole(void *ahandle, int8_t role) { // failed to forward, need revert insert static void sdbHandleFailedConfirm(SSWriteMsg *pWrite) { - SWalHead *pHead = (SWalHead *)((char *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); + SWalHead *pHead = pWrite->pHead; int32_t action = pHead->msgType % 10; sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pWrite->pRow, @@ -285,13 +284,6 @@ static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { } dnodeSendRpcMWriteRsp(pMsg, pWrite->code); - - // if ahandle, means this func is called by sdb write - if (ahandle == NULL) { - sdbDecRef(pWrite->pTable, pWrite->pRow); - } - - taosFreeQitem(pWrite); } static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); } @@ -379,7 +371,7 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.ahandle = NULL; syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.writeToCache = sdbWriteToQueue; + syncInfo.writeToCache = sdbWriteWalToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; tsSdbMgmt.cfg = syncCfg; @@ -389,6 +381,7 @@ void sdbUpdateSync(void *pMnodes) { } else { tsSdbMgmt.sync = syncStart(&syncInfo); } + sdbUpdateMnodeRoles(); } @@ -565,7 +558,7 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { return TSDB_CODE_SUCCESS; } -static int sdbWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { +static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { SSWriteMsg *pWrite = wparam; SWalHead *pHead = hparam; int32_t tableId = pHead->msgType / 10; @@ -665,8 +658,7 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; if (sdbGetRowFromObj(pTable, pWrite->pRow)) { - sdbError("vgId:1, sdb:%s, failed to insert key:%s since it already exist", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow)); + sdbError("vgId:1, sdb:%s, failed to insert:%s since it exist", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } @@ -675,14 +667,14 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); // let vgId increase from 2 - if (pTable->autoIndex == 1 && strcmp(pTable->name, "vgroups") == 0) { + if (pTable->autoIndex == 1 && pTable->id == SDB_TABLE_VGROUP) { *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); } } int32_t code = sdbInsertHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to insert into hash", pTable->name); + sdbError("vgId:1, sdb:%s, failed to insert:%s into hash", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); return code; } @@ -694,39 +686,10 @@ int32_t sdbInsertRow(SSWriteMsg *pWrite) { if (pWrite->fpReq) { return (*pWrite->fpReq)(pWrite->pMsg); } else { - return sdbInsertRowImp(pWrite); + return sdbWriteRowToQueue(pWrite, SDB_ACTION_INSERT); } } -int32_t sdbInsertRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; - if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - - int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewWrite = taosAllocateQitem(size); - - SWalHead *pHead = (SWalHead *)((char *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); - pHead->version = 0; - pHead->len = pWrite->rowSize; - pHead->msgType = pTable->id * 10 + SDB_ACTION_INSERT; - - pWrite->rowData = pHead->cont; - (*pTable->fpEncode)(pWrite); - pHead->len = pWrite->rowSize; - - memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - - if (pNewWrite->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, insert action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); - } - - sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} - bool sdbCheckRowDeleted(void *tparam, void *pRow) { SSdbTable *pTable = tparam; if (pTable == NULL) return false; @@ -745,55 +708,24 @@ int32_t sdbDeleteRow(SSWriteMsg *pWrite) { return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - sdbIncRef(pTable, pWrite->pRow); - int32_t code = sdbDeleteHash(pTable, pWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->name); - sdbDecRef(pTable, pWrite->pRow); return code; } // just delete data from memory if (pWrite->type != SDB_OPER_GLOBAL) { - sdbDecRef(pTable, pWrite->pRow); return TSDB_CODE_SUCCESS; } if (pWrite->fpReq) { return (*pWrite->fpReq)(pWrite->pMsg); } else { - return sdbDeleteRowImp(pWrite); + return sdbWriteRowToQueue(pWrite, SDB_ACTION_DELETE); } } -int32_t sdbDeleteRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; - if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - - int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewWrite = taosAllocateQitem(size); - - SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); - pHead->version = 0; - pHead->msgType = pTable->id * 10 + SDB_ACTION_DELETE; - - pWrite->rowData = pHead->cont; - (*pTable->fpEncode)(pWrite); - pHead->len = pWrite->rowSize; - - memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - - if (pNewWrite->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, delete action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); - } - - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} - int32_t sdbUpdateRow(SSWriteMsg *pWrite) { SSdbTable *pTable = pWrite->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -818,38 +750,10 @@ int32_t sdbUpdateRow(SSWriteMsg *pWrite) { if (pWrite->fpReq) { return (*pWrite->fpReq)(pWrite->pMsg); } else { - return sdbUpdateRowImp(pWrite); + return sdbWriteRowToQueue(pWrite, SDB_ACTION_UPDATE); } } -int32_t sdbUpdateRowImp(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; - if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - - int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK; - SSWriteMsg *pNewWrite = taosAllocateQitem(size); - - SWalHead *pHead = (SWalHead *)((void *)pNewWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK); - pHead->version = 0; - pHead->msgType = pTable->id * 10 + SDB_ACTION_UPDATE; - - pWrite->rowData = pHead->cont; - (*pTable->fpEncode)(pWrite); - pHead->len = pWrite->rowSize; - - memcpy(pNewWrite, pWrite, sizeof(SSWriteMsg)); - - if (pNewWrite->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s, update action is add to sdb queue", pNewWrite->pMsg->rpcMsg.ahandle, - pNewWrite->pMsg, pTable->name, pWrite->pRow, sdbGetRowStr(pTable, pWrite->pRow)); - } - - sdbIncRef(pNewWrite->pTable, pNewWrite->pRow); - taosWriteQitem(tsSdbWQueue, TAOS_QTYPE_RPC, pNewWrite); - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} - void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { SSdbTable *pTable = tparam; *ppRow = NULL; @@ -942,7 +846,7 @@ void sdbCloseTable(void *handle) { free(pTable); } -int32_t sdbInitWorker() { +static int32_t sdbInitWorker() { tsSdbPool.num = 1; tsSdbPool.worker = calloc(sizeof(SSdbWorker), tsSdbPool.num); @@ -958,7 +862,7 @@ int32_t sdbInitWorker() { return 0; } -void sdbCleanupWorker() { +static void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; if (pWorker->thread) { @@ -979,7 +883,7 @@ void sdbCleanupWorker() { mInfo("vgId:1, sdb write is closed"); } -int32_t sdbAllocQueue() { +static int32_t sdbAllocQueue() { tsSdbWQueue = taosOpenQueue(); if (tsSdbWQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1021,7 +925,7 @@ int32_t sdbAllocQueue() { return TSDB_CODE_SUCCESS; } -void sdbFreeQueue() { +static void sdbFreeQueue() { taosCloseQueue(tsSdbWQueue); taosFreeQall(tsSdbWQall); taosCloseQset(tsSdbWQset); @@ -1030,54 +934,96 @@ void sdbFreeQueue() { tsSdbWQueue = NULL; } -int32_t sdbWriteToQueue(void *wparam, void *hparam, int32_t qtype, void *unsed) { - SWalHead *pHead = hparam; - int32_t size = sizeof(SWalHead) + pHead->len; - SWalHead *pWal = taosAllocateQitem(size); - memcpy(pWal, pHead, size); +static int32_t sdbWriteToQueue(SSWriteMsg *pWrite, int32_t qtype) { + SWalHead *pHead = pWrite->pHead; - taosWriteQitem(tsSdbWQueue, qtype, pWal); - return 0; + if (pHead->len > TSDB_MAX_WAL_SIZE) { + sdbError("vgId:1, wal len:%d exceeds limit, hver:%" PRIu64, pHead->len, pHead->version); + taosFreeQitem(pWrite); + return TSDB_CODE_WAL_SIZE_LIMIT; + } + + int32_t queued = atomic_add_fetch_32(&tsSdbMgmt.queuedMsg, 1); + if (queued > MAX_QUEUED_MSG_NUM) { + sdbDebug("vgId:1, too many msg:%d in sdb queue, flow control", queued); + taosMsleep(1); + } + + sdbIncRef(pWrite->pTable, pWrite->pRow); + + sdbTrace("vgId:1, msg:%p write into to sdb queue", pWrite->pMsg); + taosWriteQitem(tsSdbWQueue, qtype, pWrite); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static void sdbFreeFromQueue(SSWriteMsg *pWrite) { + int32_t queued = atomic_sub_fetch_32(&tsSdbMgmt.queuedMsg, 1); + sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pWrite->pMsg, queued); + + sdbDecRef(pWrite->pTable, pWrite->pRow); + taosFreeQitem(pWrite); +} + +static int32_t sdbWriteWalToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { + SWalHead *pHead = wparam; + + int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pHead->len; + SSWriteMsg *pWrite = taosAllocateQitem(size); + if (pWrite == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + return sdbWriteToQueue(pWrite, qtype); +} + +static int32_t sdbWriteRowToQueue(SSWriteMsg *pInputWrite, int32_t action) { + SSdbTable *pTable = pInputWrite->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + + int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize; + SSWriteMsg *pWrite = taosAllocateQitem(size); + if (pWrite == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + memcpy(pWrite, pInputWrite, sizeof(SSWriteMsg)); + pWrite->processedCount = 1; + + SWalHead *pHead = pWrite->pHead; + pWrite->rowData = pHead->cont; + (*pTable->fpEncode)(pWrite); + + pHead->len = pWrite->rowSize; + pHead->version = 0; + pHead->msgType = pTable->id * 10 + action; + + return sdbWriteToQueue(pWrite, TAOS_QTYPE_RPC); +} + +int32_t sdbInsertRowToQueue(SSWriteMsg *pWrite) { return sdbWriteRowToQueue(pWrite, SDB_ACTION_INSERT); } + static void *sdbWorkerFp(void *pWorker) { - SWalHead *pHead; SSWriteMsg *pWrite; - int32_t qtype; - int32_t numOfMsgs; - void * item; - void * unUsed; + int32_t qtype; + void * unUsed; while (1) { - numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); + int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWQset); break; } for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &qtype, &item); - if (qtype == TAOS_QTYPE_RPC) { - pWrite = (SSWriteMsg *)item; - pWrite->processedCount = 1; - pHead = (void *)pWrite + sizeof(SSWriteMsg) + SDB_SYNC_HACK; - if (pWrite->pMsg != NULL) { - sdbDebug("vgId:1, ahandle:%p msg:%p, sdb:%s row:%p:%s hver:%" PRIu64 ", will be processed in sdb queue", - pWrite->pMsg->rpcMsg.ahandle, pWrite->pMsg, pWrite->pTable->name, pWrite->pRow, - sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version); - } - } else { - pHead = (SWalHead *)item; - pWrite = NULL; - } + taosGetQitem(tsSdbWQall, &qtype, (void **)&pWrite); + sdbTrace("vgId:1, msg:%p, row:%p hver:%" PRIu64 ", will be processed in sdb queue", pWrite->pMsg, pWrite->pRow, + pWrite->pHead->version); - int32_t code = sdbWrite(pWrite, pHead, qtype, NULL); - if (code > 0) code = 0; - if (pWrite) { - pWrite->code = code; - } else { - pHead->len = code; // hackway - } + pWrite->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pWrite : NULL, pWrite->pHead, qtype, NULL); + if (pWrite->code > 0) pWrite->code = 0; + + sdbTrace("vgId:1, msg:%p is processed in sdb queue, code:%x", pWrite->pMsg, pWrite->code); } walFsync(tsSdbMgmt.wal, true); @@ -1085,18 +1031,16 @@ static void *sdbWorkerFp(void *pWorker) { // browse all items, and process them one by one taosResetQitems(tsSdbWQall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &qtype, &item); + taosGetQitem(tsSdbWQall, &qtype, (void **)&pWrite); if (qtype == TAOS_QTYPE_RPC) { - pWrite = (SSWriteMsg *)item; sdbConfirmForward(NULL, pWrite, pWrite->code); } else if (qtype == TAOS_QTYPE_FWD) { - pHead = (SWalHead *)item; - syncConfirmForward(tsSdbMgmt.sync, pHead->version, pHead->len); - taosFreeQitem(item); + syncConfirmForward(tsSdbMgmt.sync, pWrite->pHead->version, pWrite->code); } else { - taosFreeQitem(item); } + + sdbFreeFromQueue(pWrite); } } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a90b573576..587c766e44 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2418,7 +2418,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { .fpRsp = mnodeDoCreateChildTableCb }; - int32_t code = sdbInsertRowImp(&desc); + int32_t code = sdbInsertRowToQueue(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pTable = NULL; mnodeDestroyChildTable(pTable); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 2f0caea57a..2eb11e1def 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -966,7 +966,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .fpRsp = mnodeCreateVgroupCb }; - int32_t code = sdbInsertRowImp(&wmsg); + int32_t code = sdbInsertRowToQueue(&wmsg); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index d3dbe3a32d..e7901e6eb8 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -156,7 +156,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { sDebug("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); - if (lastVer != 0 && lastVer == pHead->version) { + if (lastVer == pHead->version) { sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer); break; } diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 591d7749ea..8fccb1442f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -111,24 +111,24 @@ echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 143" >> $TAOS_CFG -echo "sdbDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 131" >> $TAOS_CFG -echo "vDebugFlag 131" >> $TAOS_CFG -echo "tsdbDebugFlag 131" >> $TAOS_CFG -echo "cDebugFlag 131" >> $TAOS_CFG -echo "jnidebugFlag 131" >> $TAOS_CFG -echo "odbcdebugFlag 131" >> $TAOS_CFG -echo "httpDebugFlag 131" >> $TAOS_CFG -echo "monitorDebugFlag 131" >> $TAOS_CFG -echo "mqttDebugFlag 131" >> $TAOS_CFG -echo "qdebugFlag 131" >> $TAOS_CFG -echo "rpcDebugFlag 131" >> $TAOS_CFG +echo "mDebugFlag 135" >> $TAOS_CFG +echo "sdbDebugFlag 135" >> $TAOS_CFG +echo "dDebugFlag 135" >> $TAOS_CFG +echo "vDebugFlag 135" >> $TAOS_CFG +echo "tsdbDebugFlag 135" >> $TAOS_CFG +echo "cDebugFlag 135" >> $TAOS_CFG +echo "jnidebugFlag 135" >> $TAOS_CFG +echo "odbcdebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 135" >> $TAOS_CFG +echo "monitorDebugFlag 135" >> $TAOS_CFG +echo "mqttDebugFlag 135" >> $TAOS_CFG +echo "qdebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 131" >> $TAOS_CFG -echo "sdebugFlag 143" >> $TAOS_CFG -echo "wdebugFlag 143" >> $TAOS_CFG -echo "cqdebugFlag 131" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG +echo "sdebugFlag 135" >> $TAOS_CFG +echo "wdebugFlag 135" >> $TAOS_CFG +echo "cqdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG @@ -140,7 +140,7 @@ echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 10" >> $TAOS_CFG +echo "maxVgroupsPerDb 4" >> $TAOS_CFG echo "minTablesPerVnode 4" >> $TAOS_CFG echo "maxTablesPerVnode 1000" >> $TAOS_CFG echo "tableIncStepPerVnode 10000" >> $TAOS_CFG From f323186f78eba56a531049bae8a69888fe6111b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 13:48:05 +0800 Subject: [PATCH 17/75] TD-2046 --- src/mnode/inc/mnodeSdb.h | 46 +++--- src/mnode/src/mnodeAcct.c | 41 +++-- src/mnode/src/mnodeCluster.c | 30 ++-- src/mnode/src/mnodeDb.c | 66 ++++---- src/mnode/src/mnodeDnode.c | 50 +++--- src/mnode/src/mnodeMnode.c | 48 +++--- src/mnode/src/mnodeSdb.c | 286 +++++++++++++++++------------------ src/mnode/src/mnodeTable.c | 216 +++++++++++++------------- src/mnode/src/mnodeUser.c | 54 +++---- src/mnode/src/mnodeVgroup.c | 86 +++++------ 10 files changed, 460 insertions(+), 463 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 965baf7c0d..29d8cf1207 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -23,8 +23,6 @@ extern "C" { #include "mnode.h" #include "twal.h" -struct SSdbTable; - typedef enum { SDB_TABLE_CLUSTER = 0, SDB_TABLE_DNODE = 1, @@ -50,20 +48,20 @@ typedef enum { SDB_OPER_LOCAL = 1 } ESdbOper; -typedef struct SSWriteMsg { - ESdbOper type; - int32_t processedCount; // for sync fwd callback - int32_t code; // for callback in sdb queue - int32_t rowSize; - void * rowData; +typedef struct SSdbRow { + ESdbOper type; + int32_t processedCount; // for sync fwd callback + int32_t code; // for callback in sdb queue + int32_t rowSize; + void * rowData; + void * pObj; + void * pTable; + SMnodeMsg *pMsg; int32_t (*fpReq)(SMnodeMsg *pMsg); int32_t (*fpRsp)(SMnodeMsg *pMsg, int32_t code); - void * pRow; - SMnodeMsg *pMsg; - struct SSdbTable *pTable; - char reserveForSync[16]; - SWalHead pHead[]; -} SSWriteMsg; + char reserveForSync[16]; + SWalHead pHead[]; +} SSdbRow; typedef struct { char * name; @@ -72,12 +70,12 @@ typedef struct { int32_t refCountPos; ESdbTable id; ESdbKey keyType; - int32_t (*fpInsert)(SSWriteMsg *pWrite); - int32_t (*fpDelete)(SSWriteMsg *pWrite); - int32_t (*fpUpdate)(SSWriteMsg *pWrite); - int32_t (*fpEncode)(SSWriteMsg *pWrite); - int32_t (*fpDecode)(SSWriteMsg *pWrite); - int32_t (*fpDestroy)(SSWriteMsg *pWrite); + int32_t (*fpInsert)(SSdbRow *pRow); + int32_t (*fpDelete)(SSdbRow *pRow); + int32_t (*fpUpdate)(SSdbRow *pRow); + int32_t (*fpEncode)(SSdbRow *pRow); + int32_t (*fpDecode)(SSdbRow *pRow); + int32_t (*fpDestroy)(SSdbRow *pRow); int32_t (*fpRestored)(); } SSdbTableDesc; @@ -89,10 +87,10 @@ bool sdbIsMaster(); bool sdbIsServing(); void sdbUpdateMnodeRoles(); -int32_t sdbInsertRow(SSWriteMsg *pWrite); -int32_t sdbDeleteRow(SSWriteMsg *pWrite); -int32_t sdbUpdateRow(SSWriteMsg *pWrite); -int32_t sdbInsertRowToQueue(SSWriteMsg *pWrite); +int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbDeleteRow(SSdbRow *pRow); +int32_t sdbUpdateRow(SSdbRow *pRow); +int32_t sdbInsertRowToQueue(SSdbRow *pRow); void *sdbGetRow(void *pTable, void *key); void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 48af408c99..9fff2f0229 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" +#include "tglobal.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -25,36 +26,34 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -#include "tglobal.h" - void * tsAcctSdb = NULL; static int32_t tsAcctUpdateSize; static int32_t mnodeCreateRootAcct(); -static int32_t mnodeAcctActionDestroy(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pRow; +static int32_t mnodeAcctActionDestroy(SSdbRow *pRow) { + SAcctObj *pAcct = pRow->pObj; pthread_mutex_destroy(&pAcct->mutex); - tfree(pWMsg->pRow); + tfree(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionInsert(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pRow; +static int32_t mnodeAcctActionInsert(SSdbRow *pRow) { + SAcctObj *pAcct = pRow->pObj; memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo)); pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS; pthread_mutex_init(&pAcct->mutex, NULL); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDelete(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pRow; +static int32_t mnodeAcctActionDelete(SSdbRow *pRow) { + SAcctObj *pAcct = pRow->pObj; mnodeDropAllUsers(pAcct); mnodeDropAllDbs(pAcct); return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionUpdate(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pRow; +static int32_t mnodeAcctActionUpdate(SSdbRow *pRow) { + SAcctObj *pAcct = pRow->pObj; SAcctObj *pSaved = mnodeGetAcct(pAcct->user); if (pAcct != pSaved) { memcpy(pSaved, pAcct, tsAcctUpdateSize); @@ -64,19 +63,19 @@ static int32_t mnodeAcctActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionEncode(SSWriteMsg *pWMsg) { - SAcctObj *pAcct = pWMsg->pRow; - memcpy(pWMsg->rowData, pAcct, tsAcctUpdateSize); - pWMsg->rowSize = tsAcctUpdateSize; +static int32_t mnodeAcctActionEncode(SSdbRow *pRow) { + SAcctObj *pAcct = pRow->pObj; + memcpy(pRow->rowData, pAcct, tsAcctUpdateSize); + pRow->rowSize = tsAcctUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeAcctActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeAcctActionDecode(SSdbRow *pRow) { SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj)); if (pAcct == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pAcct, pWMsg->rowData, tsAcctUpdateSize); - pWMsg->pRow = pAcct; + memcpy(pAcct, pRow->rowData, tsAcctUpdateSize); + pRow->pObj = pAcct; return TSDB_CODE_SUCCESS; } @@ -226,13 +225,13 @@ static int32_t mnodeCreateRootAcct() { pAcct->acctId = sdbGetId(tsAcctSdb); pAcct->createdTime = taosGetTimestampMs(); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsAcctSdb, - .pRow = pAcct, + .pObj = pAcct, }; - return sdbInsertRow(&wmsg); + return sdbInsertRow(&row); } #ifndef _ACCT diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index ff84b7ac3f..5be67e4ad9 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -32,36 +32,36 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mnodeClusterActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pRow); +static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) { + tfree(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionInsert(SSWriteMsg *pWMsg) { +static int32_t mnodeClusterActionInsert(SSdbRow *pRow) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDelete(SSWriteMsg *pWMsg) { +static int32_t mnodeClusterActionDelete(SSdbRow *pRow) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionUpdate(SSWriteMsg *pWMsg) { +static int32_t mnodeClusterActionUpdate(SSdbRow *pRow) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionEncode(SSWriteMsg *pWMsg) { - SClusterObj *pCluster = pWMsg->pRow; - memcpy(pWMsg->rowData, pCluster, tsClusterUpdateSize); - pWMsg->rowSize = tsClusterUpdateSize; +static int32_t mnodeClusterActionEncode(SSdbRow *pRow) { + SClusterObj *pCluster = pRow->pObj; + memcpy(pRow->rowData, pCluster, tsClusterUpdateSize); + pRow->rowSize = tsClusterUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeClusterActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeClusterActionDecode(SSdbRow *pRow) { SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj)); if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pCluster, pWMsg->rowData, tsClusterUpdateSize); - pWMsg->pRow = pCluster; + memcpy(pCluster, pRow->rowData, tsClusterUpdateSize); + pRow->pObj = pCluster; return TSDB_CODE_SUCCESS; } @@ -145,13 +145,13 @@ static int32_t mnodeCreateCluster() { mDebug("uid is %s", pCluster->uid); } - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsClusterSdb, - .pRow = pCluster, + .pObj = pCluster, }; - return sdbInsertRow(&wmsg); + return sdbInsertRow(&row); } const char* mnodeGetClusterId() { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 96658ac73c..d121208447 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -56,8 +56,8 @@ static void mnodeDestroyDb(SDbObj *pDb) { tfree(pDb); } -static int32_t mnodeDbActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyDb(pWMsg->pRow); +static int32_t mnodeDbActionDestroy(SSdbRow *pRow) { + mnodeDestroyDb(pRow->pObj); return TSDB_CODE_SUCCESS; } @@ -65,8 +65,8 @@ int64_t mnodeGetDbNum() { return sdbGetNumOfRows(tsDbSdb); } -static int32_t mnodeDbActionInsert(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pRow; +static int32_t mnodeDbActionInsert(SSdbRow *pRow) { + SDbObj *pDb = pRow->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); pthread_mutex_init(&pDb->mutex, NULL); @@ -91,8 +91,8 @@ static int32_t mnodeDbActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDelete(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pRow; +static int32_t mnodeDbActionDelete(SSdbRow *pRow) { + SDbObj *pDb = pRow->pObj; SAcctObj *pAcct = mnodeGetAcct(pDb->acct); mnodeDropAllChildTables(pDb); @@ -107,11 +107,11 @@ static int32_t mnodeDbActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionUpdate(SSWriteMsg *pWMsg) { - SDbObj *pNew = pWMsg->pRow; +static int32_t mnodeDbActionUpdate(SSdbRow *pRow) { + SDbObj *pNew = pRow->pObj; SDbObj *pDb = mnodeGetDb(pNew->name); if (pDb != NULL && pNew != pDb) { - memcpy(pDb, pNew, pWMsg->rowSize); + memcpy(pDb, pNew, pRow->rowSize); free(pNew->vgList); free(pNew); } @@ -120,19 +120,19 @@ static int32_t mnodeDbActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionEncode(SSWriteMsg *pWMsg) { - SDbObj *pDb = pWMsg->pRow; - memcpy(pWMsg->rowData, pDb, tsDbUpdateSize); - pWMsg->rowSize = tsDbUpdateSize; +static int32_t mnodeDbActionEncode(SSdbRow *pRow) { + SDbObj *pDb = pRow->pObj; + memcpy(pRow->rowData, pDb, tsDbUpdateSize); + pRow->rowSize = tsDbUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDbActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeDbActionDecode(SSdbRow *pRow) { SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); if (pDb == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pDb, pWMsg->rowData, tsDbUpdateSize); - pWMsg->pRow = pDb; + memcpy(pDb, pRow->rowData, tsDbUpdateSize); + pRow->pObj = pDb; return TSDB_CODE_SUCCESS; } @@ -412,16 +412,16 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * pMsg->pDb = pDb; mnodeIncDbRef(pDb); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pRow = pDb, + .pObj = pDb, .rowSize = sizeof(SDbObj), .pMsg = pMsg, .fpRsp = mnodeCreateDbCb }; - code = sdbInsertRow(&wmsg); + code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); pMsg->pDb = NULL; @@ -440,8 +440,8 @@ bool mnodeCheckIsMonitorDB(char *db, char *monitordb) { } #if 0 -void mnodePrintVgroups(SDbObj *pDb, char *wmsg) { - mInfo("db:%s, vgroup link from head, wmsg:%s", pDb->name, wmsg); +void mnodePrintVgroups(SDbObj *pDb, char *row) { + mInfo("db:%s, vgroup link from head, row:%s", pDb->name, row); SVgObj *pVgroup = pDb->pHead; while (pVgroup != NULL) { mInfo("vgId:%d", pVgroup->vgId); @@ -807,13 +807,13 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { if (pDb->status) return TSDB_CODE_SUCCESS; pDb->status = true; - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pRow = pDb + .pObj = pDb }; - int32_t code = sdbUpdateRow(&wmsg); + int32_t code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to set dropping state, reason:%s", pDb->name, tstrerror(code)); } @@ -1019,15 +1019,15 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; pDb->cfgVersion++; - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pRow = pDb, + .pObj = pDb, .pMsg = pMsg, .fpRsp = mnodeAlterDbCb }; - code = sdbUpdateRow(&wmsg); + code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter, reason:%s", pDb->name, tstrerror(code)); } @@ -1071,15 +1071,15 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { SDbObj *pDb = pMsg->pDb; mInfo("db:%s, drop db from sdb", pDb->name); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, - .pRow = pDb, + .pObj = pDb, .pMsg = pMsg, .fpRsp = mnodeDropDbCb }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to drop, reason:%s", pDb->name, tstrerror(code)); } @@ -1134,13 +1134,13 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { if (pDb->pAcct == pAcct) { mInfo("db:%s, drop db from sdb for acct:%s is dropped", pDb->name, pAcct->user); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsDbSdb, - .pRow = pDb + .pObj = pDb }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfDbs++; } mnodeDecDbRef(pDb); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 7b52747951..f76533c760 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -87,13 +87,13 @@ static char* offlineReason[] = { "unknown", }; -static int32_t mnodeDnodeActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pRow); +static int32_t mnodeDnodeActionDestroy(SSdbRow *pRow) { + tfree(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionInsert(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pRow; +static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { + SDnodeObj *pDnode = pRow->pObj; if (pDnode->status != TAOS_DN_STATUS_DROPPING) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->lastAccess = tsAccessSquence; @@ -107,8 +107,8 @@ static int32_t mnodeDnodeActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDelete(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pRow; +static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) { + SDnodeObj *pDnode = pRow->pObj; #ifndef _SYNC mnodeDropAllDnodeVgroups(pDnode); @@ -121,11 +121,11 @@ static int32_t mnodeDnodeActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pWMsg) { - SDnodeObj *pNew = pWMsg->pRow; +static int32_t mnodeDnodeActionUpdate(SSdbRow *pRow) { + SDnodeObj *pNew = pRow->pObj; SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId); if (pDnode != NULL && pNew != pDnode) { - memcpy(pDnode, pNew, pWMsg->rowSize); + memcpy(pDnode, pNew, pRow->rowSize); free(pNew); } mnodeDecDnodeRef(pDnode); @@ -134,19 +134,19 @@ static int32_t mnodeDnodeActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionEncode(SSWriteMsg *pWMsg) { - SDnodeObj *pDnode = pWMsg->pRow; - memcpy(pWMsg->rowData, pDnode, tsDnodeUpdateSize); - pWMsg->rowSize = tsDnodeUpdateSize; +static int32_t mnodeDnodeActionEncode(SSdbRow *pRow) { + SDnodeObj *pDnode = pRow->pObj; + memcpy(pRow->rowData, pDnode, tsDnodeUpdateSize); + pRow->rowSize = tsDnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeDnodeActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) { SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); if (pDnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pDnode, pWMsg->rowData, tsDnodeUpdateSize); - pWMsg->pRow = pDnode; + memcpy(pDnode, pRow->rowData, tsDnodeUpdateSize); + pRow->pObj = pDnode; return TSDB_CODE_SUCCESS; } @@ -296,13 +296,13 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode) { } void mnodeUpdateDnode(SDnodeObj *pDnode) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pRow = pDnode + .pObj = pDnode }; - int32_t code = sdbUpdateRow(&wmsg); + int32_t code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnodeId:%d, failed update", pDnode->dnodeId); } @@ -644,15 +644,15 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN); taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pRow = pDnode, + .pObj = pDnode, .rowSize = sizeof(SDnodeObj), .pMsg = pMsg }; - int32_t code = sdbInsertRow(&wmsg); + int32_t code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { int dnodeId = pDnode->dnodeId; tfree(pDnode); @@ -665,14 +665,14 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { } int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDnodeSdb, - .pRow = pDnode, + .pObj = pDnode, .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop from cluster, result:%s", pDnode->dnodeId, tstrerror(code)); } else { diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index a97aef61e9..205bfda4b9 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -58,13 +58,13 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #define mnodeMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock) #endif -static int32_t mnodeMnodeActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pRow); +static int32_t mnodeMnodeActionDestroy(SSdbRow *pRow) { + tfree(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionInsert(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pRow; +static int32_t mnodeMnodeActionInsert(SSdbRow *pRow) { + SMnodeObj *pMnode = pRow->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -76,8 +76,8 @@ static int32_t mnodeMnodeActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDelete(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pRow; +static int32_t mnodeMnodeActionDelete(SSdbRow *pRow) { + SMnodeObj *pMnode = pRow->pObj; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -88,30 +88,30 @@ static int32_t mnodeMnodeActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionUpdate(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pRow; +static int32_t mnodeMnodeActionUpdate(SSdbRow *pRow) { + SMnodeObj *pMnode = pRow->pObj; SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId); if (pMnode != pSaved) { - memcpy(pSaved, pMnode, pWMsg->rowSize); + memcpy(pSaved, pMnode, pRow->rowSize); free(pMnode); } mnodeDecMnodeRef(pSaved); return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionEncode(SSWriteMsg *pWMsg) { - SMnodeObj *pMnode = pWMsg->pRow; - memcpy(pWMsg->rowData, pMnode, tsMnodeUpdateSize); - pWMsg->rowSize = tsMnodeUpdateSize; +static int32_t mnodeMnodeActionEncode(SSdbRow *pRow) { + SMnodeObj *pMnode = pRow->pObj; + memcpy(pRow->rowData, pMnode, tsMnodeUpdateSize); + pRow->rowSize = tsMnodeUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeMnodeActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeMnodeActionDecode(SSdbRow *pRow) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); if (pMnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pMnode, pWMsg->rowData, tsMnodeUpdateSize); - pWMsg->pRow = pMnode; + memcpy(pMnode, pRow->rowData, tsMnodeUpdateSize); + pRow->pObj = pMnode; return TSDB_CODE_SUCCESS; } @@ -325,10 +325,10 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { pMnode->mnodeId = dnodeId; pMnode->createdTime = taosGetTimestampMs(); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsMnodeSdb, - .pRow = pMnode, + .pObj = pMnode, .fpRsp = mnodeCreateMnodeCb }; @@ -342,7 +342,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { return; } - code = sdbInsertRow(&wmsg); + code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code)); tfree(pMnode); @@ -352,8 +352,8 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { void mnodeDropMnodeLocal(int32_t dnodeId) { SMnodeObj *pMnode = mnodeGetMnode(dnodeId); if (pMnode != NULL) { - SSWriteMsg wmsg = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pRow = pMnode}; - sdbDeleteRow(&wmsg); + SSdbRow row = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pObj = pMnode}; + sdbDeleteRow(&row); mnodeDecMnodeRef(pMnode); } @@ -367,13 +367,13 @@ int32_t mnodeDropMnode(int32_t dnodeId) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsMnodeSdb, - .pRow = pMnode + .pObj = pMnode }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); sdbDecRef(tsMnodeSdb, pMnode); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 15e638d436..a79eec16aa 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -65,12 +65,12 @@ typedef struct SSdbTable { int32_t autoIndex; int64_t numOfRows; void * iHandle; - int32_t (*fpInsert)(SSWriteMsg *pWrite); - int32_t (*fpDelete)(SSWriteMsg *pWrite); - int32_t (*fpUpdate)(SSWriteMsg *pWrite); - int32_t (*fpDecode)(SSWriteMsg *pWrite); - int32_t (*fpEncode)(SSWriteMsg *pWrite); - int32_t (*fpDestroy)(SSWriteMsg *pWrite); + int32_t (*fpInsert)(SSdbRow *pRow); + int32_t (*fpDelete)(SSdbRow *pRow); + int32_t (*fpUpdate)(SSdbRow *pRow); + int32_t (*fpDecode)(SSdbRow *pRow); + int32_t (*fpEncode)(SSdbRow *pRow); + int32_t (*fpDestroy)(SSdbRow *pRow); int32_t (*fpRestored)(); pthread_mutex_t mutex; } SSdbTable; @@ -106,17 +106,17 @@ static taos_qall tsSdbWQall; static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; -static int32_t sdbProcessWrite(void *pWrite, void *pHead, int32_t qtype, void *unused); +static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); static int32_t sdbWriteWalToQueue(void *vparam, void *pHead, int32_t qtype, void *rparam); -static int32_t sdbWriteRowToQueue(SSWriteMsg *pInputWrite, int32_t action); +static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); static void * sdbWorkerFp(void *pWorker); static int32_t sdbInitWorker(); static void sdbCleanupWorker(); static int32_t sdbAllocQueue(); static void sdbFreeQueue(); -static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite); -static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite); -static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite); +static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow); +static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow); +static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow); int32_t sdbGetId(void *pTable) { return ((SSdbTable *)pTable)->autoIndex; @@ -248,28 +248,28 @@ static void sdbNotifyRole(void *ahandle, int8_t role) { } // failed to forward, need revert insert -static void sdbHandleFailedConfirm(SSWriteMsg *pWrite) { - SWalHead *pHead = pWrite->pHead; +static void sdbHandleFailedConfirm(SSdbRow *pRow) { + SWalHead *pHead = pRow->pHead; int32_t action = pHead->msgType % 10; - sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pWrite->pRow, - sdbGetKeyStr(pWrite->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pWrite->code)); + sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pRow->pObj, + sdbGetKeyStr(pRow->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pRow->code)); // It's better to create a table in two stages, create it first and then set it success if (action == SDB_ACTION_INSERT) { - SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = pWrite->pTable, .pRow = pWrite->pRow}; - sdbDeleteRow(&wmsg); + SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj}; + sdbDeleteRow(&row); } } FORCE_INLINE static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { if (wparam == NULL) return; - SSWriteMsg *pWrite = wparam; - SMnodeMsg * pMsg = pWrite->pMsg; + SSdbRow *pRow = wparam; + SMnodeMsg * pMsg = pRow->pMsg; - if (code <= 0) pWrite->code = code; - int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); + if (code <= 0) pRow->code = code; + int32_t count = atomic_add_fetch_32(&pRow->processedCount, 1); if (count <= 1) { if (pMsg != NULL) sdbTrace("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, count, code); return; @@ -277,13 +277,13 @@ static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { if (pMsg != NULL) sdbTrace("vgId:1, msg:%p is confirmed, code:%x", pMsg, code); } - if (pWrite->code != TSDB_CODE_SUCCESS) sdbHandleFailedConfirm(pWrite); + if (pRow->code != TSDB_CODE_SUCCESS) sdbHandleFailedConfirm(pRow); - if (pWrite->fpRsp != NULL) { - pWrite->code = (*pWrite->fpRsp)(pMsg, pWrite->code); + if (pRow->fpRsp != NULL) { + pRow->code = (*pRow->fpRsp)(pMsg, pRow->code); } - dnodeSendRpcMWriteRsp(pMsg, pWrite->code); + dnodeSendRpcMWriteRsp(pMsg, pRow->code); } static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); } @@ -447,8 +447,8 @@ void sdbDecRef(void *tparam, void *pRow) { int32_t *updateEnd = pRow + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); - SSWriteMsg wmsg = {.pRow = pRow}; - (*pTable->fpDestroy)(&wmsg); + SSdbRow row = {.pObj = pRow}; + (*pTable->fpDestroy)(&row); } } @@ -485,8 +485,8 @@ static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { return sdbGetRow(pTable, sdbGetObjKey(pTable, key)); } -static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { - void * key = sdbGetObjKey(pTable, pWrite->pRow); +static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { + void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { @@ -494,43 +494,43 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSWriteMsg *pWrite) { } pthread_mutex_lock(&pTable->mutex); - taosHashPut(pTable->iHandle, key, keySize, &pWrite->pRow, sizeof(int64_t)); + taosHashPut(pTable->iHandle, key, keySize, &pRow->pObj, sizeof(int64_t)); pthread_mutex_unlock(&pTable->mutex); - sdbIncRef(pTable, pWrite->pRow); + sdbIncRef(pTable, pRow->pObj); atomic_add_fetch_32(&pTable->numOfRows, 1); if (pTable->keyType == SDB_KEY_AUTO) { - pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pWrite->pRow)); + pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pRow->pObj)); } else { atomic_add_fetch_32(&pTable->autoIndex, 1); } sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow), pWrite->rowSize, pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pRow->pObj), pRow->rowSize, pTable->numOfRows, pRow->pMsg); - int32_t code = (*pTable->fpInsert)(pWrite); + int32_t code = (*pTable->fpInsert)(pRow); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow)); - sdbDeleteHash(pTable, pWrite); + sdbGetRowStr(pTable, pRow->pObj)); + sdbDeleteHash(pTable, pRow); } return TSDB_CODE_SUCCESS; } -static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { - int32_t *updateEnd = pWrite->pRow + pTable->refCountPos - 4; +static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { + int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4; bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow)); + sdbGetRowStr(pTable, pRow->pObj)); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - (*pTable->fpDelete)(pWrite); + (*pTable->fpDelete)(pRow); - void * key = sdbGetObjKey(pTable, pWrite->pRow); + void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); @@ -543,23 +543,23 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSWriteMsg *pWrite) { atomic_sub_fetch_32(&pTable->numOfRows, 1); sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg); - sdbDecRef(pTable, pWrite->pRow); + sdbDecRef(pTable, pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t sdbUpdateHash(SSdbTable *pTable, SSWriteMsg *pWrite) { +static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) { sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, - sdbGetRowStr(pTable, pWrite->pRow), pTable->numOfRows, pWrite->pMsg); + sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg); - (*pTable->fpUpdate)(pWrite); + (*pTable->fpUpdate)(pRow); return TSDB_CODE_SUCCESS; } static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { - SSWriteMsg *pWrite = wparam; + SSdbRow *pRow = wparam; SWalHead *pHead = hparam; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -598,22 +598,22 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus pthread_mutex_unlock(&tsSdbMgmt.mutex); - // from app, wmsg is created - if (pWrite != NULL) { + // from app, row is created + if (pRow != NULL) { // forward to peers - pWrite->processedCount = 0; - int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pWrite, TAOS_QTYPE_RPC); - if (syncCode <= 0) pWrite->processedCount = 1; + pRow->processedCount = 0; + int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC); + if (syncCode <= 0) pRow->processedCount = 1; if (syncCode < 0) { sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, - tstrerror(syncCode), actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + tstrerror(syncCode), actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg); } else if (syncCode > 0) { sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, - actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg); } else { sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name, - actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pWrite->pMsg); + actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg); } return syncCode; } @@ -622,71 +622,71 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync - syncForwardToPeer(tsSdbMgmt.sync, pHead, pWrite, TAOS_QTYPE_RPC); + syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC); - // from wal or forward msg, wmsg not created, should add into hash + // from wal or forward msg, row not created, should add into hash if (action == SDB_ACTION_INSERT) { - SSWriteMsg wmsg = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&wmsg); - return sdbInsertHash(pTable, &wmsg); + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + code = (*pTable->fpDecode)(&row); + return sdbInsertHash(pTable, &row); } else if (action == SDB_ACTION_DELETE) { - void *pRow = sdbGetRowMeta(pTable, pHead->cont); - if (pRow == NULL) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSWriteMsg wmsg = {.pTable = pTable, .pRow = pRow}; - return sdbDeleteHash(pTable, &wmsg); + SSdbRow row = {.pTable = pTable, .pObj = pObj}; + return sdbDeleteHash(pTable, &row); } else if (action == SDB_ACTION_UPDATE) { - void *pRow = sdbGetRowMeta(pTable, pHead->cont); - if (pRow == NULL) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, sdbGetKeyStr(pTable, pHead->cont)); return TSDB_CODE_SUCCESS; } - SSWriteMsg wmsg = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&wmsg); - return sdbUpdateHash(pTable, &wmsg); + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + code = (*pTable->fpDecode)(&row); + return sdbUpdateHash(pTable, &row); } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } } -int32_t sdbInsertRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; +int32_t sdbInsertRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - if (sdbGetRowFromObj(pTable, pWrite->pRow)) { - sdbError("vgId:1, sdb:%s, failed to insert:%s since it exist", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); - sdbDecRef(pTable, pWrite->pRow); + if (sdbGetRowFromObj(pTable, pRow->pObj)) { + sdbError("vgId:1, sdb:%s, failed to insert:%s since it exist", pTable->name, sdbGetRowStr(pTable, pRow->pObj)); + sdbDecRef(pTable, pRow->pObj); return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE; } if (pTable->keyType == SDB_KEY_AUTO) { - *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pRow->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); // let vgId increase from 2 if (pTable->autoIndex == 1 && pTable->id == SDB_TABLE_VGROUP) { - *((uint32_t *)pWrite->pRow) = atomic_add_fetch_32(&pTable->autoIndex, 1); + *((uint32_t *)pRow->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1); } } - int32_t code = sdbInsertHash(pTable, pWrite); + int32_t code = sdbInsertHash(pTable, pRow); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to insert:%s into hash", pTable->name, sdbGetRowStr(pTable, pWrite->pRow)); + sdbError("vgId:1, sdb:%s, failed to insert:%s into hash", pTable->name, sdbGetRowStr(pTable, pRow->pObj)); return code; } // just insert data into memory - if (pWrite->type != SDB_OPER_GLOBAL) { + if (pRow->type != SDB_OPER_GLOBAL) { return TSDB_CODE_SUCCESS; } - if (pWrite->fpReq) { - return (*pWrite->fpReq)(pWrite->pMsg); + if (pRow->fpReq) { + return (*pRow->fpReq)(pRow->pMsg); } else { - return sdbWriteRowToQueue(pWrite, SDB_ACTION_INSERT); + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); } } @@ -698,59 +698,59 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) { return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } -int32_t sdbDeleteRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; +int32_t sdbDeleteRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); - if (pRow == NULL) { + void *pObj = sdbGetRowMetaFromObj(pTable, pRow->pObj); + if (pObj == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->name); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - int32_t code = sdbDeleteHash(pTable, pWrite); + int32_t code = sdbDeleteHash(pTable, pRow); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->name); return code; } // just delete data from memory - if (pWrite->type != SDB_OPER_GLOBAL) { + if (pRow->type != SDB_OPER_GLOBAL) { return TSDB_CODE_SUCCESS; } - if (pWrite->fpReq) { - return (*pWrite->fpReq)(pWrite->pMsg); + if (pRow->fpReq) { + return (*pRow->fpReq)(pRow->pMsg); } else { - return sdbWriteRowToQueue(pWrite, SDB_ACTION_DELETE); + return sdbWriteRowToQueue(pRow, SDB_ACTION_DELETE); } } -int32_t sdbUpdateRow(SSWriteMsg *pWrite) { - SSdbTable *pTable = pWrite->pTable; +int32_t sdbUpdateRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - void *pRow = sdbGetRowMetaFromObj(pTable, pWrite->pRow); - if (pRow == NULL) { + void *pObj = sdbGetRowMetaFromObj(pTable, pRow->pObj); + if (pObj == NULL) { sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->name); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - int32_t code = sdbUpdateHash(pTable, pWrite); + int32_t code = sdbUpdateHash(pTable, pRow); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, sdb:%s, failed to update hash", pTable->name); return code; } // just update data in memory - if (pWrite->type != SDB_OPER_GLOBAL) { + if (pRow->type != SDB_OPER_GLOBAL) { return TSDB_CODE_SUCCESS; } - if (pWrite->fpReq) { - return (*pWrite->fpReq)(pWrite->pMsg); + if (pRow->fpReq) { + return (*pRow->fpReq)(pRow->pMsg); } else { - return sdbWriteRowToQueue(pWrite, SDB_ACTION_UPDATE); + return sdbWriteRowToQueue(pRow, SDB_ACTION_UPDATE); } } @@ -830,12 +830,12 @@ void sdbCloseTable(void *handle) { void **ppRow = taosHashIterGet(pIter); if (ppRow == NULL) continue; - SSWriteMsg wmsg = { - .pRow = *ppRow, + SSdbRow row = { + .pObj = *ppRow, .pTable = pTable, }; - (*pTable->fpDestroy)(&wmsg); + (*pTable->fpDestroy)(&row); } taosHashDestroyIter(pIter); @@ -934,12 +934,12 @@ static void sdbFreeQueue() { tsSdbWQueue = NULL; } -static int32_t sdbWriteToQueue(SSWriteMsg *pWrite, int32_t qtype) { - SWalHead *pHead = pWrite->pHead; +static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) { + SWalHead *pHead = pRow->pHead; if (pHead->len > TSDB_MAX_WAL_SIZE) { sdbError("vgId:1, wal len:%d exceeds limit, hver:%" PRIu64, pHead->len, pHead->version); - taosFreeQitem(pWrite); + taosFreeQitem(pRow); return TSDB_CODE_WAL_SIZE_LIMIT; } @@ -949,64 +949,64 @@ static int32_t sdbWriteToQueue(SSWriteMsg *pWrite, int32_t qtype) { taosMsleep(1); } - sdbIncRef(pWrite->pTable, pWrite->pRow); + sdbIncRef(pRow->pTable, pRow->pObj); - sdbTrace("vgId:1, msg:%p write into to sdb queue", pWrite->pMsg); - taosWriteQitem(tsSdbWQueue, qtype, pWrite); + sdbTrace("vgId:1, msg:%p write into to sdb queue", pRow->pMsg); + taosWriteQitem(tsSdbWQueue, qtype, pRow); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static void sdbFreeFromQueue(SSWriteMsg *pWrite) { +static void sdbFreeFromQueue(SSdbRow *pRow) { int32_t queued = atomic_sub_fetch_32(&tsSdbMgmt.queuedMsg, 1); - sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pWrite->pMsg, queued); + sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pRow->pMsg, queued); - sdbDecRef(pWrite->pTable, pWrite->pRow); - taosFreeQitem(pWrite); + sdbDecRef(pRow->pTable, pRow->pObj); + taosFreeQitem(pRow); } static int32_t sdbWriteWalToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { SWalHead *pHead = wparam; - int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pHead->len; - SSWriteMsg *pWrite = taosAllocateQitem(size); - if (pWrite == NULL) { + int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len; + SSdbRow *pRow = taosAllocateQitem(size); + if (pRow == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } - return sdbWriteToQueue(pWrite, qtype); + return sdbWriteToQueue(pRow, qtype); } -static int32_t sdbWriteRowToQueue(SSWriteMsg *pInputWrite, int32_t action) { - SSdbTable *pTable = pInputWrite->pTable; +static int32_t sdbWriteRowToQueue(SSdbRow *pInputRow, int32_t action) { + SSdbTable *pTable = pInputRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - int32_t size = sizeof(SSWriteMsg) + sizeof(SWalHead) + pTable->maxRowSize; - SSWriteMsg *pWrite = taosAllocateQitem(size); - if (pWrite == NULL) { + int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pTable->maxRowSize; + SSdbRow *pRow = taosAllocateQitem(size); + if (pRow == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } - memcpy(pWrite, pInputWrite, sizeof(SSWriteMsg)); - pWrite->processedCount = 1; + memcpy(pRow, pInputRow, sizeof(SSdbRow)); + pRow->processedCount = 1; - SWalHead *pHead = pWrite->pHead; - pWrite->rowData = pHead->cont; - (*pTable->fpEncode)(pWrite); + SWalHead *pHead = pRow->pHead; + pRow->rowData = pHead->cont; + (*pTable->fpEncode)(pRow); - pHead->len = pWrite->rowSize; + pHead->len = pRow->rowSize; pHead->version = 0; pHead->msgType = pTable->id * 10 + action; - return sdbWriteToQueue(pWrite, TAOS_QTYPE_RPC); + return sdbWriteToQueue(pRow, TAOS_QTYPE_RPC); } -int32_t sdbInsertRowToQueue(SSWriteMsg *pWrite) { return sdbWriteRowToQueue(pWrite, SDB_ACTION_INSERT); } +int32_t sdbInsertRowToQueue(SSdbRow *pRow) { return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); } static void *sdbWorkerFp(void *pWorker) { - SSWriteMsg *pWrite; - int32_t qtype; - void * unUsed; + SSdbRow *pRow; + int32_t qtype; + void * unUsed; while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); @@ -1016,14 +1016,14 @@ static void *sdbWorkerFp(void *pWorker) { } for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &qtype, (void **)&pWrite); - sdbTrace("vgId:1, msg:%p, row:%p hver:%" PRIu64 ", will be processed in sdb queue", pWrite->pMsg, pWrite->pRow, - pWrite->pHead->version); + taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow); + sdbTrace("vgId:1, msg:%p, row:%p hver:%" PRIu64 ", will be processed in sdb queue", pRow->pMsg, pRow->pObj, + pRow->pHead->version); - pWrite->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pWrite : NULL, pWrite->pHead, qtype, NULL); - if (pWrite->code > 0) pWrite->code = 0; + pRow->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pRow : NULL, pRow->pHead, qtype, NULL); + if (pRow->code > 0) pRow->code = 0; - sdbTrace("vgId:1, msg:%p is processed in sdb queue, code:%x", pWrite->pMsg, pWrite->code); + sdbTrace("vgId:1, msg:%p is processed in sdb queue, code:%x", pRow->pMsg, pRow->code); } walFsync(tsSdbMgmt.wal, true); @@ -1031,16 +1031,16 @@ static void *sdbWorkerFp(void *pWorker) { // browse all items, and process them one by one taosResetQitems(tsSdbWQall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(tsSdbWQall, &qtype, (void **)&pWrite); + taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow); if (qtype == TAOS_QTYPE_RPC) { - sdbConfirmForward(NULL, pWrite, pWrite->code); + sdbConfirmForward(NULL, pRow, pRow->code); } else if (qtype == TAOS_QTYPE_FWD) { - syncConfirmForward(tsSdbMgmt.sync, pWrite->pHead->version, pWrite->code); + syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); } else { } - sdbFreeFromQueue(pWrite); + sdbFreeFromQueue(pRow); } } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 587c766e44..29957f8ec0 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -99,13 +99,13 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) { tfree(pTable); } -static int32_t mnodeChildTableActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyChildTable(pWMsg->pRow); +static int32_t mnodeChildTableActionDestroy(SSdbRow *pRow) { + mnodeDestroyChildTable(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionInsert(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pRow; +static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { + SCTableObj *pTable = pRow->pObj; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -153,8 +153,8 @@ static int32_t mnodeChildTableActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDelete(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pRow; +static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) { + SCTableObj *pTable = pRow->pObj; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -189,8 +189,8 @@ static int32_t mnodeChildTableActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pWMsg) { - SCTableObj *pNew = pWMsg->pRow; +static int32_t mnodeChildTableActionUpdate(SSdbRow *pRow) { + SCTableObj *pNew = pRow->pObj; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -216,50 +216,50 @@ static int32_t mnodeChildTableActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionEncode(SSWriteMsg *pWMsg) { - SCTableObj *pTable = pWMsg->pRow; - assert(pTable != NULL && pWMsg->rowData != NULL); +static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) { + SCTableObj *pTable = pRow->pObj; + assert(pTable != NULL && pRow->rowData != NULL); int32_t len = strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; - memcpy(pWMsg->rowData, pTable->info.tableId, len); - memset(pWMsg->rowData + len, 0, 1); + memcpy(pRow->rowData, pTable->info.tableId, len); + memset(pRow->rowData + len, 0, 1); len++; - memcpy(pWMsg->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pWMsg->rowData + len, pTable->schema, schemaSize); + memcpy(pRow->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pWMsg->rowData + len, pTable->sql, pTable->sqlLen); + memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } - pWMsg->rowSize = len; + pRow->rowSize = len; return TSDB_CODE_SUCCESS; } -static int32_t mnodeChildTableActionDecode(SSWriteMsg *pWMsg) { - assert(pWMsg->rowData != NULL); +static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { + assert(pRow->rowData != NULL); SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pWMsg->rowData); + int32_t len = strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; } - pTable->info.tableId = strdup(pWMsg->rowData); + pTable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pWMsg->rowData + len, tsChildTableUpdateSize); + memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -269,7 +269,7 @@ static int32_t mnodeChildTableActionDecode(SSWriteMsg *pWMsg) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pWMsg->rowData + len, schemaSize); + memcpy(pTable->schema, pRow->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -278,11 +278,11 @@ static int32_t mnodeChildTableActionDecode(SSWriteMsg *pWMsg) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pWMsg->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen); } } - pWMsg->pRow = pTable; + pRow->pObj = pTable; return TSDB_CODE_SUCCESS; } @@ -297,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() { SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; + SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); @@ -309,7 +309,7 @@ static int32_t mnodeChildTableActionRestored() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; + SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -320,7 +320,7 @@ static int32_t mnodeChildTableActionRestored() { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; + SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -331,7 +331,7 @@ static int32_t mnodeChildTableActionRestored() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; - SSWriteMsg desc = {.type = SDB_OPER_LOCAL, .pRow = pTable, .pTable = tsChildTableSdb}; + SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); continue; @@ -430,13 +430,13 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) { tfree(pStable); } -static int32_t mnodeSuperTableActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroySuperTable(pWMsg->pRow); +static int32_t mnodeSuperTableActionDestroy(SSdbRow *pRow) { + mnodeDestroySuperTable(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pRow; +static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) { + SSTableObj *pStable = pRow->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); @@ -446,8 +446,8 @@ static int32_t mnodeSuperTableActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pRow; +static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) { + SSTableObj *pStable = pRow->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); @@ -458,8 +458,8 @@ static int32_t mnodeSuperTableActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pWMsg) { - SSTableObj *pNew = pWMsg->pRow; +static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) { + SSTableObj *pNew = pRow->pObj; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { void *oldTableId = pTable->info.tableId; @@ -483,43 +483,43 @@ static int32_t mnodeSuperTableActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionEncode(SSWriteMsg *pWMsg) { - SSTableObj *pStable = pWMsg->pRow; - assert(pWMsg->pRow != NULL && pWMsg->rowData != NULL); +static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) { + SSTableObj *pStable = pRow->pObj; + assert(pRow->pObj != NULL && pRow->rowData != NULL); int32_t len = strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; - memcpy(pWMsg->rowData, pStable->info.tableId, len); - memset(pWMsg->rowData + len, 0, 1); + memcpy(pRow->rowData, pStable->info.tableId, len); + memset(pRow->rowData + len, 0, 1); len++; - memcpy(pWMsg->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pWMsg->rowData + len, pStable->schema, schemaSize); + memcpy(pRow->rowData + len, pStable->schema, schemaSize); len += schemaSize; - pWMsg->rowSize = len; + pRow->rowSize = len; return TSDB_CODE_SUCCESS; } -static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pWMsg) { - assert(pWMsg->rowData != NULL); +static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { + assert(pRow->rowData != NULL); SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pWMsg->rowData); + int32_t len = strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; } - pStable->info.tableId = strdup(pWMsg->rowData); + pStable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pWMsg->rowData + len, tsSuperTableUpdateSize); + memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -529,9 +529,9 @@ static int32_t mnodeSuperTableActionDecode(SSWriteMsg *pWMsg) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pWMsg->rowData + len, schemaSize); + memcpy(pStable->schema, pRow->rowData + len, schemaSize); - pWMsg->pRow = pStable; + pRow->pObj = pStable; return TSDB_CODE_SUCCESS; } @@ -828,7 +828,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pTable, .pTable = tsSuperTableSdb}; + SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsSuperTableSdb}; sdbDeleteRow(&desc); } @@ -878,16 +878,16 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pMsg->pTable = (STableObj *)pStable; mnodeIncTableRef(pMsg->pTable); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .rowSize = sizeof(SSTableObj) + schemaSize, .pMsg = pMsg, .fpRsp = mnodeCreateSuperTableCb }; - int32_t code = sdbInsertRow(&wmsg); + int32_t code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroySuperTable(pStable); pMsg->pTable = NULL; @@ -937,15 +937,15 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mnodeDropAllChildTablesInStable(pStable); } - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeDropSuperTableCb }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("app:%p:%p, table:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); @@ -1010,15 +1010,15 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t mInfo("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, schema[0].name); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeAddSuperTableTagCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { @@ -1044,15 +1044,15 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeDropSuperTableTagCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { @@ -1088,15 +1088,15 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c mInfo("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldTagName, newTagName); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeModifySuperTableTagNameCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) { @@ -1162,15 +1162,15 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeAddSuperTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { @@ -1207,15 +1207,15 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeDropSuperTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { @@ -1251,15 +1251,15 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, oldName, newName); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, - .pRow = pStable, + .pObj = pStable, .pMsg = pMsg, .fpRsp = mnodeChangeSuperTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } // show super tables @@ -1417,12 +1417,12 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsSuperTableSdb, - .pRow = pTable, + .pObj = pTable, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfTables ++; } @@ -1694,7 +1694,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pTable, .pTable = tsChildTableSdb}; + SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); return code; } @@ -1780,9 +1780,9 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { pMsg->pTable = (STableObj *)pTable; mnodeIncTableRef(pMsg->pTable); - SSWriteMsg desc = { + SSdbRow desc = { .type = SDB_OPER_GLOBAL, - .pRow = pTable, + .pObj = pTable, .pTable = tsChildTableSdb, .pMsg = pMsg, .fpReq = mnodeDoCreateChildTableFp @@ -1901,15 +1901,15 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_APP_ERROR; } - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, .pMsg = pMsg, .fpRsp = mnodeDropChildTableCb }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("app:%p:%p, ctable:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); @@ -2005,15 +2005,15 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, .pMsg = pMsg, .fpRsp = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { @@ -2038,15 +2038,15 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, .pMsg = pMsg, .fpRsp = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { @@ -2075,15 +2075,15 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, oldName, newName); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, .pMsg = pMsg, .fpRsp = mnodeAlterNormalTableColumnCb }; - return sdbUpdateRow(&wmsg); + return sdbUpdateRow(&row); } static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) { @@ -2218,12 +2218,12 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { if (pTable == NULL) break; if (pTable->vgId == pVgroup->vgId) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfTables++; } mnodeDecTableRef(pTable); @@ -2251,12 +2251,12 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfTables++; } mnodeDecTableRef(pTable); @@ -2280,12 +2280,12 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { if (pTable == NULL) break; if (pTable->superTable == pStable) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsChildTableSdb, - .pRow = pTable, + .pObj = pTable, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfTables++; } @@ -2410,9 +2410,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { } if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - SSWriteMsg desc = { + SSdbRow desc = { .type = SDB_OPER_GLOBAL, - .pRow = pTable, + .pObj = pTable, .pTable = tsChildTableSdb, .pMsg = mnodeMsg, .fpRsp = mnodeDoCreateChildTableCb @@ -2440,8 +2440,8 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); - SSWriteMsg wmsg = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pRow = pTable}; - sdbDeleteRow(&wmsg); + SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pObj = pTable}; + sdbDeleteRow(&row); if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { //Avoid retry again in client diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 9de14dbfd1..dc76d92eb8 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -42,13 +42,13 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); -static int32_t mnodeUserActionDestroy(SSWriteMsg *pWMsg) { - tfree(pWMsg->pRow); +static int32_t mnodeUserActionDestroy(SSdbRow *pRow) { + tfree(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionInsert(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pRow; +static int32_t mnodeUserActionInsert(SSdbRow *pRow) { + SUserObj *pUser = pRow->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -62,8 +62,8 @@ static int32_t mnodeUserActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDelete(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pRow; +static int32_t mnodeUserActionDelete(SSdbRow *pRow) { + SUserObj *pUser = pRow->pObj; SAcctObj *pAcct = mnodeGetAcct(pUser->acct); if (pAcct != NULL) { @@ -74,8 +74,8 @@ static int32_t mnodeUserActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionUpdate(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pRow; +static int32_t mnodeUserActionUpdate(SSdbRow *pRow) { + SUserObj *pUser = pRow->pObj; SUserObj *pSaved = mnodeGetUser(pUser->user); if (pUser != pSaved) { memcpy(pSaved, pUser, tsUserUpdateSize); @@ -85,19 +85,19 @@ static int32_t mnodeUserActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionEncode(SSWriteMsg *pWMsg) { - SUserObj *pUser = pWMsg->pRow; - memcpy(pWMsg->rowData, pUser, tsUserUpdateSize); - pWMsg->rowSize = tsUserUpdateSize; +static int32_t mnodeUserActionEncode(SSdbRow *pRow) { + SUserObj *pUser = pRow->pObj; + memcpy(pRow->rowData, pUser, tsUserUpdateSize); + pRow->rowSize = tsUserUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeUserActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeUserActionDecode(SSdbRow *pRow) { SUserObj *pUser = (SUserObj *)calloc(1, sizeof(SUserObj)); if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pUser, pWMsg->rowData, tsUserUpdateSize); - pWMsg->pRow = pUser; + memcpy(pUser, pRow->rowData, tsUserUpdateSize); + pRow->pObj = pUser; return TSDB_CODE_SUCCESS; } @@ -205,14 +205,14 @@ void mnodeDecUserRef(SUserObj *pUser) { } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pRow = pUser, + .pObj = pUser, .pMsg = pMsg }; - int32_t code = sdbUpdateRow(&wmsg); + int32_t code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); } else { @@ -259,15 +259,15 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { pUser->superAuth = 1; } - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pRow = pUser, + .pObj = pUser, .rowSize = sizeof(SUserObj), .pMsg = pMsg }; - code = sdbInsertRow(&wmsg); + code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); tfree(pUser); @@ -279,14 +279,14 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { } static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsUserSdb, - .pRow = pUser, + .pObj = pUser, .pMsg = pMsg }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); } else { @@ -562,12 +562,12 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { if (pUser == NULL) break; if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsUserSdb, - .pRow = pUser, + .pObj = pUser, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfUsers++; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 2eb11e1def..16e5a601e3 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -72,13 +72,13 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { tfree(pVgroup); } -static int32_t mnodeVgroupActionDestroy(SSWriteMsg *pWMsg) { - mnodeDestroyVgroup(pWMsg->pRow); +static int32_t mnodeVgroupActionDestroy(SSdbRow *pRow) { + mnodeDestroyVgroup(pRow->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionInsert(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pRow; +static int32_t mnodeVgroupActionInsert(SSdbRow *pRow) { + SVgObj *pVgroup = pRow->pObj; // refer to db SDbObj *pDb = mnodeGetDb(pVgroup->dbName); @@ -115,8 +115,8 @@ static int32_t mnodeVgroupActionInsert(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDelete(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pRow; +static int32_t mnodeVgroupActionDelete(SSdbRow *pRow) { + SVgObj *pVgroup = pRow->pObj; if (pVgroup->pDb == NULL) { mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName); @@ -137,8 +137,8 @@ static int32_t mnodeVgroupActionDelete(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pWMsg) { - SVgObj *pNew = pWMsg->pRow; +static int32_t mnodeVgroupActionUpdate(SSdbRow *pRow) { + SVgObj *pNew = pRow->pObj; SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId); if (pVgroup != pNew) { @@ -176,25 +176,25 @@ static int32_t mnodeVgroupActionUpdate(SSWriteMsg *pWMsg) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionEncode(SSWriteMsg *pWMsg) { - SVgObj *pVgroup = pWMsg->pRow; - memcpy(pWMsg->rowData, pVgroup, tsVgUpdateSize); - SVgObj *pTmpVgroup = pWMsg->rowData; +static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) { + SVgObj *pVgroup = pRow->pObj; + memcpy(pRow->rowData, pVgroup, tsVgUpdateSize); + SVgObj *pTmpVgroup = pRow->rowData; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; } - pWMsg->rowSize = tsVgUpdateSize; + pRow->rowSize = tsVgUpdateSize; return TSDB_CODE_SUCCESS; } -static int32_t mnodeVgroupActionDecode(SSWriteMsg *pWMsg) { +static int32_t mnodeVgroupActionDecode(SSdbRow *pRow) { SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); if (pVgroup == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - memcpy(pVgroup, pWMsg->rowData, tsVgUpdateSize); - pWMsg->pRow = pVgroup; + memcpy(pVgroup, pRow->rowData, tsVgUpdateSize); + pRow->pObj = pVgroup; return TSDB_CODE_SUCCESS; } @@ -253,13 +253,13 @@ SVgObj *mnodeGetVgroup(int32_t vgId) { } void mnodeUpdateVgroup(SVgObj *pVgroup) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup + .pObj = pVgroup }; - int32_t code = sdbUpdateRow(&wmsg); + int32_t code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("vgId:%d, failed to update vgroup", pVgroup->vgId); } @@ -519,14 +519,14 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, tstrerror(code)); - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; + SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); return code; } else { mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); pVgroup->status = TAOS_VG_STATUS_READY; - SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; + SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; (void)sdbUpdateRow(&desc); dnodeReprocessMWriteMsg(pMsg); @@ -535,7 +535,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, // pDb->name, pVgroup->numOfVnodes); // pVgroup->status = TAOS_VG_STATUS_READY; - // SSWriteMsg desc = {.type = SDB_OPER_GLOBAL, .pRow = pVgroup, .pTable = tsVgroupSdb}; + // SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -571,16 +571,16 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { pMsg->pVgroup = pVgroup; mnodeIncVgroupRef(pVgroup); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup, + .pObj = pVgroup, .rowSize = sizeof(SVgObj), .pMsg = pMsg, .fpReq = mnodeCreateVgroupFp }; - code = sdbInsertRow(&wmsg); + code = sdbInsertRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); @@ -595,12 +595,12 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mDebug("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mnodeSendDropVgroupMsg(pVgroup, NULL); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup + .pObj = pVgroup }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); } } @@ -957,28 +957,28 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; if (mnodeMsg->received == mnodeMsg->successed) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup, + .pObj = pVgroup, .rowSize = sizeof(SVgObj), .pMsg = mnodeMsg, .fpRsp = mnodeCreateVgroupCb }; - int32_t code = sdbInsertRowToQueue(&wmsg); + int32_t code = sdbInsertRowToQueue(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup + .pObj = pVgroup }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1031,12 +1031,12 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->received != mnodeMsg->expected) return; - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, - .pRow = pVgroup + .pObj = pVgroup }; - int32_t code = sdbDeleteRow(&wmsg); + int32_t code = sdbDeleteRow(&row); if (code != 0) { code = TSDB_CODE_MND_SDB_ERROR; } @@ -1084,12 +1084,12 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) { mnodeDropAllChildTablesInVgroups(pVgroup); - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsVgroupSdb, - .pRow = pVgroup, + .pObj = pVgroup, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfVgroups++; } mnodeDecVgroupRef(pVgroup); @@ -1135,12 +1135,12 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { if (pVgroup == NULL) break; if (pVgroup->pDb == pDropDb) { - SSWriteMsg wmsg = { + SSdbRow row = { .type = SDB_OPER_LOCAL, .pTable = tsVgroupSdb, - .pRow = pVgroup, + .pObj = pVgroup, }; - sdbDeleteRow(&wmsg); + sdbDeleteRow(&row); numOfVgroups++; } From 007af6805718413134dc358d801b6dcf61a67009 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 14:35:27 +0800 Subject: [PATCH 18/75] TD-2046 --- src/mnode/src/mnodeSdb.c | 10 ++++++---- tests/script/sh/deploy.sh | 34 +++++++++++++++++----------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index a79eec16aa..bfa106fa75 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -109,6 +109,7 @@ static SSdbWorkerPool tsSdbPool; static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); static int32_t sdbWriteWalToQueue(void *vparam, void *pHead, int32_t qtype, void *rparam); static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); +static void sdbFreeFromQueue(SSdbRow *pRow); static void * sdbWorkerFp(void *pWorker); static int32_t sdbInitWorker(); static void sdbCleanupWorker(); @@ -284,6 +285,7 @@ static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { } dnodeSendRpcMWriteRsp(pMsg, pRow->code); + sdbFreeFromQueue(pRow); } static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); } @@ -1035,12 +1037,12 @@ static void *sdbWorkerFp(void *pWorker) { if (qtype == TAOS_QTYPE_RPC) { sdbConfirmForward(NULL, pRow, pRow->code); - } else if (qtype == TAOS_QTYPE_FWD) { - syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); } else { + if (qtype == TAOS_QTYPE_FWD) { + syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); + } + sdbFreeFromQueue(pRow); } - - sdbFreeFromQueue(pRow); } } diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 8fccb1442f..e26778e86b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -111,24 +111,24 @@ echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 135" >> $TAOS_CFG -echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "dDebugFlag 135" >> $TAOS_CFG -echo "vDebugFlag 135" >> $TAOS_CFG -echo "tsdbDebugFlag 135" >> $TAOS_CFG -echo "cDebugFlag 135" >> $TAOS_CFG -echo "jnidebugFlag 135" >> $TAOS_CFG -echo "odbcdebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 135" >> $TAOS_CFG -echo "monitorDebugFlag 135" >> $TAOS_CFG -echo "mqttDebugFlag 135" >> $TAOS_CFG -echo "qdebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "mDebugFlag 143" >> $TAOS_CFG +echo "sdbDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 143" >> $TAOS_CFG +echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "cDebugFlag 143" >> $TAOS_CFG +echo "jnidebugFlag 143" >> $TAOS_CFG +echo "odbcdebugFlag 143" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG +echo "monitorDebugFlag 143" >> $TAOS_CFG +echo "mqttDebugFlag 143" >> $TAOS_CFG +echo "qdebugFlag 143" >> $TAOS_CFG +echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 135" >> $TAOS_CFG -echo "sdebugFlag 135" >> $TAOS_CFG -echo "wdebugFlag 135" >> $TAOS_CFG -echo "cqdebugFlag 135" >> $TAOS_CFG +echo "udebugFlag 143" >> $TAOS_CFG +echo "sdebugFlag 143" >> $TAOS_CFG +echo "wdebugFlag 143" >> $TAOS_CFG +echo "cqdebugFlag 143" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG From f575d942a214a99029b06c26692e86b103ad61c1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 15:12:56 +0800 Subject: [PATCH 19/75] change log format --- src/dnode/src/dnodeMRead.c | 3 +- src/dnode/src/dnodeMWrite.c | 10 +- src/dnode/src/dnodeMgmt.c | 2 +- src/dnode/src/dnodeVRead.c | 2 +- src/dnode/src/dnodeVWrite.c | 4 +- src/mnode/src/mnodeInt.c | 2 +- src/mnode/src/mnodePeer.c | 14 +- src/mnode/src/mnodeRead.c | 10 +- src/mnode/src/mnodeTable.c | 316 ++++++++++++++++++------------------ src/mnode/src/mnodeVgroup.c | 26 +-- src/mnode/src/mnodeWrite.c | 14 +- 11 files changed, 204 insertions(+), 199 deletions(-) diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index c14c7a8158..ee9da72e4d 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -168,7 +168,8 @@ static void *dnodeProcessMReadQueue(void *param) { break; } - dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]); + dDebug("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, + taosMsg[pRead->rpcMsg.msgType]); int32_t code = mnodeProcessRead(pRead); dnodeSendRpcMReadRsp(pRead, code); } diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index bde4b95bc6..65c0d53819 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -127,7 +127,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { dnodeSendRedirectMsg(pMsg, true); } else { SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, + dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } @@ -136,7 +136,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { } static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { - dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, + dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); mnodeCleanupMsg(pWrite); @@ -174,7 +174,7 @@ static void *dnodeProcessMWriteQueue(void *param) { break; } - dDebug("app:%p:%p, msg:%s will be processed in mwrite queue", pWrite->rpcMsg.ahandle, pWrite, + dDebug("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); int32_t code = mnodeProcessWrite(pWrite); @@ -188,13 +188,13 @@ void dnodeReprocessMWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; if (!mnodeIsRunning() || tsMWriteQueue == NULL) { - dDebug("app:%p:%p, msg:%s is redirected for mnode not running, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, + dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); dnodeSendRedirectMsg(pMsg, true); dnodeFreeMWriteMsg(pWrite); } else { - dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, + dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index dcb48f7833..da1852e05e 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -216,7 +216,7 @@ static void *dnodeProcessMgmtQueue(void *param) { } pMsg = &pMgmt->rpcMsg; - dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]); + dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index e1c1b1585d..4cce54bf59 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -175,7 +175,7 @@ static void *dnodeProcessReadQueue(void *pWorker) { break; } - dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead, + dDebug("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, taosMsg[pRead->msgType], qtype); int32_t code = vnodeProcessRead(pVnode, pRead); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index da1a902fb3..959eb3c9c5 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -205,8 +205,8 @@ static void *dnodeProcessVWriteQueue(void *wparam) { bool forceFsync = false; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); - dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite, - taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); + dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite, + pWrite->rpcAhandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index 8431baf4f4..534deec09a 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -48,7 +48,7 @@ void *mnodeCreateMsg(SRpcMsg *pRpcMsg) { int32_t mnodeInitMsg(SMnodeMsg *pMsg) { if (pMsg->pUser != NULL) { - mDebug("app:%p:%p, user info already inited", pMsg->rpcMsg.ahandle, pMsg); + mTrace("msg:%p, app:%p user info already inited", pMsg, pMsg->rpcMsg.ahandle); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index e43f8c1b78..f13ef75398 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -47,7 +47,7 @@ void mnodeAddPeerRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { if (pMsg->rpcMsg.pCont == NULL) { - mError("%p, msg:%s in mpeer queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, ahandle:%p type:%s in mpeer queue, content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_INVALID_MSG_LEN; } @@ -58,8 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, - taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); + mDebug("msg:%p, ahandle:%p type:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg, + pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; @@ -73,7 +73,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { } if (tsMnodeProcessPeerMsgFp[pMsg->rpcMsg.msgType] == NULL) { - mError("%p, msg:%s in mpeer queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, ahandle:%p type:%s in mpeer queue, not processed", pMsg, pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_MSG_NOT_PROCESSED; } @@ -82,13 +83,14 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { void mnodeProcessPeerRsp(SRpcMsg *pMsg) { if (!sdbIsMaster()) { - mError("%p, msg:%s is not processed for it is not master", pMsg->ahandle, taosMsg[pMsg->msgType]); + mError("msg:%p, ahandle:%p type:%s is not processed for it is not master", pMsg, pMsg->ahandle, + taosMsg[pMsg->msgType]); return; } if (tsMnodeProcessPeerRspFp[pMsg->msgType]) { (*tsMnodeProcessPeerRspFp[pMsg->msgType])(pMsg); } else { - mError("%p, msg:%s is not processed", pMsg->ahandle, taosMsg[pMsg->msgType]); + mError("msg:%p, ahandle:%p type:%s is not processed", pMsg, pMsg->ahandle, taosMsg[pMsg->msgType]); } } diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index 93b944febb..ea7ce783e8 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -43,7 +43,7 @@ void mnodeAddReadMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *pMsg)) { int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (pMsg->rpcMsg.pCont == NULL) { - mError("%p, msg:%s in mread queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, app:%p type:%s in mread queue, content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_INVALID_MSG_LEN; } @@ -52,7 +52,7 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); mnodeGetMnodeEpSetForShell(epSet); - mDebug("%p, msg:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, + mDebug("msg:%p, app:%p type:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { @@ -70,13 +70,15 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { } if (tsMnodeProcessReadMsgFp[pMsg->rpcMsg.msgType] == NULL) { - mError("%p, msg:%s in mread queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, app:%p type:%s in mread queue, not processed", pMsg, pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_MSG_NOT_PROCESSED; } int32_t code = mnodeInitMsg(pMsg); if (code != TSDB_CODE_SUCCESS) { - mError("%p, msg:%s in mread queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code)); + mError("msg:%p, app:%p type:%s in mread queue, not processed reason:%s", pMsg, pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType], tstrerror(code)); return code; } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 29957f8ec0..4d5f0808f5 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -703,7 +703,7 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db); if (pMsg->pDb == NULL) { - mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -715,24 +715,24 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pCreate->getMeta) { - mDebug("app:%p:%p, table:%s, continue to get meta", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return mnodeGetChildTableMeta(pMsg); } else if (pCreate->igExists) { - mDebug("app:%p:%p, table:%s, is already exist", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_SUCCESS; } else { - mError("app:%p:%p, table:%s, failed to create, table already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } if (pCreate->numOfTags != 0) { - mDebug("app:%p:%p, table:%s, create stable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { - mDebug("app:%p:%p, table:%s, create ctable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } @@ -742,17 +742,18 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId); if (pMsg->pDb == NULL) { - mError("app:%p:%p, table:%s, failed to drop table, db not selected or db in dropping", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mError("msg:%p, app:%p table:%s, failed to drop table, db not selected or db in dropping", pMsg, + pMsg->rpcMsg.ahandle, pDrop->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to drop table, in monitor database", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } @@ -760,22 +761,22 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mDebug("app:%p:%p, table:%s, table is not exist, treat as success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); return TSDB_CODE_SUCCESS; } else { - mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; - mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", - pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); + mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", + pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; - mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mInfo("msg:%p, app:%p table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); } @@ -784,12 +785,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); - mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg, pMsg->rpcMsg.ahandle, pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId); if (pMsg->pDb == NULL) { - mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to get table meta, db not selected", pMsg, pMsg->rpcMsg.ahandle, pInfo->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -802,11 +803,11 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId); if (pMsg->pTable == NULL) { if (!pInfo->createFlag) { - mError("app:%p:%p, table:%s, failed to get table meta, table not exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to get table meta, table not exist", pMsg, pMsg->rpcMsg.ahandle, pInfo->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - mDebug("app:%p:%p, table:%s, failed to get table meta, start auto create table ", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, failed to get table meta, start auto create table ", pMsg, pMsg->rpcMsg.ahandle, pInfo->tableId); return mnodeAutoCreateChildTable(pMsg); } @@ -826,7 +827,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { if (code == TSDB_CODE_SUCCESS) { mLInfo("stable:%s, is created in sdb, uid:%" PRIu64, pTable->info.tableId, pTable->uid); } else { - mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mError("msg:%p, app:%p stable:%s, failed to create in sdb, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsSuperTableSdb}; sdbDeleteRow(&desc); @@ -841,7 +842,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { - mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -860,7 +861,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { free(pStable); - mError("app:%p:%p, table:%s, failed to create, no schema input", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); @@ -891,7 +892,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroySuperTable(pStable); pMsg->pTable = NULL; - mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); } return code; @@ -900,9 +901,9 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); } else { - mLInfo("app:%p:%p, stable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); } return code; @@ -925,8 +926,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { pDrop->uid = htobe64(pStable->uid); mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); - mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, - pVgroup->vgId); + mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, pVgroup->vgId); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&epSet, &rpcMsg); @@ -947,7 +948,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("app:%p:%p, table:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mError("msg:%p, app:%p table:%s, failed to drop, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); } @@ -967,7 +968,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagN static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, add tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; @@ -976,20 +977,20 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { - mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); + mError("msg:%p, app:%p stable:%s, add tag, too many tags", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; } for (int32_t i = 0; i < ntags; i++) { if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - mError("app:%p:%p, stable:%s, add tag, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_TAG_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { - mError("app:%p:%p, stable:%s, add tag, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, - schema[i].name); + mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } } @@ -1007,7 +1008,7 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t pStable->numOfTags += ntags; pStable->tversion++; - mInfo("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mInfo("msg:%p, app:%p stable %s, start to add tag %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[0].name); SSdbRow row = { @@ -1023,7 +1024,7 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, drop tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; } @@ -1032,7 +1033,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { - mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mError("msg:%p, app:%p stable:%s, drop tag, tag:%s not exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tagName); return TSDB_CODE_MND_TAG_NOT_EXIST; } @@ -1042,7 +1043,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { pStable->numOfTags--; pStable->tversion++; - mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName); + mInfo("msg:%p, app:%p stable %s, start to drop tag %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tagName); SSdbRow row = { .type = SDB_OPER_GLOBAL, @@ -1057,7 +1058,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, modify tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; } @@ -1066,7 +1067,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { - mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_MND_TAG_NOT_EXIST; } @@ -1085,7 +1086,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); tstrncpy(schema->name, newTagName, sizeof(schema->name)); - mInfo("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mInfo("msg:%p, app:%p stable %s, start to modify tag %s to %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, oldTagName, newTagName); SSdbRow row = { @@ -1112,7 +1113,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, add column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; } @@ -1121,19 +1122,19 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 SDbObj *pDb = pMsg->pDb; SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (ncols <= 0) { - mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols); + mError("msg:%p, app:%p stable:%s, add column, ncols:%d <= 0", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, ncols); return TSDB_CODE_MND_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - mError("app:%p:%p, stable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p stable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { - mError("app:%p:%p, stable:%s, add column, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p stable:%s, add column, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); return TSDB_CODE_MND_TAG_ALREAY_EXIST; } @@ -1160,7 +1161,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 mnodeDecAcctRef(pAcct); } - mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); + mInfo("msg:%p, app:%p stable %s, start to add column", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId); SSdbRow row = { .type = SDB_OPER_GLOBAL, @@ -1175,7 +1176,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, delete column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; } @@ -1185,7 +1186,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { - mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mError("msg:%p, app:%p stable:%s, drop column, column:%s not exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, colName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -1205,7 +1206,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { mnodeDecAcctRef(pAcct); } - mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); + mInfo("msg:%p, app:%p stable %s, start to delete column", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId); SSdbRow row = { .type = SDB_OPER_GLOBAL, @@ -1220,7 +1221,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mLInfo("msg:%p, app:%p stable %s, change column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tstrerror(code)); return code; } @@ -1229,7 +1230,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName); if (col < 0) { - mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p stable:%s, change column, oldName:%s, newName:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, oldName, newName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -1248,7 +1249,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char SSchema *schema = (SSchema *) (pStable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); - mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + mInfo("msg:%p, app:%p stable %s, start to modify column %s to %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, oldName, newName); SSdbRow row = { @@ -1470,7 +1471,7 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMsg->rpcRsp.rsp = pMeta; - mDebug("app:%p:%p, stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid); return TSDB_CODE_SUCCESS; } @@ -1503,12 +1504,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { - mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); + mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName); mnodeDecTableRef(pTable); continue; } if (pTable->vgHash == NULL) { - mDebug("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p stable:%s, no vgroup exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName); mnodeDecTableRef(pTable); @@ -1650,7 +1651,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); - mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid); SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; @@ -1679,21 +1680,21 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { if (code == TSDB_CODE_SUCCESS) { if (pCreate->getMeta) { - mDebug("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pMsg->rpcMsg.handle); + mDebug("msg:%p, app:%p table:%s, created in dnode and continue to get meta, thandle:%p", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle); pMsg->retry = 0; dnodeReprocessMWriteMsg(pMsg); } else { - mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mDebug("msg:%p, app:%p table:%s, created in dnode, thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle); dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { - mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); + mError("msg:%p, app:%p table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); return code; @@ -1705,7 +1706,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { - mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1724,7 +1725,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { STagData *pTagData = (STagData *)pCreate->schema; // it is a tag key if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name); if (pMsg->pSTable == NULL) { - mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pTagData->name); mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_NAME; @@ -1772,7 +1773,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mDebug("msg:%p, app:%p table:%s, stream sql len:%d sql:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->sqlLen, pTable->sql); } } @@ -1792,10 +1793,10 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroyChildTable(pTable); pMsg->pTable = NULL; - mError("app:%p:%p, table:%s, failed to create, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, + mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, tstrerror(code)); } else { - mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pVgroup->vgId, pTable->tid, pTable->uid); } @@ -1806,7 +1807,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, failed to create, grant timeseries failed", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to create, grant timeseries failed", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return code; } @@ -1817,7 +1818,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { int32_t tid = 0; code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid); if (code != TSDB_CODE_SUCCESS) { - mDebug("app:%p:%p, table:%s, failed to get available vgroup, reason:%s", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, tstrerror(code)); return code; } @@ -1836,22 +1837,22 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } if (pMsg->pTable == NULL) { - mError("app:%p:%p, table:%s, object not found, retry:%d reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, pMsg->retry, + mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->retry, tstrerror(terrno)); return terrno; } else { - mDebug("app:%p:%p, table:%s, send create msg to vnode again", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); + mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return mnodeDoCreateChildTableFp(pMsg); } } static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; - mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mLInfo("msg:%p, app:%p ctable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("app:%p:%p, ctable:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p ctable:%s, failed to drop ctable, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1864,7 +1865,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); - mInfo("app:%p:%p, ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mInfo("msg:%p, app:%p ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pTable->vgId, pTable->tid, pTable->uid); SRpcMsg rpcMsg = { @@ -1885,7 +1886,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; - mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mError("msg:%p, app:%p ctable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return code; } @@ -1896,7 +1897,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { - mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to drop ctable, vgroup not exist", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return TSDB_CODE_MND_APP_ERROR; } @@ -1911,7 +1912,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { int32_t code = sdbDeleteRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("app:%p:%p, ctable:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mError("msg:%p, app:%p ctable:%s, failed to drop, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, tstrerror(code)); } @@ -1932,7 +1933,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mError("msg:%p, app:%p ctable %s, failed to alter column, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, tstrerror(code)); return code; } @@ -1946,7 +1947,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { rpcFreeCont(pMDCreate); - mError("app:%p:%p, ctable %s, vgId:%d not exist in mnode", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mError("msg:%p, app:%p ctable %s, vgId:%d not exist in mnode", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -1961,7 +1962,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { .msgType = TSDB_MSG_TYPE_MD_ALTER_TABLE }; - mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mDebug("msg:%p, app:%p ctable %s, send alter column msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->pVgroup->vgId); dnodeSendMsgToDnode(&epSet, &rpcMsg); @@ -1972,13 +1973,13 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 SCTableObj *pTable = (SCTableObj *)pMsg->pTable; SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { - mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols); + mError("msg:%p, app:%p ctable:%s, add column, ncols:%d <= 0", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, ncols); return TSDB_CODE_MND_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - mError("app:%p:%p, ctable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p ctable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, schema[i].name); return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } @@ -2003,7 +2004,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 mnodeDecAcctRef(pAcct); } - mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mInfo("msg:%p, app:%p ctable %s, start to add column", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); SSdbRow row = { .type = SDB_OPER_GLOBAL, @@ -2021,8 +2022,8 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { - mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, - colName); + mError("msg:%p, app:%p ctable:%s, drop column, column:%s not exist", pMsg, pMsg->rpcMsg.ahandle, + pTable->info.tableId, colName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -2036,7 +2037,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { mnodeDecAcctRef(pAcct); } - mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName); + mInfo("msg:%p, app:%p ctable %s, start to drop column %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, colName); SSdbRow row = { .type = SDB_OPER_GLOBAL, @@ -2053,7 +2054,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName); if (col < 0) { - mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p ctable:%s, change column, oldName: %s, newName: %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, oldName, newName); return TSDB_CODE_MND_FIELD_NOT_EXIST; } @@ -2072,7 +2073,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char SSchema *schema = (SSchema *) (pTable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); - mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mInfo("msg:%p, app:%p ctable %s, start to modify column %s to %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, oldName, newName); SSdbRow row = { @@ -2128,7 +2129,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { - mError("app:%p:%p, table:%s, failed to get table meta, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to get table meta, vgroup not exist", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -2143,7 +2144,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { } pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); - mDebug("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid, pTable->vgId, pTable->tid); return TSDB_CODE_SUCCESS; @@ -2154,16 +2155,16 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { STagData *pTags = (STagData *)pInfo->tags; int32_t tagLen = htonl(pTags->dataLen); if (pTags->name[0] == 0) { - mError("app:%p:%p, table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg->rpcMsg.ahandle, - pMsg, pInfo->tableId, tagLen); - return TSDB_CODE_MND_INVALID_STABLE_NAME; + mError("msg:%p, app:%p table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg, + pMsg->rpcMsg.ahandle, pInfo->tableId, tagLen); + return TSDB_CODE_MND_INVALID_STABLE_NAME; } int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { - mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle, - pMsg, pInfo->tableId); + mError("msg:%p, app:%p table:%s, failed to create table while get meta info, no enough memory", pMsg, + pMsg->rpcMsg.ahandle, pInfo->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -2175,8 +2176,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { pCreateMsg->contLen = htonl(contLen); memcpy(pCreateMsg->schema, pTags, contLen - sizeof(SCMCreateTableMsg)); - mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s", - pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name); + mDebug("msg:%p, app:%p table:%s, start to create on demand, tagLen:%d stable:%s", pMsg, pMsg->rpcMsg.ahandle, + pInfo->tableId, tagLen, pTags->name); if (pMsg->rpcMsg.pCont != pMsg->pCont) { tfree(pMsg->rpcMsg.pCont); @@ -2192,7 +2193,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) { STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); if (pMeta == NULL) { - mError("app:%p:%p, table:%s, failed to get table meta, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p table:%s, failed to get table meta, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pMsg->pTable->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -2317,12 +2318,12 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); pCfg->sid = htonl(pCfg->sid); - mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, + mDebug("msg:%p, app:%p dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg, pMsg->rpcMsg.ahandle, pCfg->dnodeId, pCfg->vgId, pCfg->sid); SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); if (pTable == NULL) { - mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, + mError("msg:%p, app:%p dnode:%d, vgId:%d sid:%d, table not found", pMsg, pMsg->rpcMsg.ahandle, pCfg->dnodeId, pCfg->vgId, pCfg->sid); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -2343,38 +2344,37 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->ahandle; - mnodeMsg->received++; + SMnodeMsg *pMsg = rpcMsg->ahandle; + pMsg->received++; - SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); - mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, - mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + mInfo("msg:%p, app:%p table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, pMsg->rpcMsg.handle, + tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, - tstrerror(rpcMsg->code)); - dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); + mError("msg:%p, app:%p table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code)); + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); return; } - if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId); - if (mnodeMsg->pVgroup == NULL) { - mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); - dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); + if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); + if (pMsg->pVgroup == NULL) { + mError("msg:%p, app:%p table:%s, failed to get vgroup", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); return; } - if (mnodeMsg->pVgroup->numOfTables <= 0) { - mInfo("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - mnodeMsg->pVgroup->vgId); - mnodeDropVgroup(mnodeMsg->pVgroup, NULL); + if (pMsg->pVgroup->numOfTables <= 0) { + mInfo("msg:%p, app:%p vgId:%d, all tables is dropped, drop vgroup", pMsg, pMsg->rpcMsg.ahandle, + pMsg->pVgroup->vgId); + mnodeDropVgroup(pMsg->pVgroup, NULL); } - dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } /* @@ -2384,28 +2384,28 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->ahandle; - mnodeMsg->received++; + SMnodeMsg *pMsg = rpcMsg->ahandle; + pMsg->received++; - SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) { - mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid); + mDebug("msg:%p, app:%p table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, + pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid); // if the vgroup is already dropped from hash, it can't be accquired by pTable->vgId // so the refCount of vgroup can not be decreased // SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); // if (pVgroup == NULL) { - // mnodeRemoveTableFromVgroup(mnodeMsg->pVgroup, pTable); + // mnodeRemoveTableFromVgroup(pMsg->pVgroup, pTable); // } // mnodeDecVgroupRef(pVgroup); - mnodeSendDropChildTableMsg(mnodeMsg, false); + mnodeSendDropChildTableMsg(pMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; - dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); return; } @@ -2414,31 +2414,31 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { .type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb, - .pMsg = mnodeMsg, + .pMsg = pMsg, .fpRsp = mnodeDoCreateChildTableCb }; int32_t code = sdbInsertRowToQueue(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mnodeMsg->pTable = NULL; + pMsg->pTable = NULL; mnodeDestroyChildTable(pTable); - dnodeSendRpcMWriteRsp(mnodeMsg, code); + dnodeSendRpcMWriteRsp(pMsg, code); } } else { - mnodeMsg->retry++; + pMsg->retry++; int32_t sec = taosGetTimestampSec(); - if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) { - mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64 + if (pMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - pMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) { + mDebug("msg:%p, app:%p table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64 " result:%s thandle:%p", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid, - pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); + pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->retry, pTable->vgId, pTable->tid, pTable->uid, + tstrerror(rpcMsg->code), pMsg->rpcMsg.handle); - dnodeDelayReprocessMWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(pMsg); } else { - mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 + mError("msg:%p, app:%p table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, - tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); + pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, + tstrerror(rpcMsg->code), pMsg->rpcMsg.handle, pMsg->incomingTs, sec, pMsg->retry); SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&row); @@ -2447,7 +2447,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { //Avoid retry again in client rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; } - dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); } } } @@ -2455,28 +2455,28 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->ahandle; - mnodeMsg->received++; + SMnodeMsg *pMsg = rpcMsg->ahandle; + pMsg->received++; - SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); + mDebug("msg:%p, app:%p ctable:%s, altered in dnode, thandle:%p result:%s", pMsg, pMsg->rpcMsg.ahandle, + pTable->info.tableId, pMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); - dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } else { - if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) { - mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), - mnodeMsg->rpcMsg.handle); + if (pMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) { + mDebug("msg:%p, app:%p table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p", + pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->retry, tstrerror(rpcMsg->code), + pMsg->rpcMsg.handle); - dnodeDelayReprocessMWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(pMsg); } else { - mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, - pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); + mError("msg:%p, app:%p table:%s, failed to alter in dnode, result:%s thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pTable->info.tableId, tstrerror(rpcMsg->code), pMsg->rpcMsg.handle); + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); } } } @@ -2719,12 +2719,12 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; - mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p table:%s, alter table msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId); if (pMsg->pDb == NULL) { - mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -2734,13 +2734,13 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { } if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId); if (pMsg->pTable == NULL) { - mError("app:%p:%p, table:%s, failed to alter table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -2749,7 +2749,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { pAlter->tagValLen = htonl(pAlter->tagValLen); if (pAlter->numOfCols > 2) { - mError("app:%p:%p, table:%s, error numOfCols:%d in alter table", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId, + mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId, pAlter->numOfCols); return TSDB_CODE_MND_APP_ERROR; } @@ -2760,7 +2760,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT; if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - mDebug("app:%p:%p, table:%s, start to alter stable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { @@ -2776,7 +2776,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { } else { } } else { - mDebug("app:%p:%p, table:%s, start to alter ctable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 16e5a601e3..f9a49e5ec2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -421,7 +421,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi int32_t sid = taosAllocateId(pVgroup->idPool); if (sid <= 0) { - mDebug("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId); + mDebug("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, pVgroup->vgId); continue; } @@ -442,8 +442,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES; if (pDb->numOfVgroups < maxVgroupsPerDb) { - mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, - pMsg, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); + mDebug("msg:%p, app:%p db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg, + pMsg->rpcMsg.ahandle, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); pthread_mutex_unlock(&pDb->mutex); code = mnodeCreateVgroup(pMsg); if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { @@ -455,8 +455,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi if (pDb->numOfVgroups < 1) { pthread_mutex_unlock(&pDb->mutex); - mDebug("app:%p:%p, db:%s, failed create new vgroup since:%s, numOfVgroups:%d maxVgroupsPerDb:%d ", - pMsg->rpcMsg.ahandle, pMsg, pDb->name, tstrerror(code), pDb->numOfVgroups, maxVgroupsPerDb); + mDebug("msg:%p, app:%p db:%s, failed create new vgroup since:%s, numOfVgroups:%d maxVgroupsPerDb:%d ", pMsg, + pMsg->rpcMsg.ahandle, pDb->name, tstrerror(code), pDb->numOfVgroups, maxVgroupsPerDb); return code; } @@ -474,7 +474,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi int32_t sid = taosAllocateId(pVgroup->idPool); if (sid <= 0) { - mError("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId); + mError("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, pVgroup->vgId); pthread_mutex_unlock(&pDb->mutex); return TSDB_CODE_MND_NO_ENOUGH_DNODES; } @@ -496,10 +496,10 @@ static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) { SDbObj *pDb = pMsg->pDb; assert(pVgroup); - mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, + mInfo("msg:%p, app:%p vgId:%d, is created in mnode, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mInfo("app:%p:%p, vgId:%d, index:%d, dnode:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, i, + mInfo("msg:%p, app:%p vgId:%d, index:%d, dnode:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId); } @@ -517,14 +517,14 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { assert(pVgroup); if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, + mError("msg:%p, app:%p vgId:%d, failed to create in sdb, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); return code; } else { - mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, - pDb->name, pVgroup->numOfVnodes); + mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, + pDb->name, pVgroup->numOfVnodes); pVgroup->status = TAOS_VG_STATUS_READY; SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; (void)sdbUpdateRow(&desc); @@ -532,7 +532,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { dnodeReprocessMWriteMsg(pMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; // if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) { - // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, + // mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, // pDb->name, pVgroup->numOfVnodes); // pVgroup->status = TAOS_VG_STATUS_READY; // SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; @@ -540,7 +540,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; // } else { - // mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle, + // mError("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle, // pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); // return TSDB_CODE_MND_VGROUP_NOT_EXIST; // } diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index d021745d2b..8893316ffc 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -43,7 +43,7 @@ void mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg)) int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (pMsg->rpcMsg.pCont == NULL) { - mError("app:%p:%p, msg:%s content is null", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, app:%p type:%s content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_INVALID_MSG_LEN; } @@ -54,15 +54,15 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); - mDebug("app:%p:%p, msg:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, pMsg, + mDebug("msg:%p, app:%p type:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { epSet->inUse = (i + 1) % epSet->numOfEps; - mDebug("app:%p:%p, mnode index:%d ep:%s:%d, set inUse to %d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i], + mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d, set inUse to %d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i], + mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i], htons(epSet->port[i])); } } @@ -71,19 +71,19 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { } if (tsMnodeProcessWriteMsgFp[pMsg->rpcMsg.msgType] == NULL) { - mError("app:%p:%p, msg:%s not processed", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]); + mError("msg:%p, app:%p type:%s not processed", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_MSG_NOT_PROCESSED; } int32_t code = mnodeInitMsg(pMsg); if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, msg:%s not processed, reason:%s", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], + mError("msg:%p, app:%p type:%s not processed, reason:%s", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code)); return code; } if (!pMsg->pUser->writeAuth) { - mError("app:%p:%p, msg:%s not processed, no write auth", pMsg->rpcMsg.ahandle, pMsg, + mError("msg:%p, app:%p type:%s not processed, no write auth", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]); return TSDB_CODE_MND_NO_RIGHTS; } From 875cd69000ad898c69b0c7ee0d073ef733f535da Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 15:39:00 +0800 Subject: [PATCH 20/75] TD-2046 --- src/mnode/src/mnodeSdb.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index bfa106fa75..cc1af11a06 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -953,7 +953,7 @@ static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) { sdbIncRef(pRow->pTable, pRow->pObj); - sdbTrace("vgId:1, msg:%p write into to sdb queue", pRow->pMsg); + sdbTrace("vgId:1, msg:%p qtype:%s write into to sdb queue, queued:%d", pRow->pMsg, qtypeStr[qtype], queued); taosWriteQitem(tsSdbWQueue, qtype, pRow); return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -961,7 +961,7 @@ static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) { static void sdbFreeFromQueue(SSdbRow *pRow) { int32_t queued = atomic_sub_fetch_32(&tsSdbMgmt.queuedMsg, 1); - sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pRow->pMsg, queued); + sdbTrace("vgId:1, msg:%p qtype:%s free from sdb queue, queued:%d", pRow->pMsg, qtypeStr[qtype], queued); sdbDecRef(pRow->pTable, pRow->pObj); taosFreeQitem(pRow); @@ -976,6 +976,9 @@ static int32_t sdbWriteWalToQueue(void *vparam, void *wparam, int32_t qtype, voi return TSDB_CODE_VND_OUT_OF_MEMORY; } + memcpy(pRow->pHead, pHead, sizeof(SWalHead) + pHead->len); + pRow->rowData = pRow->pHead->cont; + return sdbWriteToQueue(pRow, qtype); } From 7749987f530cd49dbbbd7edacc325520d6b83c52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 15:40:10 +0800 Subject: [PATCH 21/75] compile error --- src/mnode/src/mnodeSdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index cc1af11a06..1728bb7e3b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -961,7 +961,7 @@ static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) { static void sdbFreeFromQueue(SSdbRow *pRow) { int32_t queued = atomic_sub_fetch_32(&tsSdbMgmt.queuedMsg, 1); - sdbTrace("vgId:1, msg:%p qtype:%s free from sdb queue, queued:%d", pRow->pMsg, qtypeStr[qtype], queued); + sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pRow->pMsg, queued); sdbDecRef(pRow->pTable, pRow->pObj); taosFreeQitem(pRow); From 66c6e5ba05a109e9328e08dd096276b9f5bd7b5e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 16:51:16 +0800 Subject: [PATCH 22/75] change sync log --- src/sync/src/syncMain.c | 15 +++++++-------- src/sync/src/syncRestore.c | 5 +++-- src/sync/src/syncRetrieve.c | 6 ++---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 2ca77fa404..4281d85386 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -//#include -//#include +#define _DEFAULT_SOURCE #include "os.h" #include "hash.h" #include "tlog.h" @@ -392,7 +391,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen); if (retLen == msgLen) { - sDebug("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); + sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); } else { sDebug("%s, failed to send forward ack, restart", pPeer->id); syncRestartConnection(pPeer); @@ -891,7 +890,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo * pFwdInfo; - sDebug("%s, forward-rsp is received, code:%x hver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version); + sTrace("%s, forward-rsp is received, code:%x hver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version); SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { @@ -910,7 +909,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)cont; - sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); + sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; @@ -1191,7 +1190,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { pFwdInfo->time = time; pSyncFwds->fwds++; - sDebug("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); + sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); } static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { @@ -1228,7 +1227,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } if (confirm && pFwdInfo->confirmed == 0) { - sDebug("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); + sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } @@ -1335,7 +1334,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sDebug("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); } else { sError("%s, failed to forward, hver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); syncRestartConnection(pPeer); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index e7901e6eb8..ebf1e91d90 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tlog.h" #include "tutil.h" @@ -154,7 +155,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); if (ret < 0) break; - sDebug("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); + sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); if (lastVer == pHead->version) { sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer); @@ -222,7 +223,7 @@ int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { memcpy(pRecv->offset, pHead, len); pRecv->offset += len; pRecv->forwards++; - sDebug("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards); + sTrace("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards); } else { sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize); pRecv->code = -1; // set error code diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 03f5a7bd94..c02bcbe11d 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include -#include +#define _DEFAULT_SOURCE #include -#include #include "os.h" #include "tlog.h" #include "tutil.h" @@ -268,7 +266,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi break; } - sDebug("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); + sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) break; pPeer->sversion = pHead->version; From 2b887c4140cf1482aa0431ccccd6b835005e4fe5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 Nov 2020 17:48:07 +0800 Subject: [PATCH 23/75] TD-2000 --- src/sync/src/syncMain.c | 13 +++++++------ src/sync/src/syncRestore.c | 5 +++-- src/sync/src/syncRetrieve.c | 18 +++++++++--------- src/sync/src/tarbitrator.c | 8 ++++---- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 4281d85386..843de9461f 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -820,7 +820,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { pthread_attr_destroy(&thattr); if (ret != 0) { - sError("%s, failed to create sync thread(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno)); syncDecPeerRef(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; @@ -1105,7 +1105,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { SFirstPkt firstPkt; if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno)); + sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); taosCloseSocket(connFd); return; } @@ -1159,7 +1159,7 @@ static void syncProcessBrokenLink(void *param) { if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&(pNode->mutex)); - sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); + sDebug("%s, TCP link is broken since %s", pPeer->id, strerror(errno)); pPeer->peerFd = -1; if (syncDecPeerRef(pPeer) != 0) { @@ -1242,9 +1242,10 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { if (index == pNode->selfIndex) continue; SSyncPeer *pPeer = pNode->peerInfo[index]; - if (pPeer->role <= TAOS_SYNC_ROLE_UNSYNCED || nodeRole <= TAOS_SYNC_ROLE_UNSYNCED) { - syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); - } + if (pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; + if (pPeer->sstatus > TAOS_SYNC_STATUS_INIT || nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; + + syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ebf1e91d90..cc2315fb15 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" #include "tlog.h" #include "tutil.h" #include "ttimer.h" @@ -127,7 +128,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { } if (code < 0) { - sError("%s, failed to restore %s(%s)", pPeer->id, name, strerror(errno)); + sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno)); } return code; @@ -167,7 +168,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } if (code < 0) { - sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno)); } free(buffer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c02bcbe11d..82f40854e8 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -32,7 +32,7 @@ static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { - sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno)); return -1; } @@ -49,14 +49,14 @@ static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { if (*wd >= 0) { if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) { - sError("%s, failed to remove wd:%d(%s)", pPeer->id, *wd, strerror(errno)); + sError("%s, failed to remove wd:%d since %s", pPeer->id, *wd, strerror(errno)); return -1; } } *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE); if (*wd == -1) { - sError("%s, failed to add %s(%s)", pPeer->id, name, strerror(errno)); + sError("%s, failed to add %s since %s", pPeer->id, name, strerror(errno)); return -1; } else { sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum); @@ -73,7 +73,7 @@ static int32_t syncAreFilesModified(SSyncPeer *pPeer) { char buf[2048]; int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno)); return -1; } @@ -159,7 +159,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { } if (code < 0) { - sError("%s, failed to retrieve file(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to retrieve file since %s", pPeer->id, strerror(errno)); } return code; @@ -199,7 +199,7 @@ static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) { taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { - sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno)); return -1; } @@ -214,7 +214,7 @@ static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) { *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE); if (*wd == -1) { - sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to watch last wal since %s", pPeer->id, strerror(errno)); return -1; } @@ -225,7 +225,7 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { char buf[2048]; int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno)); return -1; } @@ -422,7 +422,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { memset(&walHead, 0, sizeof(walHead)); code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); } else { - sError("%s, failed to send wal(%s)", pPeer->id, strerror(errno)); + sError("%s, failed to send wal since %s", pPeer->id, strerror(errno)); } return code; diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index b7f819a3cd..4016042de2 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -115,14 +115,14 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { SFirstPkt firstPkt; if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno)); + sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); taosCloseSocket(connFd); return; } - SNodeConn *pNode = (SNodeConn *)calloc(sizeof(SNodeConn), 1); + SNodeConn *pNode = calloc(sizeof(SNodeConn), 1); if (pNode == NULL) { - sError("failed to allocate memory(%s)", strerror(errno)); + sError("failed to allocate memory since %s", strerror(errno)); taosCloseSocket(connFd); return; } @@ -146,7 +146,7 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { static void arbProcessBrokenLink(void *param) { SNodeConn *pNode = param; - sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno)); + sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); tfree(pNode); } From 30e18f31b7217aff476b01a6527718211026e633 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 20 Nov 2020 20:08:41 +0800 Subject: [PATCH 24/75] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 741f76da43..2f0ec81aea 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.7.0") + SET(TD_VER_NUMBER "2.0.8.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1738ff7ec8..059c0650c2 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.7.0' +version: '2.0.8.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.6.0 + - usr/lib/libtaos.so.2.0.8.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 3e43a6533189e060a5e7f848959fd0bda55685da Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 Nov 2020 20:45:06 +0800 Subject: [PATCH 25/75] TD-1983 change hash in mnode --- src/balance/src/balance.c | 40 ++++++++++++------------------ src/mnode/inc/mnodeAcct.h | 1 + src/mnode/inc/mnodeDb.h | 1 + src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 6 ++--- src/mnode/inc/mnodeShow.h | 2 ++ src/mnode/inc/mnodeTable.h | 2 ++ src/mnode/inc/mnodeUser.h | 1 + src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 6 +++-- src/mnode/src/mnodeCluster.c | 8 +++++- src/mnode/src/mnodeDb.c | 12 +++++---- src/mnode/src/mnodeDnode.c | 23 +++++++++--------- src/mnode/src/mnodeMnode.c | 9 ++++--- src/mnode/src/mnodeProfile.c | 24 +++++++++--------- src/mnode/src/mnodeSdb.c | 36 ++++++++++++--------------- src/mnode/src/mnodeShow.c | 9 ++++++- src/mnode/src/mnodeTable.c | 47 ++++++++++++++++++------------------ src/mnode/src/mnodeUser.c | 9 ++++--- src/mnode/src/mnodeVgroup.c | 15 +++++------- src/vnode/src/vnodeMain.c | 7 +++--- tests/script/basicSuite.sim | 1 - 23 files changed, 139 insertions(+), 123 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index df78f4fe27..b172867929 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -162,7 +162,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); if (mnodeGetOnlineDnodesNum() == 0) { return TSDB_CODE_MND_NOT_READY; @@ -377,15 +376,13 @@ static bool balanceMonitorBalance() { srcScore, pDestDnode->score, destScore); balanceAddVnode(pVgroup, pSrcDnode, pDestDnode); mnodeDecVgroupRef(pVgroup); - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); return true; } } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } return false; @@ -413,8 +410,6 @@ void balanceReset() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - tsAccessSquence = 0; } @@ -443,12 +438,11 @@ static int32_t balanceMonitorVgroups() { mnodeDecVgroupRef(pVgroup); if (code == TSDB_CODE_SUCCESS) { + mnodeCancelGetNextVgroup(pIter); break; } } - sdbFreeIter(pIter); - return hasUpdatingVgroup; } @@ -465,11 +459,12 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup); mnodeDecVgroupRef(pVgroup); - if (hasThisDnode) break; + if (hasThisDnode) { + mnodeCancelGetNextVgroup(pIter); + break; + } } - sdbFreeIter(pIter); - if (!hasThisDnode) { mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId); mnodeDropDnode(pDnode, NULL); @@ -499,20 +494,18 @@ static bool balanceMontiorDropping() { pDnode->status = TAOS_DN_STATUS_DROPPING; mnodeUpdateDnode(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return true; } if (pDnode->status == TAOS_DN_STATUS_DROPPING) { bool ret = balanceMonitorDnodeDropping(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return ret; } } - sdbFreeIter(pIter); - return false; } @@ -556,8 +549,6 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) { } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } static void balanceCheckDnodeAccess() { @@ -578,8 +569,6 @@ static void balanceCheckDnodeAccess() { } mnodeDecDnodeRef(pDnode); } - - sdbFreeIter(pIter); } static void balanceProcessBalanceTimer(void *handle, void *tmrId) { @@ -630,6 +619,7 @@ void balanceAsyncNotify() { int32_t balanceInit() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); pthread_mutex_init(&tsBalanceMutex, NULL); balanceInitDnodeList(); @@ -667,8 +657,6 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) { mnodeDecDnodeRef(pTempDnode); } - sdbFreeIter(pIter); - if (pDnode->openVnodes > totalFreeVnodes) { mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes); return TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -780,8 +768,12 @@ void balanceAccquireDnodeList() { SDnodeObj *pDnode = NULL; int32_t dnodeIndex = 0; - while (1) { - if (dnodeIndex >= dnodesNum) break; + while (1) { + if (dnodeIndex >= dnodesNum) { + mnodeCancelGetNextDnode(pIter); + break; + } + pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { @@ -802,8 +794,6 @@ void balanceAccquireDnodeList() { dnodeIndex++; } - sdbFreeIter(pIter); - tsBalanceDnodeListSize = dnodeIndex; } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 744a62f948..595dcca413 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -27,6 +27,7 @@ void mnodeCleanupAccts(); void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo); void * mnodeGetAcct(char *acctName); void * mnodeGetNextAcct(void *pIter, SAcctObj **pAcct); +void mnodeCancelGetNextAcct(void *pIter); void mnodeIncAcctRef(SAcctObj *pAcct); void mnodeDecAcctRef(SAcctObj *pAcct); void mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index 7cbd08ed92..9354b923d7 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -34,6 +34,7 @@ int64_t mnodeGetDbNum(); SDbObj *mnodeGetDb(char *db); SDbObj *mnodeGetDbByTableId(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); +void mnodeCancelGetNextDb(void *pIter); void mnodeIncDbRef(SDbObj *pDb); void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index b6ddb7a9bf..8bc29ef9ef 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); +void mnodeCancelGetNextDnode(void *pIter); void mnodeIncDnodeRef(SDnodeObj *pDnode); void mnodeDecDnodeRef(SDnodeObj *pDnode); void * mnodeGetDnode(int32_t dnodeId); diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index a28a03ea40..10cbcebe22 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -38,6 +38,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId); void * mnodeGetMnode(int32_t mnodeId); int32_t mnodeGetMnodesNum(); void * mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode); +void mnodeCancelGetNextMnode(void *pIter); void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 29d8cf1207..81629ede78 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,9 +92,9 @@ int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); -void *sdbGetRow(void *pTable, void *key); -void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); -void sdbFreeIter(void *pIter); +void * sdbGetRow(void *pTable, void *key); +void * sdbFetchRow(void *pTable, void *pIter, void **ppRow); +void sdbFreeIter(void *pTable, void *pIter); void sdbIncRef(void *pTable, void *pRow); void sdbDecRef(void *pTable, void *pRow); int64_t sdbGetNumOfRows(void *pTable); diff --git a/src/mnode/inc/mnodeShow.h b/src/mnode/inc/mnodeShow.h index da66e71678..f985fe792d 100644 --- a/src/mnode/inc/mnodeShow.h +++ b/src/mnode/inc/mnodeShow.h @@ -26,8 +26,10 @@ void mnodeCleanUpShow(); typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); +typedef void (*SShowFreeIterFp)(void *pIter); void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index 7c0077aa60..bf04f26a90 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -31,6 +31,8 @@ void mnodeIncTableRef(void *pTable); void mnodeDecTableRef(void *pTable); void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable); void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable); +void mnodeCancelGetNextChildTable(void *pIter); +void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 073460f9d3..156bc7aaeb 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -25,6 +25,7 @@ int32_t mnodeInitUsers(); void mnodeCleanupUsers(); SUserObj *mnodeGetUser(char *name); void * mnodeGetNextUser(void *pIter, SUserObj **pUser); +void mnodeCancelGetNextUser(void *pIter); void mnodeIncUserRef(SUserObj *pUser); void mnodeDecUserRef(SUserObj *pUser); SUserObj *mnodeGetUserFromConn(void *pConn); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 0e6d9dfde4..ee9ec7ae93 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -34,6 +34,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); +void mnodeCancelGetNextVgroup(void *pIter); void mnodeUpdateVgroup(SVgObj *pVgroup); void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload); void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 9fff2f0229..3da889b284 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -144,7 +144,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries; mnodeDecAcctRef(pAcct); } - sdbFreeIter(pIter); SVgObj *pVgroup = NULL; pIter = NULL; @@ -158,7 +157,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->totalPoints += pVgroup->pointsWritten; mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } void *mnodeGetAcct(char *name) { @@ -169,6 +167,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) { return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); } +void mnodeCancelGetNextAcct(void *pIter) { + sdbFreeIter(tsAcctSdb, pIter); +} + void mnodeIncAcctRef(SAcctObj *pAcct) { sdbIncRef(tsAcctSdb, pAcct); } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 5be67e4ad9..839407c5b0 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -31,6 +31,7 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextCluster(void *pIter); static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) { tfree(pRow->pObj); @@ -108,6 +109,7 @@ int32_t mnodeInitCluster() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); mDebug("table:%s, hash is created", desc.name); return TSDB_CODE_SUCCESS; @@ -122,6 +124,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) { return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); } +void mnodeCancelGetNextCluster(void *pIter) { + sdbFreeIter(tsClusterSdb, pIter); +} + void mnodeIncClusterRef(SClusterObj *pCluster) { sdbIncRef(tsClusterSdb, pCluster); } @@ -167,7 +173,7 @@ void mnodeUpdateClusterId() { } mnodeDecClusterRef(pCluster); - sdbFreeIter(pIter); + mnodeCancelGetNextCluster(pIter); } static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index d121208447..77f3b93eb1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -171,6 +171,7 @@ int32_t mnodeInitDbs() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb); mDebug("table:dbs table is created"); return 0; @@ -180,6 +181,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) { return sdbFetchRow(tsDbSdb, pIter, (void **)pDb); } +void mnodeCancelGetNextDb(void *pIter) { + sdbFreeIter(tsDbSdb, pIter); +} + SDbObj *mnodeGetDb(char *db) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } @@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { SDbObj *pDb = pMsg->pDb; void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; + SVgObj *pVgroup = NULL; + while (1) { pIter = mnodeGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; if (pVgroup->pDb == pDb) { @@ -995,7 +1000,6 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { } mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); mDebug("db:%s, all vgroups is altered", pDb->name); mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); @@ -1146,7 +1150,5 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mnodeDecDbRef(pDb); } - sdbFreeIter(pIter); - mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index f76533c760..0e9b56e459 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -206,6 +206,7 @@ int32_t mnodeInitDnodes() { mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode); mDebug("table:dnodes table is created"); return 0; @@ -223,6 +224,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) { return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); } +void mnodeCancelGetNextDnode(void *pIter) { + sdbFreeIter(tsDnodeSdb, pIter); +} + int32_t mnodeGetDnodesNum() { return sdbGetNumOfRows(tsDnodeSdb); } @@ -241,8 +246,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (cpuCores < 2) cpuCores = 2; return cpuCores; } @@ -259,8 +262,6 @@ int32_t mnodeGetOnlineDnodesNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - return onlineDnodes; } @@ -276,13 +277,12 @@ void *mnodeGetDnodeByEp(char *ep) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (strcmp(ep, pDnode->dnodeEp) == 0) { - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return pDnode; } mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); return NULL; } @@ -464,7 +464,10 @@ static void mnodeUpdateDnodeEps() { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - if (dnodesNum >= totalDnodes) break; + if (dnodesNum >= totalDnodes) { + mnodeCancelGetNextDnode(pIter); + break; + } SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum]; dnodesNum++; @@ -474,7 +477,6 @@ static void mnodeUpdateDnodeEps() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); pthread_mutex_unlock(&tsDnodeEpsMutex); } @@ -1100,7 +1102,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pDnode = mnodeGetDnodeByEp(pShow->payload); } else { void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); } if (pDnode != NULL) { @@ -1148,7 +1150,6 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } else { numOfRows = 0; } @@ -1217,8 +1218,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (pSelDnode == NULL) { mError("failed to alloc vnode to vgroup"); return TSDB_CODE_MND_NO_ENOUGH_DNODES; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 205bfda4b9..8e3ad0a248 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() { pMnode->role = TAOS_SYNC_ROLE_MASTER; mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); + mnodeCancelGetNextMnode(pIter); } mnodeUpdateMnodeEpSet(); @@ -161,6 +161,7 @@ int32_t mnodeInitMnodes() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode); mDebug("table:mnodes table is created"); return TSDB_CODE_SUCCESS; @@ -192,6 +193,10 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); } +void mnodeCancelGetNextMnode(void *pIter) { + sdbFreeIter(tsMnodeSdb, pIter); +} + void mnodeUpdateMnodeEpSet() { mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); @@ -239,8 +244,6 @@ void mnodeUpdateMnodeEpSet() { tsMnodeEpSetForShell.numOfEps = index; tsMnodeEpSetForPeer.numOfEps = index; - sdbFreeIter(pIter); - mnodeMnodeUnLock(); } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 12ac64854c..8467adfc17 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -42,6 +42,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextConn(void *pIter); static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeFreeConn(void *data); @@ -52,10 +53,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg); int32_t mnodeInitProfile() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); @@ -137,21 +141,15 @@ static void mnodeFreeConn(void *data) { mDebug("connId:%d, is destroyed", pConn->connId); } -static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { +static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) { *pConn = NULL; - if (pIter == NULL) { - pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable); - } + pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - SCacheDataNode **pNode = taosHashIterGet(pIter); + SCacheDataNode **pNode = pIter; if (pNode == NULL || *pNode == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); return NULL; } @@ -159,6 +157,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { return pIter; } +static void mnodeCancelGetNextConn(void *pIter) { + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); +} + static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mnodeGetUserFromConn(pConn); if (pUser == NULL) return 0; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1728bb7e3b..575845b4f2 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -325,7 +325,6 @@ void sdbUpdateSync(void *pMnodes) { mnodeDecDnodeRef(pDnode); mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); syncCfg.replica = index; mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { @@ -756,24 +755,17 @@ int32_t sdbUpdateRow(SSdbRow *pRow) { } } -void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { +void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) { SSdbTable *pTable = tparam; *ppRow = NULL; if (pTable == NULL) return NULL; - SHashMutableIterator *pIter = pNode; - if (pIter == NULL) { - pIter = taosHashCreateIter(pTable->iHandle); - } + pIter = taosHashIterate(pTable->iHandle, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - void **ppMetaRow = taosHashIterGet(pIter); + void **ppMetaRow = pIter; if (ppMetaRow == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); return NULL; } @@ -783,10 +775,11 @@ void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { return pIter; } -void sdbFreeIter(void *pIter) { - if (pIter != NULL) { - taosHashDestroyIter(pIter); - } +void sdbFreeIter(void *tparam, void *pIter) { + SSdbTable *pTable = tparam; + if (pTable == NULL || pIter == NULL) return; + + taosHashCancelIterate(pTable->iHandle, pIter); } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -827,9 +820,10 @@ void sdbCloseTable(void *handle) { tsSdbMgmt.numOfTables--; tsSdbMgmt.tableList[pTable->id] = NULL; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); - while (taosHashIterNext(pIter)) { - void **ppRow = taosHashIterGet(pIter); + void *pIter = taosHashIterate(pTable->iHandle, NULL); + while (pIter) { + void **ppRow = pIter; + pIter = taosHashIterate(pTable->iHandle, pIter); if (ppRow == NULL) continue; SSdbRow row = { @@ -840,7 +834,7 @@ void sdbCloseTable(void *handle) { (*pTable->fpDestroy)(&row); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 0d53fa9617..2621441cb5 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -57,6 +57,7 @@ static void *tsMnodeShowCache = NULL; static int32_t tsShowObjIndex = 0; static SShowMetaFp tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0}; static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0}; +static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0}; int32_t mnodeInitShow() { mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg); @@ -85,6 +86,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) { tsMnodeShowRetrieveFp[msgType] = fp; } +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) { + tsMnodeShowFreeIterFp[msgType] = fp; +} + static char *mnodeGetShowType(int32_t showType) { switch (showType) { case TSDB_MGMT_TABLE_ACCT: return "show accounts"; @@ -412,7 +417,9 @@ static void* mnodePutShowObj(SShowObj *pShow) { static void mnodeFreeShowObj(void *data) { SShowObj *pShow = *(SShowObj **)data; - sdbFreeIter(pShow->pIter); + if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) { + (*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter); + } mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); tfree(pShow); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4d5f0808f5..8b07f73c65 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -342,8 +342,7 @@ static int32_t mnodeChildTableActionRestored() { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - + mnodeCancelGetNextChildTable(pIter); return 0; } @@ -602,10 +601,13 @@ int32_t mnodeInitTables() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable); return TSDB_CODE_SUCCESS; } @@ -626,14 +628,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) { pIter = mnodeGetNextSuperTable(pIter, &pStable); if (pStable == NULL) break; if (pStable->uid == uid) { - sdbFreeIter(pIter); + mnodeCancelGetNextSuperTable(pIter); return pStable; } mnodeDecTableRef(pStable); } - sdbFreeIter(pIter); - return NULL; } @@ -655,10 +655,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) { return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextChildTable(void *pIter) { + sdbFreeIter(tsChildTableSdb, pIter); +} + void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) { return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextSuperTable(void *pIter) { + sdbFreeIter(tsSuperTableSdb, pIter); +} + void mnodeIncTableRef(void *p1) { STableObj *pTable = (STableObj *)p1; if (pTable->type == TSDB_SUPER_TABLE) { @@ -914,10 +922,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { - SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); + int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); + while (pVgId) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pStable->vgHash, pVgId); if (pVgroup == NULL) break; SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg)); @@ -933,7 +941,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + + taosHashCancelIterate(pStable->vgHash, pVgId); mnodeDropAllChildTablesInStable(pStable); } @@ -1430,8 +1439,6 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -1523,11 +1530,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash); - int32_t vgSize = 0; - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); - SVgObj * pVgroup = mnodeGetVgroup(*pVgId); + int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); + int32_t vgSize = 0; + while (pVgId) { + SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pTable->vgHash, pVgId); if (pVgroup == NULL) continue; pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); @@ -1547,7 +1554,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->vgHash, pVgId); mnodeDecTableRef(pTable); pVgroupMsg->numOfVgroups = htonl(vgSize); @@ -2230,8 +2237,6 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId); } @@ -2263,8 +2268,6 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -2293,8 +2296,6 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index dc76d92eb8..aee167631f 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -123,7 +123,6 @@ static void mnodePrintUserAuth() { } fflush(fp); - sdbFreeIter(pIter); fclose(fp); } @@ -177,6 +176,8 @@ int32_t mnodeInitUsers() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser); + mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg); mDebug("table:%s, hash is created", desc.name); @@ -196,6 +197,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) { return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); } +void mnodeCancelGetNextUser(void *pIter) { + sdbFreeIter(tsUserSdb, pIter); +} + void mnodeIncUserRef(SUserObj *pUser) { return sdbIncRef(tsUserSdb, pUser); } @@ -574,8 +579,6 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { mnodeDecUserRef(pUser); } - sdbFreeIter(pIter); - mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index f9a49e5ec2..a8343083aa 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -230,6 +230,7 @@ int32_t mnodeInitVgroups() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); @@ -304,7 +305,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); } void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) { @@ -491,6 +492,10 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); } +void mnodeCancelGetNextVgroup(void *pIter) { + sdbFreeIter(tsVgroupSdb, pIter); +} + static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) { SVgObj *pVgroup = pMsg->pVgroup; SDbObj *pDb = pMsg->pDb; @@ -1095,8 +1100,6 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups); } @@ -1118,8 +1121,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name); } #endif @@ -1147,8 +1148,6 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } @@ -1170,7 +1169,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { numOfVgroups++; } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 339391369f..74df989090 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -542,10 +542,11 @@ void vnodeBuildStatusMsg(void *param) { void *pIter = taosHashIterate(tsVnodesHash, NULL); while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - vnodeBuildVloadMsg(*pVnode, pStatus); + SVnodeObj **pVnode = pIter; + if (*pVnode) { + vnodeBuildVloadMsg(*pVnode, pStatus); } + pIter = taosHashIterate(tsVnodesHash, pIter); } } diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 61a9d68d26..5e22e02297 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -14,7 +14,6 @@ run general/table/vgroup.sim run general/user/authority.sim run general/vector/metrics_mix.sim run general/vector/table_field.sim -run general/user/authority.sim run general/tag/set.sim run general/table/delete_writing.sim run general/stable/disk.sim From 58bbdc329348a3c2fd81e48e186dd04043b2ccce Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 17:19:34 +0800 Subject: [PATCH 26/75] TD-2166 add refcount while perform sync --- src/inc/tsync.h | 15 ++-- src/mnode/src/mnodeSdb.c | 15 ++-- src/sync/inc/syncInt.h | 1 - src/sync/src/syncMain.c | 23 +++--- src/sync/src/syncRestore.c | 14 ++-- src/sync/src/syncRetrieve.c | 10 +-- src/sync/test/syncServer.c | 13 ++- src/vnode/src/vnodeMain.c | 157 +++++++++++++++++++++++++----------- 8 files changed, 153 insertions(+), 95 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 967b254992..10b2e3f66a 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -64,33 +64,32 @@ typedef struct { if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. */ -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); +typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); +typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); // when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); +typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); // when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); +typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); // when role is changed, call this to notify app -typedef void (*FNotifyRole)(void *ahandle, int8_t role); +typedef void (*FNotifyRole)(int32_t vgId, int8_t role); // if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds); +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t mseconds); // when data file is synced successfully, notity app -typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); +typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt char path[128]; // path to the file - void * ahandle; // handle provided by APP FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1728bb7e3b..e5df1d732a 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -107,7 +107,7 @@ static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); -static int32_t sdbWriteWalToQueue(void *vparam, void *pHead, int32_t qtype, void *rparam); +static int32_t sdbWriteWalToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); static void sdbFreeFromQueue(SSdbRow *pRow); static void * sdbWorkerFp(void *pWorker); @@ -228,16 +228,16 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } -static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { +static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { return walGetWalFile(tsSdbMgmt.wal, fileName, fileId); } -static void sdbNotifyRole(void *ahandle, int8_t role) { +static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]); if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) { @@ -264,7 +264,7 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { } FORCE_INLINE -static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { +static void sdbConfirmForward(int32_t vgId, void *wparam, int32_t code) { if (wparam == NULL) return; SSdbRow *pRow = wparam; SMnodeMsg * pMsg = pRow->pMsg; @@ -370,7 +370,6 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.ahandle = NULL; syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; syncInfo.writeToCache = sdbWriteWalToQueue; @@ -967,7 +966,7 @@ static void sdbFreeFromQueue(SSdbRow *pRow) { taosFreeQitem(pRow); } -static int32_t sdbWriteWalToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { +static int32_t sdbWriteWalToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { SWalHead *pHead = wparam; int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len; @@ -1039,7 +1038,7 @@ static void *sdbWorkerFp(void *pWorker) { taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow); if (qtype == TAOS_QTYPE_RPC) { - sdbConfirmForward(NULL, pRow, pRow->code); + sdbConfirmForward(1, pRow, pRow->code); } else { if (qtype == TAOS_QTYPE_FWD) { syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7d846ebc80..f1371bdc59 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -153,7 +153,6 @@ typedef struct SSyncNode { int8_t selfIndex; uint32_t vgId; int64_t rid; - void *ahandle; SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator SSyncPeer *pMaster; SRecvBuffer *pRecv; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 843de9461f..de0fc8fadc 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -181,7 +181,6 @@ int64_t syncStart(const SSyncInfo *pInfo) { tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); pthread_mutex_init(&pNode->mutex, NULL); - pNode->ahandle = pInfo->ahandle; pNode->getFileInfo = pInfo->getFileInfo; pNode->getWalInfo = pInfo->getWalInfo; pNode->writeToCache = pInfo->writeToCache; @@ -255,7 +254,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); if (pNode->notifyRole) { - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } return pNode->rid; @@ -348,7 +347,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { if (pNewCfg->replica <= 1) { sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } pthread_mutex_unlock(&(pNode->mutex)); @@ -412,7 +411,7 @@ void syncRecover(int64_t rid) { // if take this node to unsync state, the whole system may not work nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeVersion = 0; pthread_mutex_lock(&(pNode->mutex)); @@ -568,7 +567,7 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { } if (pNode->notifyFlowCtrl) { - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + (*pNode->notifyFlowCtrl)(pNode->vgId, 0); } } @@ -631,7 +630,7 @@ static void syncChooseMaster(SSyncNode *pNode) { } #endif syncResetFlowCtrl(pNode); - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } else { pPeer = pNode->peerInfo[index]; sInfo("%s, it shall work as master", pPeer->id); @@ -662,7 +661,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { @@ -675,7 +674,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } } } @@ -692,7 +691,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); code = -1; for (int32_t index = 0; index < pNode->replica; ++index) { @@ -729,7 +728,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } else { sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); @@ -913,7 +912,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); @@ -1228,7 +1227,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code if (confirm && pFwdInfo->confirmed == 0) { sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); - (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code); + (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index cc2315fb15..b31ec5f7c7 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -38,7 +38,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex while (1) { name[0] = 0; - magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion); + magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); if (magic == 0) break; snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); @@ -84,7 +84,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // check the file info sinfo = minfo; sDebug("%s, get file info:%s", pPeer->id, minfo.name); - sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, + sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); // if file not there or magic is not the same, file shall be synced @@ -164,7 +164,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } lastVer = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); } if (code < 0) { @@ -179,7 +179,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; @@ -276,7 +276,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { // if code > 0, data file is changed, notify app, and pass the version if (code > 0 && pNode->notifyFileSynced) { - if ((*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0) { + if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) { sError("%s, app not in ready state", pPeer->id); return -1; } @@ -307,7 +307,7 @@ void *syncRestoreData(void *param) { taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); - (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING); + (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); @@ -324,7 +324,7 @@ void *syncRestoreData(void *param) { } } - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; taosClose(pPeer->syncFd); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 82f40854e8..7c1286f419 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -108,7 +108,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { while (1) { // retrieve file info fileInfo.name[0] = 0; - fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, + fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); @@ -354,7 +354,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) index++; wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); + code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); if (code < 0) break; if (wname[0] == 0) { code = 0; @@ -382,7 +382,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); + code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); if (code < 0) break; // error if (wname[0] == 0) { // no wal file sDebug("%s, no wal file", pPeer->id); @@ -487,10 +487,10 @@ void *syncRetrieveData(void *param) { // if file is changed 3 times continuously, start flow control pPeer->numOfRetrieves++; if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2)); + (*pNode->notifyFlowCtrl)(pNode->vgId, 4 << (pPeer->numOfRetrieves - 2)); } else { pPeer->numOfRetrieves = 0; - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); } pPeer->fileChanged = 0; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 9dd3feb461..5a64a0a36a 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -70,7 +70,7 @@ int writeIntoWal(SWalHead *pHead) { return 0; } -void confirmForward(void *ahandle, void *mhandle, int32_t code) { +void confirmForward(int32_t vgId, void *mhandle, int32_t code) { SRpcMsg * pMsg = (SRpcMsg *)mhandle; SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); @@ -227,7 +227,7 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); } -uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { uint32_t magic; struct stat fstat; char aname[280]; @@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex return magic; } -int getWalInfo(void *ahandle, char *name, int64_t *index) { +int getWalInfo(int32_t vgId, char *name, int64_t *index) { struct stat fstat; char aname[280]; @@ -272,7 +272,7 @@ int getWalInfo(void *ahandle, char *name, int64_t *index) { return 1; } -int writeToCache(void *ahandle, void *data, int type) { +int writeToCache(int32_t vgId, void *data, int type) { SWalHead *pHead = data; uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); @@ -285,9 +285,9 @@ int writeToCache(void *ahandle, void *data, int type) { return 0; } -void confirmFwd(void *ahandle, int64_t version) { return; } +void confirmFwd(int32_t vgId, int64_t version) { return; } -void notifyRole(void *ahandle, int8_t r) { +void notifyRole(int32_t vgId, int8_t r) { role = r; printf("current role:%s\n", syncRole[role]); } @@ -296,7 +296,6 @@ void initSync() { pCfg->replica = 1; pCfg->quorum = 1; syncInfo.vgId = 1; - syncInfo.ahandle = &syncInfo; syncInfo.getFileInfo = getFileInfo; syncInfo.getWalInfo = getWalInfo; syncInfo.writeToCache = writeToCache; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3f20efb150..50744d59bc 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -30,19 +30,21 @@ static SHashObj*tsVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); -static int vnodeProcessTsdbStatus(void *arg, int status, int eno); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); -static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId); -static void vnodeNotifyRole(void *ahandle, int8_t role); -static void vnodeCtrlFlow(void *handle, int32_t mseconds); -static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); +static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); +static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); +static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); +static void vnodeNotifyRole(int32_t vgId, int8_t role); +static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds); +static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); +static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); +static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); #ifndef _SYNC int64_t syncStart(const SSyncInfo *info) { return NULL; } -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; } +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype) { return 0; } void syncStop(int64_t rid) {} -int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; } -int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; } +int32_t syncReconfig(int64_t rid, const SSyncCfg *cfg) { return 0; } +int32_t syncGetNodesRole(int64_t rid, SNodesRole *cfg) { return 0; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {} #endif @@ -55,7 +57,7 @@ char* vnodeStatus[] = { }; int32_t vnodeInitResources() { - int code = syncInit(); + int32_t code = syncInit(); if (code != 0) return code; vnodeInitWriteFp(); @@ -325,16 +327,29 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { walRemoveAllOldFiles(pVnode->wal); walRenew(pVnode->wal); + pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + + pVnode->events = NULL; + pVnode->status = TAOS_VN_STATUS_READY; + + vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); + tsdbIncCommitRef(pVnode->vgId); + + taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; sprintf(syncInfo.path, "%s", rootDir); - syncInfo.ahandle = pVnode; syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToWQueue; - syncInfo.confirmForward = dnodeSendRpcVWriteRsp; + syncInfo.writeToCache = vnodeWriteToCache; + syncInfo.confirmForward = vnodeConfirmForard; syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; @@ -346,23 +361,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->sync <= 0) { vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); + vnodeRelease(pVnode); vnodeCleanUp(pVnode); return terrno; } -#endif - - pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); - if (pVnode->qMgmt == NULL) { - vnodeCleanUp(pVnode); - return terrno; - } - - pVnode->events = NULL; - pVnode->status = TAOS_VN_STATUS_READY; - vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - - tsdbIncCommitRef(pVnode->vgId); - taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); +#endif return TSDB_CODE_SUCCESS; } @@ -389,7 +392,7 @@ void vnodeRelease(void *vparam) { assert(refCount >= 0); if (refCount > 0) { - if (pVnode->status == TAOS_VN_STATUS_RESET && refCount == 2) { + if (pVnode->status == TAOS_VN_STATUS_RESET && refCount <= 3) { tsem_post(&pVnode->sem); } return; @@ -571,7 +574,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) != TAOS_VN_STATUS_READY) { if (++i % 1000 == 0) { @@ -595,7 +598,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // TODO: this is a simple implement -static int vnodeProcessTsdbStatus(void *arg, int status, int eno) { +static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { SVnodeObj *pVnode = arg; if (eno != TSDB_CODE_SUCCESS) { @@ -625,20 +628,41 @@ static int vnodeProcessTsdbStatus(void *arg, int status, int eno) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, +static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { - SVnodeObj *pVnode = ahandle; + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get file info", vgId); + return 0; + } + *fversion = pVnode->fversion; - return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + + vnodeRelease(pVnode); + return ret; } -static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { - SVnodeObj *pVnode = ahandle; - return walGetWalFile(pVnode->wal, fileName, fileId); +static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get wal info", vgId); + return -1; + } + + int32_t code = walGetWalFile(pVnode->wal, fileName, fileId); + + vnodeRelease(pVnode); + return code; } -static void vnodeNotifyRole(void *ahandle, int8_t role) { - SVnodeObj *pVnode = ahandle; +static void vnodeNotifyRole(int32_t vgId, int8_t role) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while notify role", vgId); + return; + } + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; dnodeSendStatusMsgToMnode(); @@ -648,17 +672,26 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { } else { cqStop(pVnode->cq); } + + vnodeRelease(pVnode); } -static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) { - SVnodeObj *pVnode = ahandle; +static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while ctrl flow", vgId); + return; + } + if (pVnode->delayMs != mseconds) { pVnode->delayMs = mseconds; vDebug("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); } + + vnodeRelease(pVnode); } -static int vnodeResetTsdb(SVnodeObj *pVnode) { +static int32_t vnodeResetTsdb(SVnodeObj *pVnode) { char rootDir[128] = "\0"; sprintf(rootDir, "%s/tsdb", pVnode->rootDir); @@ -672,7 +705,7 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) { // acquire vnode int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - if (refCount > 2) { + if (refCount > 3) { tsem_wait(&pVnode->sem); } @@ -692,14 +725,44 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) { return 0; } -static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { - SVnodeObj *pVnode = ahandle; +static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while notify file synced", vgId); + return 0; + } pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); - vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, - pVnode->version); - return vnodeResetTsdb(pVnode); + vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + int32_t code = vnodeResetTsdb(pVnode); + + vnodeRelease(pVnode); + return code; +} + +void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { + void *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while confirm forward", vgId); + return; + } + + dnodeSendRpcVWriteRsp(pVnode, wparam, code); + vnodeRelease(pVnode); +} + +static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + + int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam); + + vnodeRelease(pVnode); + return code; } From 212fe11d41db46fffbb306f7e794e3ca91bc0be5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 17:46:23 +0800 Subject: [PATCH 27/75] TD-2166 --- src/vnode/src/vnodeMain.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 50744d59bc..75eacc8b49 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -63,7 +63,7 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -175,8 +175,8 @@ int32_t vnodeDrop(int32_t vgId) { return TSDB_CODE_SUCCESS; } -int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) { - SVnodeObj *pVnode = param; +int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { + SVnodeObj *pVnode = vparam; // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS // cfgVersion can be corrected by status msg @@ -334,7 +334,6 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } pVnode->events = NULL; - pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); tsdbIncCommitRef(pVnode->vgId); @@ -367,6 +366,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } #endif + pVnode->status = TAOS_VN_STATUS_READY; return TSDB_CODE_SUCCESS; } @@ -611,17 +611,18 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->fversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); - if (pVnode->status == TAOS_VN_STATUS_INIT) { - return 0; - } else { + if (pVnode->status != TAOS_VN_STATUS_INIT) { return walRenew(pVnode->wal); } + return 0; } if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); pVnode->isFull = 0; - walRemoveOneOldFile(pVnode->wal); + if (pVnode->status != TAOS_VN_STATUS_INIT) { + walRemoveOneOldFile(pVnode->wal); + } return vnodeSaveVersion(pVnode); } @@ -701,7 +702,7 @@ static int32_t vnodeResetTsdb(SVnodeObj *pVnode) { void *tsdb = pVnode->tsdb; pVnode->tsdb = NULL; - + // acquire vnode int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); From 4f1666e79f4fdb08b14d1ba3f6b6e27940fdd0ec Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 22 Nov 2020 18:52:41 +0800 Subject: [PATCH 28/75] [TD-1880] add test case --- tests/pytest/cluster/clusterSetup.py | 27 ++++++++++++++++++---- tests/pytest/query/queryJoin.py | 34 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index 36af8ac42e..dbda5657b6 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -31,6 +31,23 @@ class Node: self.homeDir = homeDir self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)}) + def buildTaosd(self): + try: + self.conn.cd("/root/TDinternal/community") + self.conn.run("git checkout develop") + self.conn.run("git pull") + self.conn.cd("/root/TDinternal") + self.conn.run("git checkout develop") + self.conn.run("git pull") + self.conn.cd("/root/TDinternal/debug") + self.conn.run("cmake ..") + self.conn.run("make") + self.conn.run("make install") + except Exception as e: + print("Build Taosd error for node %d " % self.index) + logging.exception(e) + pass + def startTaosd(self): try: self.conn.run("sudo systemctl start taosd") @@ -50,7 +67,7 @@ class Node: self.conn.run("sudo systemctl restart taosd") except Exception as e: print("Stop Taosd error for node %d " % self.index) - logging.exception(e) + logging.exception(e) def removeTaosd(self): try: @@ -105,9 +122,11 @@ class Node: class Nodes: def __init__(self): - self.node1 = Node(1, 'ubuntu', '192.168.1.52', 'node1', 'tbase125!', '/home/ubuntu') - self.node2 = Node(2, 'ubuntu', '192.168.1.53', 'node2', 'tbase125!', '/home/ubuntu') - self.node3 = Node(3, 'ubuntu', '192.168.1.54', 'node3', 'tbase125!', '/home/ubuntu') + self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/') + self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/') + self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/') + self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/') + self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/') def stopAllTaosd(self): self.node1.stopTaosd() diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 5ad49a265e..57b7e8868d 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -141,6 +141,40 @@ class TDTestCase: tdSql.query("select * from meters1, meters3 where meters1.ts = meters3.ts and meters1.tag1 = meters3.tag1") tdSql.checkRows(0) + tdSql.execute("create table join_mt0(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12), t3 int)") + + ts = 1538548685000 + for i in range(3): + tdSql.execute("create table join_tb%d using join_mt0 tags(%d, 'abc')" % (i, i)) + sql = "insert into join_tb%d values" % i + for j in range(500): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + sql = "insert into join_tb%d values" % i + for j in range(500, 1000): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + + for i in range(3): + tdSql.execute("create table join_1_tb%d using join_mt1 tags(%d, 'abc%d', %d)" % (i, i, i, i)) + sql = "insert into join_1_tb%d values" % i + for j in range(500): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + sql = "insert into join_1_tb%d values" % i + for j in range(500, 1000): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + + tdSql.error("select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") def stop(self): tdSql.close() From 38af2df40bc68cde730e32c6e550c2209edab656 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 21:06:35 +0800 Subject: [PATCH 29/75] TD-2166 --- src/dnode/src/dnodeEps.c | 2 +- src/inc/tsync.h | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 2 +- src/sync/inc/syncInt.h | 10 +++++----- src/sync/src/syncMain.c | 4 ++-- src/vnode/src/vnodeRead.c | 3 +-- src/vnode/src/vnodeWrite.c | 8 ++++---- 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index 9c90c39181..83f294e05e 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -33,7 +33,7 @@ static void dnodePrintEps(SDnodeEps *eps); int32_t dnodeInitEps() { pthread_mutex_init(&tsEpsMutex, NULL); - tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); dnodeResetEps(NULL); int32_t ret = dnodeReadEps(); if (ret == 0) { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 10b2e3f66a..77a3b36e73 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -89,7 +89,7 @@ typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt - char path[128]; // path to the file + char path[TSDB_FILENAME_LEN]; // path to the file FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index e5df1d732a..26a881d477 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -812,7 +812,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); } - pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); + pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, HASH_ENTRY_LOCK); tsSdbMgmt.numOfTables++; tsSdbMgmt.tableList[pTable->id] = pTable; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4d5f0808f5..c4601658b3 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -394,7 +394,7 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } if (pStable->vgHash != NULL) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index f1371bdc59..0790134997 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -153,12 +153,12 @@ typedef struct SSyncNode { int8_t selfIndex; uint32_t vgId; int64_t rid; - SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator - SSyncPeer *pMaster; + SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator + SSyncPeer * pMaster; SRecvBuffer *pRecv; - SSyncFwds *pSyncFwds; // saved forward info if quorum >1 - void *pFwdTimer; - void *pRoleTimer; + SSyncFwds * pSyncFwds; // saved forward info if quorum >1 + void * pFwdTimer; + void * pRoleTimer; FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index de0fc8fadc..2402d8ebfa 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -100,7 +100,7 @@ uint16_t syncGenTranId() { } int32_t syncInit() { - SPoolInfo info; + SPoolInfo info = {0}; info.numOfThreads = tsSyncTcpThreads; info.serverIp = 0; @@ -124,7 +124,7 @@ int32_t syncInit() { return -1; } - tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { sError("failed to init tsVgIdHash"); taosTmrCleanUp(tsSyncTmrCtrl); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index e10d62a0c9..5ef79cfbf0 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -53,8 +53,7 @@ int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead); } -static int32_t vnodeCheckRead(void *vparam) { - SVnodeObj *pVnode = vparam; +static int32_t vnodeCheckRead(SVnodeObj *pVnode) { if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], pVnode->refCount, pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 2d2be602ed..a0227d84ba 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -48,10 +48,10 @@ void vnodeInitWriteFp(void) { } int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { - int32_t code = 0; - SVnodeObj * pVnode = vparam; - SWalHead * pHead = wparam; - SRspRet * pRspRet = rparam; + int32_t code = 0; + SVnodeObj *pVnode = vparam; + SWalHead * pHead = wparam; + SRspRet * pRspRet = rparam; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, From 94e662c9b0245774aa46ffc85f20890328b4626b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 22:51:57 +0800 Subject: [PATCH 30/75] minor changes --- src/mnode/src/mnodeTable.c | 2 +- src/sync/src/syncMain.c | 40 +++++++++++++++++++------------------- src/vnode/src/vnodeMain.c | 4 ++-- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index c4601658b3..d0a5402986 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -413,7 +413,7 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); if (pVgroup == NULL) { - taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); + taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)); mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, (int32_t)taosHashGetSize(pStable->vgHash)); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 2402d8ebfa..41d68b3149 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -201,10 +201,10 @@ int64_t syncStart(const SSyncInfo *pInfo) { return -1; } - for (int32_t i = 0; i < pCfg->replica; ++i) { - const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; - pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); - if (pNode->peerInfo[i] == NULL) { + for (int32_t index = 0; index < pCfg->replica; ++index) { + const SNodeInfo *pNodeInfo = pCfg->nodeInfo + index; + pNode->peerInfo[index] = syncAddPeer(pNode, pNodeInfo); + if (pNode->peerInfo[index] == NULL) { sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn, pNodeInfo->nodePort); syncStop(pNode->rid); @@ -212,7 +212,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { - pNode->selfIndex = i; + pNode->selfIndex = index; } } @@ -251,7 +251,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } syncAddArbitrator(pNode); - taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); + taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *)); if (pNode->notifyRole) { (*pNode->notifyRole)(pNode->vgId, nodeRole); @@ -268,14 +268,14 @@ void syncStop(int64_t rid) { sInfo("vgId:%d, cleanup sync", pNode->vgId); - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); - if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); + if (tsVgIdHash) taosHashRemove(tsVgIdHash, &pNode->vgId, sizeof(int32_t)); if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer); - for (int32_t i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; + for (int32_t index = 0; index < pNode->replica; ++index) { + pPeer = pNode->peerInfo[index]; if (pPeer) syncRemovePeer(pPeer); } @@ -297,7 +297,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, pNode->replica); - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { @@ -414,7 +414,7 @@ void syncRecover(int64_t rid) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeVersion = 0; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; @@ -831,7 +831,7 @@ static void syncNotStarted(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); pPeer->timer = NULL; sInfo("%s, sync connection is still not up, restart", pPeer->id); syncRestartConnection(pPeer); @@ -842,7 +842,7 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&(pNode->mutex)); } @@ -968,7 +968,7 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { char * cont = buffer; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); int32_t code = syncReadPeerMsg(pPeer, &head, cont); @@ -1066,7 +1066,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); sDebug("%s, check peer connection", pPeer->id); syncSetupPeerConnection(pPeer); @@ -1118,7 +1118,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } SSyncNode *pNode = *ppNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); SSyncPeer *pPeer; for (i = 0; i < pNode->replica; ++i) { @@ -1156,7 +1156,7 @@ static void syncProcessBrokenLink(void *param) { SSyncNode *pNode = pPeer->pSyncNode; if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s", pPeer->id, strerror(errno)); pPeer->peerFd = -1; @@ -1262,7 +1262,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds > 0) { - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; if (ABS(time - pFwdInfo->time) < 2000) break; @@ -1320,7 +1320,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle pSyncHead->len = sizeof(SWalHead) + pWalHead->len; fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 75eacc8b49..b94fea52bd 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -338,7 +338,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); tsdbIncCommitRef(pVnode->vgId); - taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; @@ -570,7 +570,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { static void vnodeCleanUp(SVnodeObj *pVnode) { // remove from hash, so new messages wont be consumed - taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); + taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t)); if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait From b6540b4417fa61c934b86a4ce983835774d07ba3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 11:35:38 +0800 Subject: [PATCH 31/75] TD-2153 --- src/mnode/src/mnodeSdb.c | 2 +- src/sync/inc/syncInt.h | 14 +++++++------- src/sync/src/syncMain.c | 22 ++++++++++------------ src/sync/src/syncRetrieve.c | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 26a881d477..36e1a1eb2b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -211,7 +211,7 @@ void sdbUpdateMnodeRoles() { if (tsSdbMgmt.sync <= 0) return; SNodesRole roles = {0}; - syncGetNodesRole(tsSdbMgmt.sync, &roles); + if (syncGetNodesRole(tsSdbMgmt.sync, &roles) != 0) return; sdbInfo("vgId:1, update mnodes role, replica:%d", tsSdbMgmt.cfg.replica); for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 0790134997..aacd72e930 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -159,14 +159,14 @@ typedef struct SSyncNode { SSyncFwds * pSyncFwds; // saved forward info if quorum >1 void * pFwdTimer; void * pRoleTimer; - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; - FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; + FGetFileInfo getFileInfo; + FGetWalInfo getWalInfo; + FWriteToCache writeToCache; + FConfirmForward confirmForward; + FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; - pthread_mutex_t mutex; + pthread_mutex_t mutex; } SSyncNode; // sync module global diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 41d68b3149..dd759ed9d4 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -552,18 +552,16 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { } void syncBroadcastStatus(SSyncNode *pNode) { - SSyncPeer *pPeer; - - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == pNode->selfIndex) continue; - pPeer = pNode->peerInfo[i]; + for (int32_t index = 0; index < pNode->replica; ++index) { + if (index == pNode->selfIndex) continue; + SSyncPeer *pPeer = pNode->peerInfo[index]; syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId()); } } static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int32_t i = 0; i < pNode->replica; ++i) { - pNode->peerInfo[i]->numOfRetrieves = 0; + for (int32_t index = 0; index < pNode->replica; ++index) { + pNode->peerInfo[index]->numOfRetrieves = 0; } if (pNode->notifyFlowCtrl) { @@ -1171,7 +1169,7 @@ static void syncProcessBrokenLink(void *param) { static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - uint64_t time = taosGetTimestampMs(); + int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds >= tsMaxFwdInfo) { pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; @@ -1289,7 +1287,6 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t fwdLen; int32_t code = 0; - if (pWalHead->version > nodeVersion + 1) { sError("vgId:%d, hver:%" PRIu64 ", inconsistent with sver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { @@ -1305,14 +1302,15 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle // always update version nodeVersion = pWalHead->version; - sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, - syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; // only pkt from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; + sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); + // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); pSyncHead->type = TAOS_SMSG_FORWARD; @@ -1332,7 +1330,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle code = 1; } - int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); + int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); } else { diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 7c1286f419..b6dacaa262 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -438,7 +438,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - if (write(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { + if (taosWriteMsg(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { sError("%s, failed to send syncCmd", pPeer->id); return -1; } From 0a90950bdfaddda5133ecd33596e6b7440105b26 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 10:08:15 +0000 Subject: [PATCH 32/75] TD-2157 --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 78 ++++++++++++++++++++++++------------- src/sync/src/syncRestore.c | 6 ++- src/sync/src/syncRetrieve.c | 6 ++- src/util/src/tsocket.c | 4 +- 5 files changed, 61 insertions(+), 34 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index aacd72e930..7dbb1985d4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -172,6 +172,7 @@ typedef struct SSyncNode { // sync module global extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; +extern char * syncStatus[]; void *syncRetrieveData(void *param); void *syncRestoreData(void *param); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index dd759ed9d4..a68f0b7bbb 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -73,6 +73,14 @@ char* syncRole[] = { "master" }; +char *syncStatus[] = { + "init", + "start", + "file", + "cache", + "invalid" +}; + typedef enum { SYNC_STATUS_BROADCAST, SYNC_STATUS_BROADCAST_RSP, @@ -282,7 +290,7 @@ void syncStop(int64_t rid) { pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; if (pPeer) syncRemovePeer(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, rid); taosRemoveRef(tsSyncRefId, rid); @@ -350,7 +358,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); @@ -423,7 +431,7 @@ void syncRecover(int64_t rid) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, rid); } @@ -498,6 +506,8 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) { } static void syncClosePeerConn(SSyncPeer *pPeer) { + sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); + taosTmrStopA(&pPeer->timer); taosClose(pPeer->syncFd); if (pPeer->peerFd >= 0) { @@ -751,7 +761,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new sDebug("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, version inconsistent, cannot choose master", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inconsistent", pNode->vgId); } } @@ -770,11 +780,12 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } static void syncRestartPeer(SSyncPeer *pPeer) { - sDebug("%s, restart peer connection", pPeer->id); + sDebug("%s, restart peer connection, last sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncClosePeerConn(pPeer); pPeer->sstatus = TAOS_SYNC_STATUS_INIT; + sDebug("%s, peer conn is restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { @@ -803,7 +814,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started", pPeer->id); + sDebug("%s, sync is already started for sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); return; // already started } @@ -821,7 +832,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { syncDecPeerRef(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, thread is created to retrieve data", pPeer->id); + sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); } } @@ -831,9 +842,10 @@ static void syncNotStarted(void *param, void *tmrId) { pthread_mutex_lock(&pNode->mutex); pPeer->timer = NULL; - sInfo("%s, sync connection is still not up, restart", pPeer->id); + pPeer->sstatus = TAOS_SYNC_STATUS_INIT; + sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { @@ -842,14 +854,14 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus); + sDebug("%s, sync is already started since sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); return; } @@ -877,7 +889,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { sError("%s, failed to send sync-req to peer", pPeer->id); } else { nodeSStatus = TAOS_SYNC_STATUS_START; - sInfo("%s, sync-req is sent", pPeer->id); + sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); } } @@ -915,7 +927,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); } else { - sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version); + sError("%s, forward discarded since sstatus:%s, hver:%" PRIu64, pPeer->id, syncStatus[nodeSStatus], + pHead->version); } } } @@ -924,8 +937,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s sver:%" PRIu64 " peer:%s sver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, - syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); + sDebug("%s, status msg is received, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sver:%" PRIu64 + ", ack:%d tranId:%u type:%s pfd:%d", + pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], + pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); pPeer->version = pPeersStatus->version; syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role); @@ -982,7 +997,7 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); return code; } @@ -1014,8 +1029,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s sver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, syncRole[pPeersStatus->role], - pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); + sDebug("%s, status msg is sent, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sstatus:%s sver:%" PRIu64 + ", ack:%d tranId:%u type:%s pfd:%d", + pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], + syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, + statusType[pPeersStatus->type], pPeer->peerFd); } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); @@ -1048,7 +1066,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { - sDebug("%s, connection to peer server is setup", pPeer->id); + sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); @@ -1069,7 +1087,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { sDebug("%s, check peer connection", pPeer->id); syncSetupPeerConnection(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1135,7 +1153,8 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer->syncFd = connFd; syncCreateRestoreDataThread(pPeer); } else { - sDebug("%s, TCP connection is already up, close one", pPeer->id); + sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd, + pPeer->syncFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); @@ -1145,7 +1164,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncProcessBrokenLink(void *param) { @@ -1156,14 +1175,14 @@ static void syncProcessBrokenLink(void *param) { if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); - sDebug("%s, TCP link is broken since %s", pPeer->id, strerror(errno)); + sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; if (syncDecPeerRef(pPeer) != 0) { syncRestartConnection(pPeer); } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, pNode->rid); } @@ -1239,10 +1258,13 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { if (index == pNode->selfIndex) continue; SSyncPeer *pPeer = pNode->peerInfo[index]; - if (pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; - if (pPeer->sstatus > TAOS_SYNC_STATUS_INIT || nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; + if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; + if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; + sDebug("%s, check roles since self:%s sstatus:%s, peer:%s sstatus:%s", pPeer->id, syncRole[pPeer->role], + syncStatus[pPeer->sstatus], syncRole[nodeRole], syncStatus[nodeSStatus]); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); + break; } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); @@ -1271,7 +1293,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { } syncRemoveConfirmedFwdInfo(pNode); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); @@ -1339,7 +1361,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); return code; } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index b31ec5f7c7..466c39060a 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -267,7 +267,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; - sDebug("%s, start to restore file", pPeer->id); + sDebug("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); @@ -291,7 +291,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } nodeSStatus = TAOS_SYNC_STATUS_CACHE; - sDebug("%s, start to insert buffered points", pPeer->id); + sDebug("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { sError("%s, failed to insert buffered points", pPeer->id); return -1; @@ -327,6 +327,8 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; + sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index b6dacaa262..04d4ce32cb 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -318,6 +318,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) if (((event & IN_MODIFY) == 0) || once) { if (fversion == 0) { pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt + sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); fversion = nodeVersion; // must read data to fversion } } @@ -416,8 +417,9 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { - sInfo("%s, wal retrieve is finished", pPeer->id); pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; + sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); @@ -445,7 +447,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sInfo("%s, start to retrieve file", pPeer->id); + sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (syncRetrieveFile(pPeer) < 0) { sError("%s, failed to retrieve file", pPeer->id); return -1; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 1be79b7bbd..2543c81708 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -107,7 +107,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { while (nleft > 0) { nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) + if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) continue; else return -1; @@ -133,7 +133,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { if (nread == 0) { break; } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { continue; } else { return -1; From dd9228e06733c5a775e953e01b1bd12e1711a5e4 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 18:51:13 +0800 Subject: [PATCH 33/75] [TD-1859]: restful implementation for taos-jdbc --- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 9 +- .../taosdata/jdbc/rs/RestfulConnection.java | 41 ++++--- .../com/taosdata/jdbc/rs/RestfulDriver.java | 2 +- .../taosdata/jdbc/rs/RestfulStatement.java | 71 ++++++++++-- .../jdbc/utils/SqlSyntaxValidator.java | 29 ++++- .../taosdata/jdbc/rs/RestfulDriverTest.java | 23 ---- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 108 ++++++++++++++++++ .../jdbc/utils/SqlSyntaxValidatorTest.java | 29 +++++ 9 files changed, 263 insertions(+), 51 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 51db837c7b..46de54ffb2 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.10 + 2.0.13 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index e7124a0599..3d1f402435 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.8 + 2.0.13 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -112,6 +112,13 @@ maven-surefire-plugin 2.12.4 + + **/*Test.java + + + **/BatchInsertTest.java + **/FailOverTest.java + true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index b82efca3ef..6b0937a9b7 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -15,7 +15,6 @@ public class RestfulConnection implements Connection { private final String database; private final String url; - public RestfulConnection(String host, String port, Properties props, String database, String url) { this.host = host; this.port = Integer.parseInt(port); @@ -28,7 +27,7 @@ public class RestfulConnection implements Connection { public Statement createStatement() throws SQLException { if (isClosed()) throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed.")); - return new RestfulStatement(this, this.database); + return new RestfulStatement(this, database); } @Override @@ -104,22 +103,28 @@ public class RestfulConnection implements Connection { @Override public void setTransactionIsolation(int level) throws SQLException { - + //transaction is not supported + throw new SQLFeatureNotSupportedException("transactions are not supported"); } + /** + * + */ @Override public int getTransactionIsolation() throws SQLException { - return 0; + //Connection.TRANSACTION_NONE specifies that transactions are not supported. + return Connection.TRANSACTION_NONE; } @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not implemented return null; } @Override public void clearWarnings() throws SQLException { - + throw new SQLFeatureNotSupportedException("clearWarnings not supported."); } @Override @@ -209,22 +214,26 @@ public class RestfulConnection implements Connection { @Override public Clob createClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Blob createBlob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public NClob createNClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public SQLXML createSQLXML() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -254,12 +263,14 @@ public class RestfulConnection implements Connection { @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -289,12 +300,16 @@ public class RestfulConnection implements Connection { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } public String getHost() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index a07d6eb2b6..9e87cfa680 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -35,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver { Properties props = parseURL(url, info); String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041"); - String database = props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME); + String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null; String loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 20510f0135..30b56638d8 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.taosdata.jdbc.TSDBConstants; import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.utils.SqlSyntaxValidator; import java.sql.*; import java.util.Arrays; @@ -11,19 +12,23 @@ import java.util.List; public class RestfulStatement implements Statement { - private final String catalog; + private boolean closed; + private String database; private final RestfulConnection conn; - public RestfulStatement(RestfulConnection c, String catalog) { + public RestfulStatement(RestfulConnection c, String database) { this.conn = c; - this.catalog = catalog; + this.database = database; } @Override public ResultSet executeQuery(String sql) throws SQLException { + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isSelectSql(sql)) + throw new SQLException("not a select sql for executeQuery: " + sql); - final String url = "http://" + conn.getHost() + ":"+conn.getPort()+"/rest/sql"; - + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; String result = HttpClientPoolUtil.execute(url, sql); String fields = ""; List words = Arrays.asList(sql.split(" ")); @@ -65,12 +70,29 @@ public class RestfulStatement implements Statement { @Override public int executeUpdate(String sql) throws SQLException { - return 0; + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw new SQLException("not a valid sql for executeUpdate: " + sql); + + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return Integer.parseInt(jsonObject.getString("rows")); } @Override public void close() throws SQLException { - + this.closed = true; } @Override @@ -115,6 +137,7 @@ public class RestfulStatement implements Statement { @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not Implemented return null; } @@ -130,7 +153,29 @@ public class RestfulStatement implements Statement { @Override public boolean execute(String sql) throws SQLException { - return false; + if (isClosed()) { + throw new SQLException("Invalid method call on a closed statement."); + } + //如果执行了use操作应该将当前Statement的catalog设置为新的database + if (SqlSyntaxValidator.isUseSql(sql)) { + this.database = sql.trim().replace("use", "").trim(); + } + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + // use database + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + // execute sql + String result = HttpClientPoolUtil.execute(url, sql); + // parse result + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return true; } @Override @@ -245,7 +290,7 @@ public class RestfulStatement implements Statement { @Override public boolean isClosed() throws SQLException { - return false; + return closed; } @Override @@ -270,11 +315,15 @@ public class RestfulStatement implements Statement { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 066dfad5d5..388c3978be 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -22,6 +22,9 @@ import java.sql.SQLException; public class SqlSyntaxValidator { + private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"}; + private static final String[] querySQL = {"select"}; + private TSDBConnection tsdbConnection; public SqlSyntaxValidator(Connection connection) { @@ -34,7 +37,7 @@ public class SqlSyntaxValidator { if (tsdbConnection == null || tsdbConnection.isClosed()) { throw new SQLException("invalid connection"); } else { - TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); + TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); if (jniConnector == null) { throw new SQLException("jniConnector is null"); } else { @@ -43,4 +46,28 @@ public class SqlSyntaxValidator { } return res; } + + public static boolean isValidForExecuteUpdate(String sql) { + for (String prefix : updateSQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isUseSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + } + + public static boolean isUpdateSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[1]); + } + + public static boolean isInsertSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[0]); + } + + public static boolean isSelectSql(String sql) { + return sql.trim().toLowerCase().startsWith(querySQL[0]); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index 768eceab8e..d07a6a2179 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -7,29 +7,6 @@ import java.sql.*; public class RestfulDriverTest { - @Test - public void testCase001() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/?user=root&password=taosdata"); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("select * from log.log"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String column = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - System.out.print(column + ":" + value + "\t"); - } - System.out.println(); - } - statement.close(); - connection.close(); - } catch (SQLException | ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test public void connect() { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java new file mode 100644 index 0000000000..d13475b96d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -0,0 +1,108 @@ +package com.taosdata.jdbc.rs; + +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Random; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class RestfulJDBCTest { + + private Connection connection; + + @Before + public void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata"); + } + + @After + public void after() throws SQLException { + if (connection != null) + connection.close(); + } + + + /** + * 查询所有log.log + **/ + @Test + public void testCase001() { + try { + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from log.log"); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String column = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.print(column + ":" + value + "\t"); + } + System.out.println(); + } + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create database + */ + @Test + public void testCase002() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create super table + ***/ + @Test + public void testCase003() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase004() { + try (Statement stmt = connection.createStatement()) { + for (int i = 1; i <= 100; i++) { + stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')"); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private Random random = new Random(System.currentTimeMillis()); + + @Test + public void testCase005() { + try (Statement stmt = connection.createStatement()) { + int rows = 0; + for (int i = 0; i < 10; i++) { + for (int j = 1; j <= 100; j++) { + long currentTimeMillis = System.currentTimeMillis(); + int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")"); + Assert.assertEquals(1, affectRows); + rows += affectRows; + } + } + Assert.assertEquals(1000, rows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java new file mode 100644 index 0000000000..25c6c4913d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -0,0 +1,29 @@ +package com.taosdata.jdbc.utils; + +import com.sun.source.tree.AssertTree; +import org.junit.Assert; +import org.junit.Test; + +public class SqlSyntaxValidatorTest { + + @Test + public void validateSqlSyntax() { + } + + @Test + public void isSelectSQL() { + Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather ")); + Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)")); + } + + @Test + public void isUseSQL() { + Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); + } +} \ No newline at end of file From a9e43baa3f8841d468efe1fedb5beda78beeb49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:08:03 +0800 Subject: [PATCH 34/75] change --- .../java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index 25c6c4913d..fb570e16c4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc.utils; -import com.sun.source.tree.AssertTree; import org.junit.Assert; import org.junit.Test; From 0481d9836f33f6e0a1e26e85af7a4bdc5a8ea49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:12:07 +0800 Subject: [PATCH 35/75] change --- src/connector/jdbc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index c565853ab0..701a39b209 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) From dbdd70aca16a1415efab789d3d44c1af7b1e5d7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 11:15:08 +0000 Subject: [PATCH 36/75] TD-2196 --- src/sync/src/syncRestore.c | 31 ++++++++++++++++++++++++------- src/sync/src/syncRetrieve.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 466c39060a..69da1d71fa 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -65,7 +65,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { while (1) { // read file info int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno)); + break; + } // if no more file from master, break; if (minfo.name[0] == 0 || minfo.magic == 0) { @@ -92,8 +95,11 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0; // send file ack - ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret < 0) break; + ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); + if (ret < 0) { + sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); + break; + } // if sync is not required, continue if (fileAck.sync == 0) { @@ -108,14 +114,17 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { - sError("%s, failed to open file:%s", pPeer->id, name); + sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); break; } ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size); fsync(dfd); close(dfd); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); + break; + } fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); @@ -125,6 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; + sDebug("%s, file changed while restore file", pPeer->id); } if (code < 0) { @@ -146,15 +156,22 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); + break; + } if (pHead->len == 0) { + sDebug("%s, wal is synced over", pPeer->id); code = 0; break; } // wal sync over ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); + break; + } sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 04d4ce32cb..58d09d080e 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -114,7 +114,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // if no file anymore, break if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { @@ -124,8 +127,11 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { } // wait for the ack from peer - ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret < 0) break; + ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); + if (ret < 0) { + sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -134,7 +140,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); // add the file into watch list - if (syncAddIntoWatchList(pPeer, name) < 0) break; + if (syncAddIntoWatchList(pPeer, name) < 0) { + sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // if sync is not required, continue if (fileAck.sync == 0) { @@ -145,21 +154,30 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file to peer int32_t sfd = open(name, O_RDONLY); - if (sfd < 0) break; + if (sfd < 0) { + sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pPeer) != 0) break; + if (syncAreFilesModified(pPeer) != 0) { + sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } } if (code < 0) { - sError("%s, failed to retrieve file since %s", pPeer->id, strerror(errno)); + sError("%s, failed to retrieve file", pPeer->id); } return code; From 5e91e8e307c232e9eaf47615499c6a7d0dc5a936 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:28:45 +0800 Subject: [PATCH 37/75] change --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9bbcc2cf40..4886e79e73 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") From aae38f8b248dcd83e12c902a73303694e9ddc170 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 20:08:45 +0800 Subject: [PATCH 38/75] add log --- src/sync/src/syncMain.c | 6 ++---- src/sync/src/syncRestore.c | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index a68f0b7bbb..8d3997a5d6 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -937,8 +937,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sver:%" PRIu64 - ", ack:%d tranId:%u type:%s pfd:%d", + sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); @@ -1029,8 +1028,7 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sstatus:%s sver:%" PRIu64 - ", ack:%d tranId:%u type:%s pfd:%d", + sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 69da1d71fa..2ca4b5424e 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -86,7 +86,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // check the file info sinfo = minfo; - sDebug("%s, get file info:%s", pPeer->id, minfo.name); + sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); From 7c9ce886942275ffbee67d15b80eff42954fe109 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 09:18:55 +0800 Subject: [PATCH 39/75] change --- src/connector/jdbc/deploy-pom.xml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 46de54ffb2..1dc2625e62 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -49,17 +49,29 @@ - - org.apache.commons - commons-lang3 - 3.5 - junit junit 4.13 test + + + + org.apache.httpcomponents + httpclient + 4.5.8 + + + org.apache.commons + commons-lang3 + 3.9 + + + com.alibaba + fastjson + 1.2.58 + From 99815378bf29bc0bda6f1fc13066f09b1dff2227 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 14:54:30 +0800 Subject: [PATCH 40/75] TD-2086 --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 18 +++++------------- src/sync/src/syncRestore.c | 10 ++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7dbb1985d4..215ae7a58d 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -41,6 +41,7 @@ typedef enum { #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) #define SYNC_FWD_TIMER 300 #define SYNC_ROLE_TIMER 10000 +#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3 #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8d3997a5d6..d5fc79b635 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -626,17 +626,9 @@ static void syncChooseMaster(SSyncNode *pNode) { sInfo("vgId:%d, start to work as master", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; -#if 0 - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == index) continue; - pPeer = pNode->peerInfo[i]; - if (pPeer->version == nodeVersion) { - pPeer->role = TAOS_SYNC_ROLE_SLAVE; - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, it shall work as slave", pPeer->id); - } - } -#endif + // Wait for other nodes to receive status to avoid version inconsistency + taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER); + syncResetFlowCtrl(pNode); (*pNode->notifyRole)(pNode->vgId, nodeRole); } else { @@ -761,7 +753,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new sDebug("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inconsistent", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); } } @@ -1124,7 +1116,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } int32_t vgId = firstPkt.syncHead.vgId; - SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t)); + SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); taosCloseSocket(connFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 2ca4b5424e..db3c6b318d 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -147,12 +147,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { static int32_t syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; + uint64_t lastVer = 0; - void *buffer = calloc(SYNC_MAX_SIZE, 1); // size for one record - if (buffer == NULL) return -1; - - SWalHead *pHead = (SWalHead *)buffer; - uint64_t lastVer = 0; + SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1); // size for one record + if (pHead == NULL) return -1; while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); @@ -188,7 +186,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno)); } - free(buffer); + free(pHead); return code; } From f9c34c937068a76cde87aa719fb177749846404d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 24 Nov 2020 17:27:42 +0800 Subject: [PATCH 41/75] [TD-2125][TD-2173]python connector --- src/connector/python/linux/python2/setup.py | 2 +- .../python/linux/python2/taos/cinterface.py | 115 ++++++++++++---- .../python/linux/python2/taos/cursor.py | 20 ++- src/connector/python/linux/python3/setup.py | 2 +- .../python/linux/python3/taos/cinterface.py | 129 ++++++++++++------ .../python/linux/python3/taos/cursor.py | 26 ++-- src/connector/python/windows/python2/setup.py | 2 +- .../python/windows/python2/taos/cinterface.py | 118 ++++++++++++---- .../python/windows/python2/taos/cursor.py | 22 ++- src/connector/python/windows/python3/setup.py | 2 +- .../python/windows/python3/taos/cinterface.py | 107 ++++++++++++--- .../python/windows/python3/taos/cursor.py | 22 ++- 12 files changed, 428 insertions(+), 139 deletions(-) diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 2e4f80b8f0..b3daa98bdc 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 269326535c..6f0435722f 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +270,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +454,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +471,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index bc6670ca77..eee5e39488 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -49,7 +49,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -190,6 +190,23 @@ class TDengineCursor(object): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -203,7 +220,6 @@ class TDengineCursor(object): for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def nextset(self): """ """ diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index 03a49fc1c5..f49ebe2b6d 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index be5b99d8c1..bbf5a0c714 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -253,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,29 +351,26 @@ class CTaosInterface(object): return fields - # @staticmethod - # def fetchBlock(result, fields): - # pblock = ctypes.c_void_p(0) - # num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - # result, ctypes.byref(pblock)) - # if num_of_rows == 0: - # return None, 0 - - # isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) - # blocks = [None] * len(fields) - # fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - # fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] - # for i in range(len(fields)): - # data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - - # if fields[i]['type'] not in _CONVERT_FUNC: - # raise DatabaseError("Invalid data type returned from database") - # print('====================',fieldLen[i]) - # blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) - - # return blocks, abs(num_of_rows) @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -350,6 +390,7 @@ class CTaosInterface(object): else: return None, 0 return blocks, abs(num_of_rows) + @staticmethod def freeResult(result): CTaosInterface.libtaos.taos_free_result(result) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index eb10bed485..d8184668c8 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -5,7 +5,6 @@ import threading # querySeqNum = 0 - class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -51,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -196,18 +195,13 @@ class TDengineCursor(object): def fetchall(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ - # if threading.get_ident() != self._threadId: - # info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: raise ProgrammingError(CTaosInterface.errStr(self._result), errno) @@ -218,6 +212,22 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) def nextset(self): """ """ diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index fd82a55650..34cb7a04d7 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index 084d38e41c..d4bf1a0350 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -225,9 +268,10 @@ class CTaosInterface(object): if connection.value == None: print('connect to TDengine failed') + raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -236,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -252,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -309,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +455,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +472,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 35846cbe11..71651afee1 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -50,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -143,7 +143,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -178,7 +196,7 @@ class TDengineCursor(object): self._description = None self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + CTaosInterface.freeResult(self._result) self._result = None self._fields = None self._block = None diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index 9abdace5a9..c09644d330 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index 68adf191c9..fb1b65fc15 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -104,16 +104,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) + return res _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +160,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +272,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +282,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +355,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index b58b494cad..b813bba357 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -51,7 +51,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -144,7 +144,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") - + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: From 4d789b0871a47fcfd6479ec4ffc7d35450ac051e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 18:05:18 +0800 Subject: [PATCH 42/75] TD-2211 --- src/sync/inc/syncInt.h | 19 ++++++++++++------- src/sync/src/syncMain.c | 13 +++++++------ src/sync/src/syncRestore.c | 20 +++++++++++++++----- src/sync/src/syncRetrieve.c | 22 +++++++++++++++++++--- src/sync/src/taosTcpPool.c | 3 +++ 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 215ae7a58d..05b7adc5f4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -28,13 +28,14 @@ extern "C" { #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} typedef enum { - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_FORWARD = 2, - TAOS_SMSG_FORWARD_RSP = 3, - TAOS_SMSG_SYNC_REQ = 4, - TAOS_SMSG_SYNC_RSP = 5, - TAOS_SMSG_SYNC_MUST = 6, - TAOS_SMSG_STATUS = 7 + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_FORWARD = 2, + TAOS_SMSG_FORWARD_RSP = 3, + TAOS_SMSG_SYNC_REQ = 4, + TAOS_SMSG_SYNC_RSP = 5, + TAOS_SMSG_SYNC_MUST = 6, + TAOS_SMSG_STATUS = 7, + TAOS_SMSG_SYNC_DATA_RSP = 8, } ESyncMsgType; #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) @@ -65,6 +66,10 @@ typedef struct { int32_t sourceId; // only for arbitrator } SFirstPkt; +typedef struct { + int8_t sync; +} SFirstPktRsp; + typedef struct { int8_t role; uint64_t version; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d5fc79b635..68bc02cb16 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1313,6 +1313,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } // always update version + sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); nodeVersion = pWalHead->version; if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; @@ -1320,10 +1322,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle // only pkt from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; - sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, - syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); - - // a hacker way to improve the performance + // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); pSyncHead->type = TAOS_SMSG_FORWARD; pSyncHead->pversion = 0; @@ -1344,9 +1343,11 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, pPeer->role, + syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); } else { - sError("%s, failed to forward, hver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); + sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, pPeer->role, + syncStatus[pPeer->sstatus], pWalHead->version, retLen); syncRestartConnection(pPeer); } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index db3c6b318d..ed6b63c92d 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -231,10 +231,13 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; - - if (pRecv == NULL) return -1; int32_t len = pHead->len + sizeof(SWalHead); + if (pRecv == NULL) { + sError("%s, recv buffer is not create yet", pPeer->id); + return -1; + } + if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) { memcpy(pRecv->offset, pHead, len); pRecv->offset += len; @@ -282,7 +285,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; - sDebug("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + SFirstPktRsp firstPktRsp = {.sync = 1}; + if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { + sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + return -1; + } + + sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); @@ -299,14 +309,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeVersion = fversion; - sDebug("%s, start to restore wal", pPeer->id); + sInfo("%s, start to restore wal", pPeer->id); if (syncRestoreWal(pPeer) < 0) { sError("%s, failed to restore wal", pPeer->id); return -1; } nodeSStatus = TAOS_SYNC_STATUS_CACHE; - sDebug("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { sError("%s, failed to insert buffered points", pPeer->id); return -1; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 58d09d080e..348f91820b 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -448,7 +448,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { return code; } -static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; @@ -458,8 +458,24 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - if (taosWriteMsg(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { - sError("%s, failed to send syncCmd", pPeer->id); + if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) { + sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno)); + return -1; + } + + SFirstPktRsp firstPktRsp; + if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { + sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + return -1; + } + + return 0; +} + +static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { + sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + if (syncRetrieveFirstPkt(pPeer) < 0) { + sError("%s, failed to start retrieve", pPeer->id); return -1; } diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index d1d9815f4a..eb05cf7c6f 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -203,16 +203,19 @@ static void *taosProcessTcpData(void *param) { assert(pConn); if (events[i].events & EPOLLERR) { + sDebug("conn is broken since EPOLLERR"); taosProcessBrokenLink(pConn); continue; } if (events[i].events & EPOLLHUP) { + sDebug("conn is broken since EPOLLHUP"); taosProcessBrokenLink(pConn); continue; } if (events[i].events & EPOLLRDHUP) { + sDebug("conn is broken since EPOLLRDHUP"); taosProcessBrokenLink(pConn); continue; } From d14e50ea5aa813642a6bc1f8886d5715909ce500 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 18:06:31 +0800 Subject: [PATCH 43/75] compile error --- src/sync/src/syncMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 68bc02cb16..bc3803b732 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1343,10 +1343,10 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, pPeer->role, + sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); } else { - sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, pPeer->role, + sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pWalHead->version, retLen); syncRestartConnection(pPeer); } From 19dedc226accda1655633aa1dd66d5ed47282445 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 18:34:02 +0800 Subject: [PATCH 44/75] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java new file mode 100644 index 0000000000..8a3dbb2bc1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -0,0 +1,23 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class AppMemoryLeakTest { + + @Test + public void testAppMemoryLeak() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + + while (true) { + DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + } + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } +} From a988483a864920241f64e62a00af4a08d17baa09 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 22:36:25 +0800 Subject: [PATCH 45/75] TD-1795 --- src/inc/taoserror.h | 2 +- src/mnode/src/mnodeSdb.c | 6 ++++++ src/sync/src/syncMain.c | 2 +- tests/script/tmp/mnodes.sim | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ff91989e5f..f1e736130c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "CPU cores TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sync Configuration") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") @@ -365,7 +366,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") - TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 36e1a1eb2b..536fdaaa11 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -261,6 +261,12 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj}; sdbDeleteRow(&row); } + + // Drop database/stable may take a long time and cause a timeout, so confirm is not enforced + if (action == SDB_ACTION_DELETE && pRow->code == TSDB_CODE_SYN_CONFIRM_EXPIRED) { + sdbDebug("vgId:1, confirm is not enforced while perform drop operation, set it success"); + pRow->code = TSDB_CODE_SUCCESS; + } } FORCE_INLINE diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index bc3803b732..d2d6d2d7fa 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1279,7 +1279,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, pFwdInfo->version, time, pFwdInfo->time); - syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED); } syncRemoveConfirmedFwdInfo(pNode); diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index 48dbc19cb2..de02ae741b 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -20,6 +20,10 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 + system sh/cfg.sh -n dnode1 -c replica -v 3 system sh/cfg.sh -n dnode2 -c replica -v 3 system sh/cfg.sh -n dnode3 -c replica -v 3 From 97434ac094328ebf1aa40b953d43f3c026cd1558 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 10:53:46 +0800 Subject: [PATCH 46/75] TD-1795 --- src/mnode/src/mnodeSdb.c | 70 ++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 536fdaaa11..e5fda26687 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -107,7 +107,7 @@ static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); -static int32_t sdbWriteWalToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); +static int32_t sdbWriteFwdToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); static void sdbFreeFromQueue(SSdbRow *pRow); static void * sdbWorkerFp(void *pWorker); @@ -261,12 +261,6 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj}; sdbDeleteRow(&row); } - - // Drop database/stable may take a long time and cause a timeout, so confirm is not enforced - if (action == SDB_ACTION_DELETE && pRow->code == TSDB_CODE_SYN_CONFIRM_EXPIRED) { - sdbDebug("vgId:1, confirm is not enforced while perform drop operation, set it success"); - pRow->code = TSDB_CODE_SUCCESS; - } } FORCE_INLINE @@ -378,7 +372,7 @@ void sdbUpdateSync(void *pMnodes) { sprintf(syncInfo.path, "%s", tsMnodeDir); syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.writeToCache = sdbWriteWalToQueue; + syncInfo.writeToCache = sdbWriteFwdToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; tsSdbMgmt.cfg = syncCfg; @@ -565,7 +559,36 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) { return TSDB_CODE_SUCCESS; } -static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { +static int32_t sdbPerformInsertAction(SWalHead *pHead, SSdbTable *pTable) { + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + (*pTable->fpDecode)(&row); + return sdbInsertHash(pTable, &row); +} + +static int32_t sdbPerformDeleteAction(SWalHead *pHead, SSdbTable *pTable) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, + sdbGetKeyStr(pTable, pHead->cont)); + return TSDB_CODE_SUCCESS; + } + SSdbRow row = {.pTable = pTable, .pObj = pObj}; + return sdbDeleteHash(pTable, &row); +} + +static int32_t sdbPerformUpdateAction(SWalHead *pHead, SSdbTable *pTable) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, + sdbGetKeyStr(pTable, pHead->cont)); + return TSDB_CODE_SUCCESS; + } + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + (*pTable->fpDecode)(&row); + return sdbUpdateHash(pTable, &row); +} + +static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { SSdbRow *pRow = wparam; SWalHead *pHead = hparam; int32_t tableId = pHead->msgType / 10; @@ -574,6 +597,8 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); + if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable); + pthread_mutex_lock(&tsSdbMgmt.mutex); if (pHead->version == 0) { @@ -633,28 +658,17 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus // from wal or forward msg, row not created, should add into hash if (action == SDB_ACTION_INSERT) { - SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&row); - return sdbInsertHash(pTable, &row); + return sdbPerformInsertAction(pHead, pTable); } else if (action == SDB_ACTION_DELETE) { - void *pObj = sdbGetRowMeta(pTable, pHead->cont); - if (pObj == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, - sdbGetKeyStr(pTable, pHead->cont)); + if (qtype == TAOS_QTYPE_FWD) { + // Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue + sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused); return TSDB_CODE_SUCCESS; + } else { + return sdbPerformDeleteAction(pHead, pTable); } - SSdbRow row = {.pTable = pTable, .pObj = pObj}; - return sdbDeleteHash(pTable, &row); } else if (action == SDB_ACTION_UPDATE) { - void *pObj = sdbGetRowMeta(pTable, pHead->cont); - if (pObj == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, - sdbGetKeyStr(pTable, pHead->cont)); - return TSDB_CODE_SUCCESS; - } - SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&row); - return sdbUpdateHash(pTable, &row); + return sdbPerformUpdateAction(pHead, pTable); } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } @@ -972,7 +986,7 @@ static void sdbFreeFromQueue(SSdbRow *pRow) { taosFreeQitem(pRow); } -static int32_t sdbWriteWalToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { +static int32_t sdbWriteFwdToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { SWalHead *pHead = wparam; int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len; From 0e88426c7f864a678095ab761870d55a49cb7eb7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 11:19:15 +0800 Subject: [PATCH 47/75] [TD-2152] add test case for join query --- tests/pytest/insert/restfulInsert.py | 7 +++++-- tests/pytest/query/filterOtherTypes.py | 18 ++++++------------ tests/pytest/query/queryJoin.py | 2 ++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index 9fa1f33a24..c39d1cbe48 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -55,9 +55,12 @@ class RestfulInsert: def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) tablesPerThread = int (self.numOfTables / self.numOfThreads) + + count = 0 while True: i = 0 - start = self.ts + start = self.ts + count * self.batchSize + count = count + 1 for i in range(tablesPerThread): tableID = i + threadID * tablesPerThread @@ -65,7 +68,7 @@ class RestfulInsert: data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) values = [] for k in range(self.batchSize): - values.append("(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) + values.append("(%d, %d, %d, %d)" % (start + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) if(self.outOfOrder == False): for k in range(len(values)): diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 85d5a67bef..f80552138d 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -376,11 +376,9 @@ class TDTestCase: tdSql.execute("insert into t1 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t2 values(1538548685000, 4) (1538548685001, 5) (1538548685002, 6)") - tdSql.query("select * from t1 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t1 where tag1 like '%g'") - tdSql.query("select * from t2 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t2 where tag1 like '%g'") tdSql.query("select * from meters where tag1 like '%g'") tdSql.checkRows(6) @@ -396,20 +394,16 @@ class TDTestCase: tdSql.execute("insert into t5 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t6 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") - tdSql.query("select * from t3 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t3 where tag1 like '%京'") - tdSql.query("select * from t4 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t4 where tag1 like '%京'") tdSql.query("select * from meters1 where tag1 like '%京'") tdSql.checkRows(6) - tdSql.query("select * from t5 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t5 where tag1 like '%g'") - tdSql.query("select * from t6 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t6 where tag1 like '%g'") tdSql.query("select * from meters1 where tag1 like '%g'") tdSql.checkRows(6) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 57b7e8868d..59e01615b4 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -175,6 +175,8 @@ class TDTestCase: tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select join_mt1.c1,join_mt0.c1 from join_mt1,join_mt0 where join_mt1.ts = join_mt0.ts and join_mt1.t1 = join_mt0.t1 order by t") + def stop(self): tdSql.close() From 092c9de4aa994bd1d15f23e02591407a6bd25edc Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 25 Nov 2020 15:14:01 +0800 Subject: [PATCH 48/75] [TD-2218]add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2218.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/pytest/query/bug2218.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..8977d96df4 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -158,6 +158,7 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py +python3 ./test.py -f query/bug2218.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug2218.py b/tests/pytest/query/bug2218.py new file mode 100644 index 0000000000..bb92e5d9ce --- /dev/null +++ b/tests/pytest/query/bug2218.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool)") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d)" % + (t0+i,i%100,i/2,i%100,i%100,i%100,i*1.0,i%2)) + print("==========step2") + print("test col*1*1 desc ") + tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts desc limit 2') + tdSql.checkData(0,0,99) + tdSql.checkData(0,1,0.0) + tdSql.checkData(0,2,0.0) + tdSql.checkData(0,3,0.0) + tdSql.checkData(0,4,0.0) + tdSql.checkData(0,5,0.0) + tdSql.checkData(0,6,0.0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 3e7c81eafbcd57880ea2f4ac799858b753a02391 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:03:43 +0800 Subject: [PATCH 49/75] [TD-2126] add test case for taosdump --- tests/pytest/tools/taosdumpTest.py | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/pytest/tools/taosdumpTest.py diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py new file mode 100644 index 0000000000..534a477b34 --- /dev/null +++ b/tests/pytest/tools/taosdumpTest.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + self.numberOfTables = 10000 + self.numberOfRecords = 100 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") + tdSql.execute("create table t1 using st tags(1, 'beijing')") + sql = "insert into t1 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + tdSql.execute("create table t2 using st tags(2, 'shanghai')") + sql = "insert into t2 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + os.system("taosdump --databases db -o /tmp") + + tdSql.execute("drop database db") + tdSql.query("show databases") + tdSql.checkRows(0) + + os.system("taosdump -i /tmp") + + tdSql.query("show databases") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'db') + + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'st') + + tdSql.query("show tables") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 't2') + tdSql.checkData(1, 0, 't1') + + tdSql.query("select * from t1") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + tdSql.query("select * from t2") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 431fc93950be86183926fa2503529810d93ec7fd Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:15:22 +0800 Subject: [PATCH 50/75] add test cases into fulltest.sh --- tests/pytest/fulltest.sh | 4 +++- tests/pytest/tools/{lowa.py => lowaTest.py} | 0 tests/pytest/tools/{taosdemo.py => taosdemoTest.py} | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename tests/pytest/tools/{lowa.py => lowaTest.py} (100%) rename tests/pytest/tools/{taosdemo.py => taosdemoTest.py} (100%) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..89b5b0f125 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -205,7 +205,9 @@ python3 test.py -f query/queryInterval.py python3 test.py -f query/queryFillTest.py # tools -python3 test.py -f tools/taosdemo.py +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdumpTest.py +python3 test.py -f tools/lowaTest.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/tools/lowa.py b/tests/pytest/tools/lowaTest.py similarity index 100% rename from tests/pytest/tools/lowa.py rename to tests/pytest/tools/lowaTest.py diff --git a/tests/pytest/tools/taosdemo.py b/tests/pytest/tools/taosdemoTest.py similarity index 100% rename from tests/pytest/tools/taosdemo.py rename to tests/pytest/tools/taosdemoTest.py From d1cd620696bbfe7f88efe04caf060d1688301d7b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 16:29:02 +0800 Subject: [PATCH 51/75] TD-1455 TD-2196 --- src/dnode/src/dnodeVWrite.c | 4 ++-- src/inc/tsync.h | 6 +++--- src/inc/vnode.h | 6 ++++-- src/sync/src/syncRetrieve.c | 5 ++--- src/vnode/inc/vnodeInt.h | 2 +- src/vnode/src/vnodeMain.c | 10 ++++----- src/vnode/src/vnodeWrite.c | 42 ++++++++++++++++++++++++++++++++----- 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 959eb3c9c5..6d4b50ee54 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -176,7 +176,7 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) { if (count <= 1) return; SRpcMsg rpcRsp = { - .handle = pWrite->rpcHandle, + .handle = pWrite->rpcMsg.handle, .pCont = pWrite->rspRet.rsp, .contLen = pWrite->rspRet.len, .code = pWrite->code, @@ -206,7 +206,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite, - pWrite->rpcAhandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); + pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 77a3b36e73..398e1bf97c 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -23,7 +23,7 @@ extern "C" { #define TAOS_SYNC_MAX_REPLICA 5 #define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF -typedef enum _TAOS_SYNC_ROLE { +typedef enum { TAOS_SYNC_ROLE_OFFLINE = 0, TAOS_SYNC_ROLE_UNSYNCED = 1, TAOS_SYNC_ROLE_SYNCING = 2, @@ -31,7 +31,7 @@ typedef enum _TAOS_SYNC_ROLE { TAOS_SYNC_ROLE_MASTER = 4 } ESyncRole; -typedef enum _TAOS_SYNC_STATUS { +typedef enum { TAOS_SYNC_STATUS_INIT = 0, TAOS_SYNC_STATUS_START = 1, TAOS_SYNC_STATUS_FILE = 2, @@ -80,7 +80,7 @@ typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); typedef void (*FNotifyRole)(int32_t vgId, int8_t role); // if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t mseconds); +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 4e8389498b..5f643295d6 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "trpc.h" #include "twal.h" typedef enum _VN_STATUS { @@ -51,8 +52,9 @@ typedef struct { typedef struct { int32_t code; int32_t processedCount; - void * rpcHandle; - void * rpcAhandle; + int32_t qtype; + void * pVnode; + SRpcMsg rpcMsg; SRspRet rspRet; char reserveForSync[16]; SWalHead pHead[]; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 348f91820b..060badba9d 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -504,6 +504,8 @@ void *syncRetrieveData(void *param) { SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); + if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); + pPeer->fileChanged = 0; pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { @@ -520,10 +522,7 @@ void *syncRetrieveData(void *param) { } if (pPeer->fileChanged) { - // if file is changed 3 times continuously, start flow control pPeer->numOfRetrieves++; - if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->vgId, 4 << (pPeer->numOfRetrieves - 2)); } else { pPeer->numOfRetrieves = 0; if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 021831a644..cf9a1baa80 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -39,7 +39,7 @@ typedef struct { int32_t refCount; // reference count int32_t queuedWMsg; int32_t queuedRMsg; - int32_t delayMs; + int32_t flowctlLevel; int8_t status; int8_t role; int8_t accessState; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b94fea52bd..1ee968e7cb 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -34,7 +34,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); static void vnodeNotifyRole(int32_t vgId, int8_t role); -static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds); +static void vnodeCtrlFlow(int32_t vgId, int32_t level); static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); @@ -677,17 +677,15 @@ static void vnodeNotifyRole(int32_t vgId, int8_t role) { vnodeRelease(pVnode); } -static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds) { +static void vnodeCtrlFlow(int32_t vgId, int32_t level) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while ctrl flow", vgId); return; } - if (pVnode->delayMs != mseconds) { - pVnode->delayMs = mseconds; - vDebug("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); - } + pVnode->flowctlLevel = level; + vDebug("vgId:%d, set flowctl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index a0227d84ba..cada188591 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -22,14 +22,17 @@ #include "tsdb.h" #include "twal.h" #include "tsync.h" +#include "ttimer.h" #include "tdataformat.h" #include "vnode.h" #include "vnodeInt.h" #include "syncInt.h" #include "tcq.h" +#include "dnode.h" #define MAX_QUEUED_MSG_NUM 10000 +extern void * tsDnodeTmr; static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *); static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); @@ -47,6 +50,32 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } +static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { + SVWriteMsg *pWrite = param; + SVnodeObj * pVnode = pWrite->pVnode; + + int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, &pWrite->rpcMsg); + if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { + vDebug("vgId:%d, failed to reprocess msg after perform flowctl since %s", pVnode->vgId, tstrerror(code)); + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } + + tfree(pWrite); + vnodeRelease(pWrite->pVnode); +} + +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + if (pVnode->flowctlLevel <= 0) return 0; + + int32_t ms = pVnode->flowctlLevel * 5; + void * unUsed = NULL; + taosTmrReset(vnodeFlowCtlMsgToWQueue, ms, pWrite, tsDnodeTmr, &unUsed); + + vDebug("vgId:%d, perform flowctl for %d ms", pVnode->vgId, ms); + return TSDB_CODE_RPC_ACTION_IN_PROGRESS; +} + int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { int32_t code = 0; SVnodeObj *pVnode = vparam; @@ -77,8 +106,6 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // assign version pHead->version = pVnode->version + 1; - if (pVnode->delayMs) taosMsleep(pVnode->delayMs); - } else { // from wal or forward // for data from WAL or forward, version may be smaller if (pHead->version <= pVnode->version) return 0; @@ -218,9 +245,10 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; + int32_t code = 0; if (qtype == TAOS_QTYPE_RPC) { - int32_t code = vnodeCheckWrite(pVnode); + code = vnodeCheckWrite(pVnode); if (code != TSDB_CODE_SUCCESS) return code; } @@ -237,14 +265,18 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar if (rparam != NULL) { SRpcMsg *pRpcMsg = rparam; - pWrite->rpcHandle = pRpcMsg->handle; - pWrite->rpcAhandle = pRpcMsg->ahandle; + pWrite->rpcMsg = *pRpcMsg; } memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); + pWrite->pVnode = pVnode; + pWrite->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return code; + int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); From ffbbb0cc44768e97698355bca586016598964b27 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 20:36:44 +0800 Subject: [PATCH 52/75] TD-1455 TD-2196 --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 12 ++++++++ src/vnode/src/vnodeMain.c | 6 ++-- src/vnode/src/vnodeWrite.c | 59 +++++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4087f638a9..efe3d7678a 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -97,6 +97,7 @@ extern int32_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; +extern int32_t tsFlowCtrl; // restful extern int32_t tsEnableHttpModule; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4495c3d928..f8bb965d28 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -133,6 +133,7 @@ int32_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400*100; // seconds 10days int32_t tsMnodeEqualVnodeNum = 4; +int32_t tsFlowCtrl = 1; // restful int32_t tsEnableHttpModule = 1; @@ -971,6 +972,17 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + // module configs + cfg.option = "flowctrl"; + cfg.ptr = &tsFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); cfg.option = "http"; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 1ee968e7cb..df4376d5d0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -660,7 +660,7 @@ static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { static void vnodeNotifyRole(int32_t vgId, int8_t role) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { - vError("vgId:%d, vnode not found while notify role", vgId); + vTrace("vgId:%d, vnode not found while notify role", vgId); return; } @@ -680,12 +680,12 @@ static void vnodeNotifyRole(int32_t vgId, int8_t role) { static void vnodeCtrlFlow(int32_t vgId, int32_t level) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { - vError("vgId:%d, vnode not found while ctrl flow", vgId); + vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } pVnode->flowctlLevel = level; - vDebug("vgId:%d, set flowctl level:%d", pVnode->vgId, level); + vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index cada188591..4bf7b30ebd 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -17,6 +17,7 @@ #include "os.h" #include "taosmsg.h" #include "taoserror.h" +#include "tglobal.h" #include "tqueue.h" #include "trpc.h" #include "tsdb.h" @@ -40,6 +41,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite); void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg; @@ -50,32 +52,6 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } -static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { - SVWriteMsg *pWrite = param; - SVnodeObj * pVnode = pWrite->pVnode; - - int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, &pWrite->rpcMsg); - if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { - vDebug("vgId:%d, failed to reprocess msg after perform flowctl since %s", pVnode->vgId, tstrerror(code)); - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); - } - - tfree(pWrite); - vnodeRelease(pWrite->pVnode); -} - -static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { - SVnodeObj *pVnode = pWrite->pVnode; - if (pVnode->flowctlLevel <= 0) return 0; - - int32_t ms = pVnode->flowctlLevel * 5; - void * unUsed = NULL; - taosTmrReset(vnodeFlowCtlMsgToWQueue, ms, pWrite, tsDnodeTmr, &unUsed); - - vDebug("vgId:%d, perform flowctl for %d ms", pVnode->vgId, ms); - return TSDB_CODE_RPC_ACTION_IN_PROGRESS; -} - int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { int32_t code = 0; SVnodeObj *pVnode = vparam; @@ -298,3 +274,34 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { taosFreeQitem(pWrite); vnodeRelease(pVnode); } + +static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { + SVWriteMsg *pWrite = param; + SVnodeObj * pVnode = pWrite->pVnode; + + int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, pWrite->rpcMsg.handle == NULL ? NULL : &pWrite->rpcMsg); + if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { + vDebug("vgId:%d, failed to reprocess msg after perform flowctrl since %s", pVnode->vgId, tstrerror(code)); + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } + + tfree(pWrite); + vnodeRelease(pWrite->pVnode); +} + +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + if (pVnode->flowctlLevel <= 0) return 0; + + if (tsFlowCtrl == 0) { + int32_t ms = pVnode->flowctlLevel * 2; + if (ms > 60000) ms = 60000; + vDebug("vgId:%d, perform flowctrl for %d ms", pVnode->vgId, ms); + taosMsleep(ms); + return 0; + } else { + void *unUsed = NULL; + taosTmrReset(vnodeFlowCtlMsgToWQueue, 5, pWrite, tsDnodeTmr, &unUsed); + return TSDB_CODE_RPC_ACTION_IN_PROGRESS; + } +} From aeed9db84c8b752eabc0577477f78733e33519a8 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 26 Nov 2020 09:27:17 +0800 Subject: [PATCH 53/75] [TD-2230] use error max len when dump in --- src/kit/taosdump/taosdump.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 88f07ee602..a7258c9724 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -2146,7 +2146,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c char * line = NULL; size_t line_len = 0; - cmd = (char *)malloc(COMMAND_SIZE); + cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); if (cmd == NULL) { fprintf(stderr, "failed to allocate memory\n"); return -1; @@ -2155,7 +2155,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c int lineNo = 0; while ((read_len = getline(&line, &line_len, fp)) != -1) { ++lineNo; - if (read_len >= COMMAND_SIZE) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; //if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -2176,7 +2176,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); } - memset(cmd, 0, COMMAND_SIZE); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } From 060299548e69b2eed1eb611429e96f5a922f8d16 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 11:09:46 +0800 Subject: [PATCH 54/75] change log print --- src/dnode/src/dnodeMPeer.c | 2 +- src/dnode/src/dnodeMRead.c | 2 +- src/dnode/src/dnodeMWrite.c | 2 +- src/dnode/src/dnodeVRead.c | 2 +- src/os/inc/osAlpine.h | 1 + src/os/inc/osArm32.h | 1 + src/os/inc/osArm64.h | 1 + src/os/inc/osDarwin.h | 1 + src/os/inc/osLinux32.h | 1 + src/os/inc/osLinux64.h | 1 + src/os/inc/osWindows.h | 1 + src/vnode/inc/vnodeInt.h | 2 +- src/vnode/src/vnodeMain.c | 2 +- 13 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 05b37bd338..ee6dc5212e 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -162,7 +162,7 @@ static void *dnodeProcessMPeerQueue(void *param) { break; } - dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); + dTrace("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); int32_t code = mnodeProcessPeerReq(pPeerMsg); dnodeSendRpcMPeerRsp(pPeerMsg, code); } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index ee9da72e4d..65f3af7b3b 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -168,7 +168,7 @@ static void *dnodeProcessMReadQueue(void *param) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, + dTrace("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, taosMsg[pRead->rpcMsg.msgType]); int32_t code = mnodeProcessRead(pRead); dnodeSendRpcMReadRsp(pRead, code); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 65c0d53819..ef2d49ef42 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -174,7 +174,7 @@ static void *dnodeProcessMWriteQueue(void *param) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); int32_t code = mnodeProcessWrite(pWrite); diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 4cce54bf59..b42a627a3a 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -175,7 +175,7 @@ static void *dnodeProcessReadQueue(void *pWorker) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, + dTrace("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, taosMsg[pRead->msgType], qtype); int32_t code = vnodeProcessRead(pVnode, pRead); diff --git a/src/os/inc/osAlpine.h b/src/os/inc/osAlpine.h index d939adfb6d..eba9459395 100644 --- a/src/os/inc/osAlpine.h +++ b/src/os/inc/osAlpine.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include typedef int(*__compar_fn_t)(const void *, const void *); void error (int, int, const char *); diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 17b4d2dbd5..24ff95522e 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -76,6 +76,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osArm64.h b/src/os/inc/osArm64.h index 3ae08b45f4..22f0000e96 100644 --- a/src/os/inc/osArm64.h +++ b/src/os/inc/osArm64.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 7bb844831e..1461ec6d3b 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -70,6 +70,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_FILE_SENDIFLE diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h index 93e917e797..cfef05368f 100644 --- a/src/os/inc/osLinux32.h +++ b/src/os/inc/osLinux32.h @@ -76,6 +76,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h index c0841c41bd..a2febd51b7 100644 --- a/src/os/inc/osLinux64.h +++ b/src/os/inc/osLinux64.h @@ -79,6 +79,7 @@ extern "C" { #include #endif #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5003e48c44..5a1e642572 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -40,6 +40,7 @@ #include #include #include +#include #include "msvcProcess.h" #include "msvcDirect.h" #include "msvcFcntl.h" diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index cf9a1baa80..7fc9b100ef 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -39,7 +39,7 @@ typedef struct { int32_t refCount; // reference count int32_t queuedWMsg; int32_t queuedRMsg; - int32_t flowctlLevel; + int32_t flowctrlLevel; int8_t status; int8_t role; int8_t accessState; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index df4376d5d0..cd6d2ea7c0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -684,7 +684,7 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level) { return; } - pVnode->flowctlLevel = level; + pVnode->flowctrlLevel = level; vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); From 4f2d7a1a83c00d900fda99ee174e8008f89bfc02 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 04:07:38 +0000 Subject: [PATCH 55/75] [TD-2216]: crash in qUtil.c --- src/query/inc/qUtil.h | 2 +- src/query/src/qUtil.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index bc23ffadd4..8b84ac0182 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -27,7 +27,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2b4cdef9b0..3f56366db8 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -133,7 +133,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type); } // move the unclosed window in the front of the window list @@ -272,9 +272,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16 * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { dst->numOfRows = src->numOfRows; - dst->win = src->win; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + dst->key = realloc(dst->key, varDataTLen(src->key)); + varDataCopy(dst->key, src->key); + } else { + dst->win = src->win; + } dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From 810023ca6894c2c220c20344246c16a7558a8c1d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 13:13:21 +0800 Subject: [PATCH 56/75] TD-1455 TD-2196 --- src/inc/taoserror.h | 1 + src/vnode/src/vnodeWrite.c | 46 ++++---- tests/script/unique/cluster/flowctrl.sim | 131 +++++++++++++++++++++++ 3 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 tests/script/unique/cluster/flowctrl.sim diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index f1e736130c..77ec5350ba 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -205,6 +205,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid ve TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 4bf7b30ebd..57bd407cd1 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -250,15 +250,15 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar atomic_add_fetch_32(&pVnode->refCount, 1); - code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return code; - int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); taosMsleep(1); } + code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return 0; + vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); taosWriteQitem(pVnode->wqueue, qtype, pWrite); @@ -268,40 +268,50 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; - atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); - vTrace("vgId:%d, free from vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); + int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); + vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, queued); taosFreeQitem(pWrite); vnodeRelease(pVnode); } -static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { +static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { SVWriteMsg *pWrite = param; SVnodeObj * pVnode = pWrite->pVnode; + int32_t code = TSDB_CODE_VND_SYNCING; - int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, pWrite->rpcMsg.handle == NULL ? NULL : &pWrite->rpcMsg); - if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { - vDebug("vgId:%d, failed to reprocess msg after perform flowctrl since %s", pVnode->vgId, tstrerror(code)); + pWrite->processedCount++; + if (pWrite->processedCount > 100) { + vError("vgId:%d, msg:%p, failed to process since %s", pVnode->vgId, pWrite, tstrerror(code)); + pWrite->processedCount = 1; dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } else { + code = vnodePerformFlowCtrl(pWrite); + if (code == 0) { + vTrace("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + pWrite->processedCount = 0; + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + } } - - tfree(pWrite); - vnodeRelease(pWrite->pVnode); } static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { SVnodeObj *pVnode = pWrite->pVnode; - if (pVnode->flowctlLevel <= 0) return 0; + if (pVnode->flowctrlLevel <= 0) return 0; + if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (tsFlowCtrl == 0) { - int32_t ms = pVnode->flowctlLevel * 2; - if (ms > 60000) ms = 60000; - vDebug("vgId:%d, perform flowctrl for %d ms", pVnode->vgId, ms); + int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + if (ms > 100) ms = 100; + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); return 0; } else { void *unUsed = NULL; - taosTmrReset(vnodeFlowCtlMsgToWQueue, 5, pWrite, tsDnodeTmr, &unUsed); - return TSDB_CODE_RPC_ACTION_IN_PROGRESS; + taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsed); + + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, count:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, + pWrite->processedCount); + return TSDB_CODE_VND_ACTION_IN_PROGRESS; } } diff --git a/tests/script/unique/cluster/flowctrl.sim b/tests/script/unique/cluster/flowctrl.sim new file mode 100644 index 0000000000..6dc60d9fba --- /dev/null +++ b/tests/script/unique/cluster/flowctrl.sim @@ -0,0 +1,131 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c http -v 0 +system sh/cfg.sh -n dnode2 -c http -v 0 +system sh/cfg.sh -n dnode3 -c http -v 0 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 + +system sh/cfg.sh -n dnode1 -c replica -v 3 +system sh/cfg.sh -n dnode2 -c replica -v 3 +system sh/cfg.sh -n dnode3 -c replica -v 3 + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +sleep 5001 +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 +$x = 0 +show1: + $x = $x + 1 + sleep 2000 + if $x == 5 then + return -1 + endi +sql show mnodes -x show1 +$mnode1Role = $data2_1 +print mnode1Role $mnode1Role +$mnode2Role = $data2_2 +print mnode2Role $mnode2Role +$mnode3Role = $data2_3 +print mnode3Role $mnode3Role + +if $mnode1Role != master then + goto show1 +endi +if $mnode2Role != slave then + goto show1 +endi +if $mnode3Role != slave then + goto show1 +endi + +print =============== step2 + +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, test int) + +$x = 0 +while $x < 100 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +print =============== step3 +sleep 3000 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print =============== step4 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step5 +sleep 8000 +while $x < 200 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +print =============== step6 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 3000 +while $x < 300 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +system sh/exec.sh -n dnode2 -s start + +sleep 6000 +print =============== step7 +while $x < 400 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 + sleep 1 +endw + +print =============== step8 +sql select * from tb +print rows $rows +if $rows != 400 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From e9c83f8251545936fe424a9004ddb4c6003c8612 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 26 Nov 2020 14:33:53 +0800 Subject: [PATCH 57/75] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java index 8a3dbb2bc1..8de2e3b442 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -5,19 +5,40 @@ import org.junit.Test; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; public class AppMemoryLeakTest { - @Test - public void testAppMemoryLeak() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - - while (true) { - DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); - } - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); + @Test(expected = SQLException.class) + public void testCreateTooManyConnection() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int conCnt = 0; + while (true) { + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + System.out.println(conCnt++ + " : " + conn); } } + + @Test + public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + public static void main(String[] args) throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + } From d3729d273b5b1f200df77b2f1fc137148b2e5ff9 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 15:09:41 +0800 Subject: [PATCH 58/75] add test for column which not exist --- tests/pytest/concurrent_inquiry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 5d1e9a7537..6f4258312d 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -146,7 +146,7 @@ class ConcurrentInquiry: col_list=self.stb_stru_list[tbi-1] tag_list=self.stb_tag_list[tbi-1] is_stb=1 - tlist=col_list+tag_list + tlist=col_list+tag_list+['abc'] #增加不存在的域'abc'，是否会引起新bug con_rand=random.randint(0,len(condition_list)) func_rand=random.randint(0,len(func_list)) col_rand=random.randint(0,len(col_list)) From f547e29a55e57031d035c77bc685e813ffaad772 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 07:39:47 +0000 Subject: [PATCH 59/75] [TD-2135]: reduce memory usage --- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 20 ++++++++++++++------ src/client/src/tscServer.c | 28 +++++++++++++++++++--------- src/client/src/tscUtil.c | 16 ++++++++++++++-- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ff36cf0f5a..c482375bff 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -282,7 +282,7 @@ typedef struct { int8_t dataSourceType; // load data from file or not int8_t submitSchema; // submit block is built with table schema - STagData tagData; + STagData *pTagData; // NOTE: pTagData->data is used as a variant length array SHashObj *pTableList; // referred table involved in sql SArray *pDataBlocks; // SArray submit data blocks after parsing sql } SSqlCmd; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a44a158f93..6de8195d73 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sql += index; tscAllocPayload(pCmd, sizeof(STagData)); - STagData *pTag = &pCmd->tagData; - - memset(pTag, 0, sizeof(STagData)); //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { @@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } + STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memset(pTag, 0, offsetof(STagData, data)); tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); + pCmd->pTagData = pTag; + code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; @@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); + + pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pCmd->pTagData = pTag; + pTag->dataLen = htonl(kvRowLen(row)); kvRowCpy(pTag->data, row); free(row); @@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); } - pTag->dataLen = htonl(pTag->dataLen); - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15859674ab..9735e26ece 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - size_t len = htonl(pCmd->tagData.dataLen); - if (pSql->cmd.autoCreated) { + if (pCmd->autoCreated && pCmd->pTagData != NULL) { + int len = htonl(pCmd->pTagData->dataLen); if (len > 0) { - len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen); - memcpy(pInfoMsg->tags, &pCmd->tagData, len); + len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen); + memcpy(pInfoMsg->tags, pCmd->pTagData, len); pMsg += len; } } @@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; - registerSqlObj(pNew); - tscAddSubqueryInfo(&pNew->cmd); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); @@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { tscError("%p malloc failed for payload to get table meta", pSql); - free(pNew); - + tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); - memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p malloc failed for new tag data to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } + tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; + registerSqlObj(pNew); + int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a75f6d8296..d22c2588cc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->pTableList = NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - + tscFreeQueryInfo(pCmd, removeFromCache); } @@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscFreeSqlResult(pSql); tscResetSqlCmdObj(pCmd, false); + tfree(pCmd->pTagData); + memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; @@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->command = cmd; pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; - memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + free(pNew); + return NULL; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pNew); From 55ce69e2f0c08aed576b46bf11cff1a7b7bc2d4a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 08:00:58 +0000 Subject: [PATCH 60/75] [TD-2182]: coverity issues --- src/client/src/tscSub.c | 7 ++++++- src/query/src/qExecutor.c | 6 +++--- src/util/src/tqueue.c | 3 --- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c81bd7c7c..a782b53e75 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -442,7 +442,12 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); size += sizeof(SQueryTableMsg) + 4096; - tscAllocPayload(&pSql->cmd, (int)size); + int code = tscAllocPayload(&pSql->cmd, (int)size); + if (code != TSDB_CODE_SUCCESS) { + tscError("failed to alloc payload"); + return NULL; + } + for (int retry = 0; retry < 3; retry++) { tscRemoveFromSqlList(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fce25ae119..826858d1dd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5836,9 +5836,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; - pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; - pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; + // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; + // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; + // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index d3c17e7626..143ce7c474 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -79,15 +79,12 @@ void taosCloseQueue(taos_queue param) { if (queue->qset) taosRemoveFromQset(qset, queue); - pthread_mutex_lock(&queue->mutex); - while (pNode) { pTemp = pNode; pNode = pNode->next; free (pTemp); } - pthread_mutex_unlock(&queue->mutex); pthread_mutex_destroy(&queue->mutex); free(queue); From 651672957d470f715b265c29892f495d62c94da0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 26 Nov 2020 16:32:10 +0800 Subject: [PATCH 61/75] [TD-2240] update restfulInsert --- tests/pytest/insert/restfulInsert.py | 37 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index c39d1cbe48..e3a963f1d4 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -18,10 +18,10 @@ import time import argparse class RestfulInsert: - def __init__(self, host, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): + def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} self.url = "http://%s:6041/rest/sql" % host - self.ts = 1500000000000 + self.ts = startTimestamp self.dbname = dbname self.numOfThreads = threads self.numOfTables = tables @@ -36,8 +36,10 @@ class RestfulInsert: for i in range(tablesPerThread): tableID = threadID * tablesPerThread name = 'beijing' if tableID % 2 == 0 else 'shanghai' - data = "create table %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) - requests.post(self.url, data, headers = self.header) + data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertData(self, threadID): print("thread %d started" % threadID) @@ -50,7 +52,9 @@ class RestfulInsert: values = [] for k in range(self.batchSize): data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) - requests.post(self.url, data, headers = self.header) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) @@ -76,15 +80,15 @@ class RestfulInsert: else: random.shuffle(values) for k in range(len(values)): - data += values[k] - requests.post(self.url, data, headers = self.header) + data += values[k] + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) - def run(self): - data = "drop database if exists %s" % self.dbname + def run(self): + data = "create database if not exists %s" % self.dbname requests.post(self.url, data, headers = self.header) - data = "create database %s" % self.dbname - requests.post(self.url, data, headers = self.header) - data = "create table %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname + data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname requests.post(self.url, data, headers = self.header) threads = [] @@ -120,6 +124,13 @@ parser.add_argument( default='127.0.0.1', type=str, help='host name to be connected (default: 127.0.0.1)') +parser.add_argument( + '-S', + '--start-timestamp', + action='store', + default=1500000000000, + type=int, + help='insert data from timestamp (default: 1500000000000)') parser.add_argument( '-d', '--db-name', @@ -169,5 +180,5 @@ parser.add_argument( help='The order of test data (default: False)') args = parser.parse_args() -ri = RestfulInsert(args.host_name, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) +ri = RestfulInsert(args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) ri.run() \ No newline at end of file From fa25d38dea8ec9e310c85fac1da6aa36c70104ef Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 16:37:11 +0800 Subject: [PATCH 62/75] full test --- tests/test-all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index ff47cbfd71..5897978bce 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -17,9 +17,9 @@ function runSimCaseOneByOne { echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ echo -e "${RED}$case failed${NC}" | tee -a out.log out_log=`tail -1 out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log fi @@ -42,9 +42,9 @@ function runPyCaseOneByOne { echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log else $line > /dev/null 2>&1 From 19eccc86ade72ded17cd4c4033b44b275137dc0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 26 Nov 2020 08:44:20 +0000 Subject: [PATCH 63/75] fix TD-2184 --- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/util/src/tskiplist.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 71944c87c6..4c51608b5d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -871,10 +871,10 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); if (pBufBlock->offset == 0) { // return the block to buffer pool - tsdbLockRepo(pRepo); + if (tsdbLockRepo(pRepo) < 0) return; SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList); tdListPrependNode(pBufPool->bufBlockList, pNode); - tsdbUnlockRepo(pRepo); + if (tsdbUnlockRepo(pRepo) < 0) return; } } else { ASSERT(listNEles(pRepo->mem->extraBuffList) > 0); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a36f7f0261..20bec16954 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -291,7 +291,10 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; } else { - if (iter->cur == pSkipList->pHead) return false; + if (iter->cur == pSkipList->pHead) { + tSkipListUnlock(pSkipList); + return false; + } iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it From 1e94822a6a9b04a9396200e8a47392eb987b3598 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 19:02:11 +0800 Subject: [PATCH 64/75] TD-1926 --- src/inc/tsync.h | 4 ++ src/sync/inc/syncInt.h | 2 + src/sync/src/syncMain.c | 1 + src/sync/src/syncRestore.c | 8 +-- src/sync/src/syncRetrieve.c | 100 +++++++----------------------------- src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeMain.c | 24 ++++++++- src/vnode/src/vnodeWrite.c | 2 +- 8 files changed, 55 insertions(+), 87 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 398e1bf97c..293e382519 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -85,6 +85,9 @@ typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); +// get file version +typedef int32_t (*FGetFileVersion)(int32_t vgId, uint64_t *fver); + typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version @@ -97,6 +100,7 @@ typedef struct { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; + FGetFileVersion getFileVersion; } SSyncInfo; typedef void *tsync_h; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 05b7adc5f4..7dc04d70d0 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -139,6 +139,7 @@ typedef struct SsyncPeer { char id[TSDB_EP_LEN + 32]; // peer vgId + end point uint64_t version; uint64_t sversion; // track the peer version in retrieve process + uint64_t lastVer; // track the file version while retrieve int32_t syncFd; int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried @@ -172,6 +173,7 @@ typedef struct SSyncNode { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; + FGetFileVersion getFileVersion; pthread_mutex_t mutex; } SSyncNode; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d2d6d2d7fa..af3241f217 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -196,6 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->confirmForward = pInfo->confirmForward; pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; pNode->notifyFileSynced = pInfo->notifyFileSynced; + pNode->getFileVersion = pInfo->getFileVersion; pNode->selfIndex = -1; pNode->vgId = pInfo->vgId; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ed6b63c92d..1f5b560c87 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -52,12 +52,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info - SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info - SFileAck fileAck; + SFileInfo minfo = {0}; + SFileInfo sinfo = {0}; + SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - uint32_t pindex = 0; // index in last restore + uint32_t pindex = 0; // index in last restore bool fileChanged = false; *fversion = 0; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 060badba9d..e199ce9845 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -25,85 +25,32 @@ #include "tsync.h" #include "syncInt.h" -static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { - sDebug("%s, start to monitor:%s", pPeer->id, name); +static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { + if (pNode->getFileVersion == NULL) return 0; - if (pPeer->notifyFd <= 0) { - pPeer->watchNum = 0; - pPeer->notifyFd = inotify_init1(IN_NONBLOCK); - if (pPeer->notifyFd < 0) { - sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno)); - return -1; - } - - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); - if (pPeer->watchFd == NULL) { - sError("%s, failed to allocate watchFd", pPeer->id); - return -1; - } - - memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); + uint64_t fver = 0; + int32_t code = (*pNode->getFileVersion)(pNode->vgId, &fver); + if (code != 0) { + sInfo("%s, file are modified while retrieve, lastver:%" PRIu64, pPeer->id, pPeer->lastVer); + return code; } - int32_t *wd = pPeer->watchFd + pPeer->watchNum; - - if (*wd >= 0) { - if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) { - sError("%s, failed to remove wd:%d since %s", pPeer->id, *wd, strerror(errno)); - return -1; - } - } - - *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE); - if (*wd == -1) { - sError("%s, failed to add %s since %s", pPeer->id, name, strerror(errno)); + if (fver != pPeer->lastVer) { + sInfo("%s, file are modified while retrieve, fver:%" PRIu64 " lastver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); return -1; - } else { - sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum); } - pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles; - return 0; } -static int32_t syncAreFilesModified(SSyncPeer *pPeer) { - if (pPeer->notifyFd <= 0) return 0; - - char buf[2048]; - int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno)); - return -1; - } - - int32_t code = 0; - if (len > 0) { - const struct inotify_event *event; - char *ptr; - for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) { - event = (const struct inotify_event *)ptr; - if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) { - sDebug("%s, processed file is changed", pPeer->id); - pPeer->fileChanged = 1; - code = 1; - break; - } - } - } - - return code; -} - static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo; - SFileAck fileAck; + SFileInfo fileInfo = {0}; + SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - memset(&fileInfo, 0, sizeof(fileInfo)); - memset(&fileAck, 0, sizeof(fileAck)); + if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer); while (1) { // retrieve file info @@ -136,15 +83,6 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // set the peer sync version pPeer->sversion = fileInfo.fversion; - // get the full path to file - snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); - - // add the file into watch list - if (syncAddIntoWatchList(pPeer, name) < 0) { - sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } - // if sync is not required, continue if (fileAck.sync == 0) { fileInfo.index++; @@ -152,6 +90,9 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { continue; } + // get the full path to file + snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); + // send the file to peer int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { @@ -170,10 +111,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pPeer) != 0) { - sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } + if (syncAreFilesModified(pNode, pPeer) != 0) break; } if (code < 0) { @@ -308,9 +246,9 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; int32_t code = -1; - char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file + char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file - if (syncAreFilesModified(pPeer) != 0) return -1; + if (syncAreFilesModified(pNode, pPeer) != 0) return -1; while (1) { int32_t once = 0; // last WAL has once ever been processed @@ -431,7 +369,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { index++; - if (syncAreFilesModified(pPeer) != 0) break; + if (syncAreFilesModified(pNode, pPeer) != 0) break; } if (code == 0) { diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 7fc9b100ef..e468c2e83e 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -44,6 +44,7 @@ typedef struct { int8_t role; int8_t accessState; int8_t isFull; + int8_t isCommiting; uint64_t version; // current version uint64_t fversion; // version on saved data file void *wqueue; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index cd6d2ea7c0..1b2f3ee93c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -38,6 +38,7 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level); static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); +static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver); #ifndef _SYNC int64_t syncStart(const SSyncInfo *info) { return NULL; } @@ -352,6 +353,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; + syncInfo.getFileVersion = vnodeGetFileVersion; pVnode->sync = syncStart(&syncInfo); #ifndef _SYNC @@ -597,18 +599,19 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { vnodeRelease(pVnode); } -// TODO: this is a simple implement static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { SVnodeObj *pVnode = arg; if (eno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno), pVnode->fversion, pVnode->version); + pVnode->isCommiting = 0; pVnode->isFull = 1; return 0; } if (status == TSDB_STATUS_COMMIT_START) { + pVnode->isCommiting = 1; pVnode->fversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (pVnode->status != TAOS_VN_STATUS_INIT) { @@ -619,6 +622,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + pVnode->isCommiting = 0; pVnode->isFull = 0; if (pVnode->status != TAOS_VN_STATUS_INIT) { walRemoveOneOldFile(pVnode->wal); @@ -765,3 +769,21 @@ static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void vnodeRelease(pVnode); return code; } + +static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return -1; + } + + int32_t code = 0; + if (pVnode->isCommiting) { + code = -1; + } else { + *fver = pVnode->fversion; + } + + vnodeRelease(pVnode); + return code; +} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 57bd407cd1..360a631e1d 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -288,7 +288,7 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { - vTrace("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + vDebug("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); pWrite->processedCount = 0; taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); } From 11230382920a402fe2c9010a2b16e4b544e22b07 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 14:24:18 +0000 Subject: [PATCH 65/75] TD-2245 --- src/dnode/src/dnodePeer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 4c44924cd0..6bf22cee4e 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -151,9 +151,9 @@ void dnodeCleanupClient() { } static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dDebug("msg:%p is ignored since dnode not running", pMsg); + dDebug("msg:%p is ignored since dnode is stopping", pMsg); rpcFreeCont(pMsg->pCont); return; } From 86479c1f6a42a97d0ef5d880e18565eae1660ded Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 11:33:44 +0800 Subject: [PATCH 66/75] TD-2246 --- tests/test/c/CMakeLists.txt | 7 ++-- tests/test/c/hashIterator.c | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/hashIterator.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 26aa20e647..11480a8ba2 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - #add_executable(createNormalTable createNormalTable.c) - #target_link_libraries(createNormalTable taos_static tutil common pthread) + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) @@ -45,5 +45,8 @@ IF (TD_LINUX) #add_executable(invalidTableId invalidTableId.c) #target_link_libraries(invalidTableId taos_static tutil common pthread) + + add_executable(hashIterator hashIterator.c) + target_link_libraries(hashIterator taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/hashIterator.c b/tests/test/c/hashIterator.c new file mode 100644 index 0000000000..cbd8a0895e --- /dev/null +++ b/tests/test/c/hashIterator.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "hash.h" + +typedef struct HashTestRow { + int32_t keySize; + char key[100]; +} HashTestRow; + +int main(int argc, char *argv[]) { + _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + void * hashHandle = taosHashInit(100, hashFp, true, HASH_ENTRY_LOCK); + + pPrint("insert 3 rows to hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow row = {0}; + row.keySize = sprintf(row.key, "0.db.st%d", t); + + taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow)); + } + + pPrint("start iterator"); + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + pPrint("drop key:%s", row->key); + taosHashRemove(hashHandle, row->key, row->keySize); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + //Before getting the next iterator, the object just deleted can be obtained + row = taosHashIterate(hashHandle, row); + } + + pPrint("stop iterator"); + taosHashCancelIterate(hashHandle, row); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + return 0; +} \ No newline at end of file From 4b09f05290596014d987bcdc8c34d2004814618e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:23:16 +0000 Subject: [PATCH 67/75] add a removed flag in pNode --- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- src/rpc/src/rpcTcp.c | 15 +++++++++------ src/util/inc/hash.h | 1 + src/util/src/hash.c | 19 +++++++++++++++---- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/connector/go b/src/connector/go index 050667e5b4..8c58c512b6 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index ec77d9049a..d598db167e 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 +Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 7b8cf3cda2..2850046d05 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) { int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; - if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; + SThreadObj *pThreadObj = pFdObj->pThreadObj; - return taosWriteMsg(pFdObj->fd, data, len); + int ret = taosWriteMsg(pFdObj->fd, data, len); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + + return ret; } static void taosReportBrokenLink(SFdObj *pFdObj) { @@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); return -1; } @@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("TCP malloc mem:%p size:%d", buffer, size); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); } msg = buffer + tsRpcOverhead; @@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index aaff48f4f8..572f1a50c6 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -36,6 +36,7 @@ typedef struct SHashNode { uint32_t keyLen; // length of the key uint32_t dataLen; // length of data int8_t count; // reference count + int8_t removed; // flag to indicate removed char data[]; } SHashNode; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index cd9a9f7d0a..3ac54eedd8 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -222,7 +222,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -361,7 +361,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) break; prevNode = pNode; @@ -372,6 +372,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe code = 0; // it is found pNode->count--; + pNode->removed = 1; if (pNode->count <= 0) { if (prevNode) { prevNode->next = pNode->next; @@ -702,8 +703,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pNode = pNode->next; } - if (pNode) { + if (pNode) { pNode = pNode->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + pOld->count--; if (pOld->count <=0) { if (prevNode) { @@ -755,6 +761,11 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { } pNode = pe->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + if (pNode) break; if (pHashObj->type == HASH_ENTRY_LOCK) { From 257af62edfc99dca2198eb4dc3fbb00c36d9b043 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 27 Nov 2020 15:29:12 +0800 Subject: [PATCH 68/75] [TD-2258] taos_fetch_block() resutl printf error --- tests/examples/c/apitest.c | 12 ++++++------ tests/examples/c/subscribe.c | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index be60a88ad7..e08d667d6b 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -79,11 +79,11 @@ static int print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { int rows = 0; while ((rows = taos_fetch_block(res, &row))) { - for (int i = 0; i < rows; i++) { - char temp[256]; - taos_print_row(temp, row + i, fields, num_fields); - puts(temp); - } + //for (int i = 0; i < rows; i++) { + // char temp[256]; + // taos_print_row(temp, row + i, fields, num_fields); + // puts(temp); + //} nRows += rows; } } else { @@ -498,4 +498,4 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); -} \ No newline at end of file +} diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index 8368af18f7..cdd8ddaf7f 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -19,10 +19,10 @@ void print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { nRows = taos_fetch_block(res, &row); - for (int i = 0; i < nRows; i++) { - taos_print_row(buf, row + i, fields, num_fields); - puts(buf); - } + //for (int i = 0; i < nRows; i++) { + // taos_print_row(buf, row + i, fields, num_fields); + // puts(buf); + //} } else { while ((row = taos_fetch_row(res))) { taos_print_row(buf, row, fields, num_fields); From 15691bfb088fd114f3e53c769c2829937e746d63 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:41:35 +0000 Subject: [PATCH 69/75] data type compatible with windows --- src/util/inc/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 572f1a50c6..b0319d3e13 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -34,7 +34,7 @@ typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - uint32_t dataLen; // length of data + size_t dataLen; // length of data int8_t count; // reference count int8_t removed; // flag to indicate removed char data[]; From feb2270a4cf02588b0795c6fe62d31f99d135c7d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 15:52:38 +0800 Subject: [PATCH 70/75] change log format --- src/common/src/tglobal.c | 2 +- src/sync/src/syncMain.c | 5 ++--- src/sync/src/syncRestore.c | 4 ++-- src/sync/src/syncRetrieve.c | 11 ++++++----- src/vnode/src/vnodeMain.c | 7 +++++-- src/vnode/src/vnodeWrite.c | 8 +++++--- src/wal/src/walWrite.c | 4 ++-- tests/script/general/wal/sync.sim | 1 + tests/script/tmp/mnodes.sim | 4 ++++ 9 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index f8bb965d28..d26746a5d1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -200,7 +200,7 @@ int32_t tsNumOfLogLines = 10000000; int32_t mDebugFlag = 135; int32_t sdbDebugFlag = 135; int32_t dDebugFlag = 135; -int32_t vDebugFlag = 131; +int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index af3241f217..ce85c940ff 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -541,7 +541,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->ip = ip; pPeer->port = pInfo->nodePort; pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; - snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port); + snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId); pPeer->peerFd = -1; pPeer->syncFd = -1; @@ -1144,8 +1144,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer->syncFd = connFd; syncCreateRestoreDataThread(pPeer); } else { - sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd, - pPeer->syncFd); + sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 1f5b560c87..ada55aaf8f 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -134,7 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; - sDebug("%s, file changed while restore file", pPeer->id); + sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion); } if (code < 0) { @@ -160,7 +160,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } if (pHead->len == 0) { - sDebug("%s, wal is synced over", pPeer->id); + sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer); code = 0; break; } // wal sync over diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index e199ce9845..c956353a32 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -58,6 +58,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); + sDebug("%s, file:%s info will be sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); @@ -107,7 +108,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { break; } - sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); + sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); fileInfo.index++; // check if processed files are modified @@ -419,18 +420,18 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (syncRetrieveFile(pPeer) < 0) { - sError("%s, failed to retrieve file", pPeer->id); + sError("%s, failed to retrieve files", pPeer->id); return -1; } // if no files are synced, there must be wal to sync, sversion must be larger than one if (pPeer->sversion == 0) pPeer->sversion = 1; - sInfo("%s, start to retrieve wal", pPeer->id); + sInfo("%s, start to retrieve wals", pPeer->id); if (syncRetrieveWal(pPeer) < 0) { - sError("%s, failed to retrieve wal", pPeer->id); + sError("%s, failed to retrieve wals", pPeer->id); return -1; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 1b2f3ee93c..076392a1dc 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -688,8 +688,10 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level) { return; } - pVnode->flowctrlLevel = level; - vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); + if (pVnode->flowctrlLevel != level) { + vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); + pVnode->flowctrlLevel = level; + } vnodeRelease(pVnode); } @@ -779,6 +781,7 @@ static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { int32_t code = 0; if (pVnode->isCommiting) { + vDebug("vgId:%d, vnode is commiting while get file version", vgId); code = -1; } else { *fver = pVnode->fversion; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 360a631e1d..268d1fb53b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -282,13 +282,15 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { pWrite->processedCount++; if (pWrite->processedCount > 100) { - vError("vgId:%d, msg:%p, failed to process since %s", pVnode->vgId, pWrite, tstrerror(code)); + vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), + pWrite->processedCount); pWrite->processedCount = 1; dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { - vDebug("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, + pWrite->processedCount); pWrite->processedCount = 0; taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); } @@ -310,7 +312,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { void *unUsed = NULL; taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsed); - vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, count:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, pWrite->processedCount); return TSDB_CODE_VND_ACTION_IN_PROGRESS; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 36b3dba165..8a532c8164 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -211,7 +211,7 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { code = (*fileId == pWal->fileId) ? 0 : 1; } - wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); + wDebug("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); pthread_mutex_unlock(&(pWal->mutex)); return code; @@ -325,7 +325,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch offset = offset + sizeof(SWalHead) + pHead->len; - wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, fileId, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index abaf22f919..c6f7402b87 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -82,6 +82,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 +sleep 1000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index de02ae741b..e11140028d 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -20,6 +20,10 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode1 -c minTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode2 -c minTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode3 -c minTablesPerVnode -v 1000 + system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 From 6bf39dfa3b6bdbf1afc622e18e4e13118d18178b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 17:16:49 +0800 Subject: [PATCH 71/75] TD-1455 TD-2196 --- src/inc/taoserror.h | 5 ++++- src/sync/src/syncRetrieve.c | 36 +++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 77ec5350ba..5fde6d40d5 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -261,6 +261,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sy TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_VND_COMMITING, 0, 0x0904, "Vnode is commiting") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_FILE_CHNAGED, 0, 0x0905, "Vnode file is changed") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_APP_ERROR, 0, 0x1000, "Unexpected generic error in sync") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") @@ -367,6 +370,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") +// odbc TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") @@ -390,7 +394,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad se TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general") - #ifdef TAOS_ERROR_C }; #endif diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c956353a32..4559285a1f 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include #include "os.h" +#include "taoserror.h" #include "tlog.h" #include "tutil.h" #include "tglobal.h" @@ -26,28 +27,31 @@ #include "syncInt.h" static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { - if (pNode->getFileVersion == NULL) return 0; + if (pNode->getFileVersion == NULL) return TSDB_CODE_SUCCESS; uint64_t fver = 0; int32_t code = (*pNode->getFileVersion)(pNode->vgId, &fver); if (code != 0) { - sInfo("%s, file are modified while retrieve, lastver:%" PRIu64, pPeer->id, pPeer->lastVer); - return code; + sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastVer); + pPeer->fileChanged = 1; + return TSDB_CODE_SYN_VND_COMMITING; } if (fver != pPeer->lastVer) { - sInfo("%s, file are modified while retrieve, fver:%" PRIu64 " lastver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); - return -1; + sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last fver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); + pPeer->fileChanged = 1; + return TSDB_CODE_SYN_FILE_CHNAGED; } - return 0; + pPeer->fileChanged = 0; + return TSDB_CODE_SUCCESS; } static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo = {0}; SFileAck fileAck = {0}; - int32_t code = -1; + int32_t code = TSDB_CODE_SYN_APP_ERROR; char name[TSDB_FILENAME_LEN * 2] = {0}; if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer); @@ -58,25 +62,27 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); - sDebug("%s, file:%s info will be sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); + sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } // if no file anymore, break if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { + code = TSDB_CODE_SUCCESS; sDebug("%s, no more files to sync", pPeer->id); - code = 0; break; } // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -97,6 +103,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file to peer int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -104,6 +111,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -112,11 +120,12 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pNode, pPeer) != 0) break; + code = syncAreFilesModified(pNode, pPeer); + if (code != TSDB_CODE_SUCCESS) break; } - if (code < 0) { - sError("%s, failed to retrieve file", pPeer->id); + if (code != TSDB_CODE_SUCCESS) { + sError("%s, failed to retrieve file since %s", pPeer->id, tstrerror(code)); } return code; @@ -368,8 +377,6 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { close(sfd); if (code < 0) break; - index++; - if (syncAreFilesModified(pNode, pPeer) != 0) break; } @@ -445,7 +452,6 @@ void *syncRetrieveData(void *param) { if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); - pPeer->fileChanged = 0; pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { sError("%s, failed to open socket to sync", pPeer->id); From 7296733b47ca10cc420727e3b47e115e817b7939 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 18:03:17 +0800 Subject: [PATCH 72/75] compile error in arm32 --- src/sync/src/syncRestore.c | 4 ++-- src/sync/src/syncRetrieve.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ada55aaf8f..d156c93865 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -52,8 +52,8 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo = {0}; - SFileInfo sinfo = {0}; + SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */ + SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */ SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 4559285a1f..305ec492b9 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -49,7 +49,7 @@ static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo = {0}; + SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); SFileAck fileAck = {0}; int32_t code = TSDB_CODE_SYN_APP_ERROR; char name[TSDB_FILENAME_LEN * 2] = {0}; From dbe3b6f8f6663e1fab22b54ee11694b4a1680da0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 23:27:13 +0800 Subject: [PATCH 73/75] TD-1895 --- src/dnode/src/dnodeMain.c | 2 ++ src/inc/twal.h | 5 ++- src/os/inc/osFile.h | 19 ++++------- src/os/src/detail/osFile.c | 2 +- src/util/inc/tfile.h | 21 +++++++----- src/util/src/tfile.c | 68 ++++++++++++++++++++++++++++++-------- src/wal/inc/walInt.h | 2 +- src/wal/src/walMgmt.c | 9 ++--- src/wal/src/walWrite.c | 55 +++++++++++++++--------------- 9 files changed, 113 insertions(+), 70 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 130be0af20..9f52dbd331 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -19,6 +19,7 @@ #include "tutil.h" #include "tconfig.h" #include "tglobal.h" +#include "tfile.h" #include "twal.h" #include "trpc.h" #include "dnode.h" @@ -55,6 +56,7 @@ typedef struct { } SDnodeComponent; static const SDnodeComponent tsDnodeComponents[] = { + {"tfile", tfInit, tfCleanup}, {"rpc", rpcInit, rpcCleanup}, {"storage", dnodeInitStorage, dnodeCleanupStorage}, {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, diff --git a/src/inc/twal.h b/src/inc/twal.h index 8dd3a8a912..1645de77aa 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -51,9 +51,8 @@ typedef struct { typedef void * twalh; // WAL HANDLE typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg); -int32_t walInit(); -void walCleanUp(); - +int32_t walInit(); +void walCleanUp(); twalh walOpen(char *path, SWalCfg *pCfg); int32_t walAlter(twalh pWal, SWalCfg *pCfg); void walStop(twalh); diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 62e44d8eb0..c9b3b9cd76 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,17 +20,6 @@ extern "C" { #endif -#define tread(fd, buf, count) read(fd, buf, count) -#define twrite(fd, buf, count) write(fd, buf, count) -#define tlseek(fd, offset, whence) lseek(fd, offset, whence) -#define tclose(fd) \ - { \ - if (FD_VALID(fd)) { \ - close(fd); \ - fd = FD_INITIALIZER; \ - } \ - } - int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); @@ -39,7 +28,13 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP #define taosRead(fd, buf, count) taosReadImp(fd, buf, count) #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) #define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence) -#define taosClose(x) tclose(x) +#define taosClose(fd) \ + { \ + if (FD_VALID(fd)) { \ + close(fd); \ + fd = FD_INITIALIZER; \ + } \ + } // TAOS_OS_FUNC_FILE_SENDIFLE int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 23fc88b8e1..2e6886aa21 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -116,7 +116,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) { } int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { - return (int64_t)tlseek(fd, (long)offset, whence); + return (int64_t)lseek(fd, (long)offset, whence); } #ifndef TAOS_OS_FUNC_FILE_SENDIFLE diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 10b7c1df35..b32f297d9c 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -20,23 +20,26 @@ extern "C" { #endif -#include - // init taos file module -int32_t tfinit(); +int32_t tfInit(); // clean up taos file module -void tfcleanup(); +void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfopen(const char *pathname, int32_t flags); -int64_t tfclose(int64_t tfd); -int64_t tfwrite(int64_t tfd, void *buf, int64_t count); -int64_t tfread(int64_t tfd, void *buf, int64_t count); +int64_t tfOpen(const char *pathname, int32_t flags); +int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode); +int64_t tfClose(int64_t tfd); +int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +int64_t tfRead(int64_t tfd, void *buf, int64_t count); +int32_t tfFsync(int64_t tfd); +bool tfValid(int64_t tfd); +int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +int32_t tfFtruncate(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif -#endif // TDENGINE_TREF_H +#endif // TDENGINE_TFILE_H diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 27ba30fe81..49b28dad6f 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" #include "tulog.h" @@ -21,40 +22,52 @@ static int32_t tsFileRsetId = -1; -static void taosCloseFile(void *p) { +static void tfCloseFile(void *p) { close((int32_t)(uintptr_t)p); } -int32_t tfinit() { - tsFileRsetId = taosOpenRef(2000, taosCloseFile); - return tsFileRsetId; +int32_t tfInit() { + tsFileRsetId = taosOpenRef(2000, tfCloseFile); + if (tsFileRsetId > 0) { + return 0; + } else { + return -1; + } } -void tfcleanup() { +void tfCleanup() { if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); tsFileRsetId = -1; } -int64_t tfopen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); - +static int64_t tfOpenImp(int32_t fd) { if (fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; - } + } - void *p = (void *)(int64_t)fd; + void * p = (void *)(int64_t)fd; int64_t rid = taosAddRef(tsFileRsetId, p); if (rid < 0) close(fd); return rid; } -int64_t tfclose(int64_t tfd) { +int64_t tfOpen(const char *pathname, int32_t flags) { + int32_t fd = open(pathname, flags); + return tfOpenImp(fd); +} + +int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { + int32_t fd = open(pathname, flags, mode); + return tfOpenImp(fd); +} + +int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { +int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; @@ -67,7 +80,7 @@ int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { return ret; } -int64_t tfread(int64_t tfd, void *buf, int64_t count) { +int64_t tfRead(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; @@ -79,3 +92,32 @@ int64_t tfread(int64_t tfd, void *buf, int64_t count) { taosReleaseRef(tsFileRsetId, tfd); return ret; } + +int64_t tfFsync(int64_t tfd) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return fsync(fd); +} + +bool tfValid(int64_t tfd) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + return p != NULL; +} + +int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return taosLSeek(fd, offset, whence); +} + +int32_t tfFtruncate(int64_t tfd, int64_t length) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return taosFtruncate(fd, length); +} diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index b0edabfbd8..06748d885f 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -44,8 +44,8 @@ typedef struct { uint64_t version; int64_t fileId; int64_t rid; + int64_t tfd; int32_t vgId; - int32_t fd; int32_t keep; int32_t level; int32_t fsyncPeriod; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 36c190be3e..72ea239817 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -17,6 +17,7 @@ #include "os.h" #include "taoserror.h" #include "tref.h" +#include "tfile.h" #include "twal.h" #include "walInt.h" @@ -61,7 +62,7 @@ void *walOpen(char *path, SWalCfg *pCfg) { } pWal->vgId = pCfg->vgId; - pWal->fd = -1; + pWal->tfd = -1; pWal->fileId = -1; pWal->level = pCfg->walLevel; pWal->keep = pCfg->keep; @@ -124,7 +125,7 @@ void walClose(void *handle) { SWal *pWal = handle; pthread_mutex_lock(&pWal->mutex); - taosClose(pWal->fd); + tfClose(pWal->tfd); pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refId, pWal->rid); } @@ -143,7 +144,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); - taosClose(pWal->fd); + tfClose(pWal->tfd); pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } @@ -172,7 +173,7 @@ static void walFsyncAll() { while (pWal) { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); - int32_t code = fsync(pWal->fd); + int32_t code = tfFsync(pWal->tfd); if (code != 0) { wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 8a532c8164..10e1b4dd61 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -18,6 +18,7 @@ #include "os.h" #include "taoserror.h" #include "tchecksum.h" +#include "tfile.h" #include "twal.h" #include "walInt.h" @@ -36,8 +37,8 @@ int32_t walRenew(void *handle) { pthread_mutex_lock(&pWal->mutex); - if (pWal->fd >= 0) { - tclose(pWal->fd); + if (tfValid(pWal->tfd)) { + tfClose(pWal->tfd); wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); } @@ -49,9 +50,9 @@ int32_t walRenew(void *handle) { } snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { + if (!tfValid(pWal->tfd)) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); } else { @@ -67,7 +68,7 @@ void walRemoveOneOldFile(void *handle) { SWal *pWal = handle; if (pWal == NULL) return; if (pWal->keep == TAOS_WAL_KEEP) return; - if (pWal->fd <= 0) return; + if (!tfValid(pWal->tfd)) return; pthread_mutex_lock(&pWal->mutex); @@ -113,7 +114,7 @@ int32_t walWrite(void *handle, SWalHead *pHead) { int32_t code = 0; // no wal - if (pWal->fd <= 0) return 0; + if (!tfValid(pWal->tfd)) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0; if (pHead->version <= pWal->version) return 0; @@ -123,12 +124,12 @@ int32_t walWrite(void *handle, SWalHead *pHead) { pthread_mutex_lock(&pWal->mutex); - if (taosWrite(pWal->fd, pHead, contLen) != contLen) { + if (tfWrite(pWal->tfd, pHead, contLen) != contLen) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, - pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len); + wTrace("vgId:%d, write wal, fileId:%" PRId64 " tfd:%" PRId64 " hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, + pWal->fileId, pWal->tfd, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; } @@ -141,11 +142,11 @@ int32_t walWrite(void *handle, SWalHead *pHead) { void walFsync(void *handle, bool forceFsync) { SWal *pWal = handle; - if (pWal == NULL || pWal->fd < 0) return; + if (pWal == NULL || !tfValid(pWal->tfd)) return; if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ", do fsync", pWal->vgId, pWal->fileId); - if (fsync(pWal->fd) < 0) { + if (tfFsync(pWal->tfd) < 0) { wError("vgId:%d, fileId:%" PRId64 ", fsync failed since %s", pWal->vgId, pWal->fileId, strerror(errno)); } } @@ -186,8 +187,8 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { // open the existing WAL file in append mode pWal->fileId = 0; snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { + pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + if (!tfValid(pWal->tfd)) { wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -217,22 +218,22 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { return code; } -static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) { - taosFtruncate(fd, offset); - fsync(fd); +static void walFtruncate(SWal *pWal, int64_t tfd, int64_t offset) { + tfFtruncate(tfd, offset); + tfFsync(tfd); } -static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) { +static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { int64_t pos = *offset; while (1) { pos++; - if (lseek(fd, pos, SEEK_SET) < 0) { + if (tfLseek(tfd, pos, SEEK_SET) < 0) { wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno)); return TSDB_CODE_WAL_FILE_CORRUPTED; } - if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) { + if (tfRead(tfd, pHead, sizeof(SWalHead)) <= 0) { wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); return TSDB_CODE_WAL_FILE_CORRUPTED; } @@ -259,8 +260,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch return TAOS_SYSTEM_ERROR(errno); } - int32_t fd = open(name, O_RDWR); - if (fd < 0) { + int64_t tfd = tfOpen(name, O_RDWR); + if (!tfValid(tfd)) { wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); tfree(buffer); return TAOS_SYSTEM_ERROR(errno); @@ -273,7 +274,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch SWalHead *pHead = buffer; while (1) { - int32_t ret = taosRead(fd, pHead, sizeof(SWalHead)); + int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -284,16 +285,16 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch if (ret < sizeof(SWalHead)) { wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret); - walFtruncate(pWal, fd, offset); + walFtruncate(pWal, tfd, offset); break; } if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); - code = walSkipCorruptedRecord(pWal, pHead, fd, &offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); if (code != TSDB_CODE_SUCCESS) { - walFtruncate(pWal, fd, offset); + walFtruncate(pWal, tfd, offset); break; } } @@ -310,7 +311,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch pHead = buffer; } - ret = taosRead(fd, pHead->cont, pHead->len); + ret = tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); @@ -332,7 +333,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } - tclose(fd); + tfClose(tfd); tfree(buffer); return code; From 754ad5ff5b6dae0bb0e85d44c67ff9febe00c63f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 23:38:07 +0800 Subject: [PATCH 74/75] compile error in windows --- src/util/inc/tfile.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index b32f297d9c..066040170e 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -35,7 +35,7 @@ int64_t tfWrite(int64_t tfd, void *buf, int64_t count); int64_t tfRead(int64_t tfd, void *buf, int64_t count); int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); -int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); int32_t tfFtruncate(int64_t tfd, int64_t length); #ifdef __cplusplus From a2331e5df637295ac558c0ac496a6370e7860e55 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 28 Nov 2020 09:47:07 +0800 Subject: [PATCH 75/75] compile in win64 --- src/util/src/tfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 49b28dad6f..1fa6a3d096 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -106,7 +106,7 @@ bool tfValid(int64_t tfd) { return p != NULL; } -int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1;