From 78f7134ef1547176034b97b248aa37f01ccfebf1 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 11 Apr 2020 00:09:37 +0800 Subject: [PATCH 01/36] [TD-52] rename some defs --- cmake/define.inc | 4 ++-- src/dnode/src/dnodeMgmt.c | 2 +- src/inc/mnode.h | 27 ++++++++-------------- src/{mnode/inc/mgmtMnode.h => inc/mpeer.h} | 18 +++++++-------- src/inc/taosdef.h | 4 ++-- src/inc/taosmsg.h | 2 +- src/mnode/src/mgmtDb.c | 4 ++-- src/mnode/src/mgmtDnode.c | 6 ++--- src/mnode/src/mgmtMain.c | 12 +++++----- src/mnode/src/mgmtMnode.c | 20 ++++++++-------- src/mnode/src/mgmtProfile.c | 8 +++---- src/mnode/src/mgmtShell.c | 14 +++++------ src/mnode/src/mgmtTable.c | 4 +--- src/mnode/src/mgmtUser.c | 8 +------ src/mnode/src/mgmtVgroup.c | 4 +--- 15 files changed, 58 insertions(+), 79 deletions(-) rename src/{mnode/inc/mgmtMnode.h => inc/mpeer.h} (74%) diff --git a/cmake/define.inc b/cmake/define.inc index e43e570979..5f17ee1216 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -11,9 +11,9 @@ ENDIF () IF (TD_VPEER) ADD_DEFINITIONS(-D_VPEER) - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) + #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) ELSE () - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) + #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) ENDIF () IF (TD_ACCOUNT) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index da80206e4c..0ec769c0af 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -45,7 +45,7 @@ static void *tsDnodeTmr = NULL; static void *tsStatusTimer = NULL; static uint32_t tsRebootTime; static int32_t tsDnodeId = 0; -static char tsDnodeName[TSDB_DNODE_NAME_LEN]; +static char tsDnodeName[TSDB_NODE_NAME_LEN]; int32_t dnodeInitMgmt() { dnodeReadDnodeId(); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index dec9292209..b19e91a90e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -41,28 +41,19 @@ struct _vg_obj; struct _db_obj; struct _acct_obj; struct _user_obj; +struct _mnode_obj; -typedef struct { +typedef struct _mnode_obj { int32_t mnodeId; - uint32_t privateIp; - uint32_t publicIp; + int32_t dnodeId; int64_t createdTime; - int64_t lostTime; - uint64_t dbVersion; - uint32_t rack; - uint16_t idc; - uint16_t slot; - int8_t role; - int8_t status; - int8_t numOfMnodes; - int32_t numOfDnodes; - char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - int8_t reserved[15]; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; - int syncFd; - void *hbTimer; - void *pSync; + int8_t role; + int8_t status; + uint32_t privateIp; + uint32_t publicIp; } SMnodeObj; typedef struct _dnode_obj { @@ -81,7 +72,7 @@ typedef struct _dnode_obj { int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t status; // set in balance function int32_t customScore; // config by user - char dnodeName[TSDB_DNODE_NAME_LEN + 1]; + char dnodeName[TSDB_NODE_NAME_LEN + 1]; int8_t reserved[15]; int8_t updateEnd[1]; int32_t refCount; diff --git a/src/mnode/inc/mgmtMnode.h b/src/inc/mpeer.h similarity index 74% rename from src/mnode/inc/mgmtMnode.h rename to src/inc/mpeer.h index ad9688c0ee..7007f19226 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/inc/mpeer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_MNODE_H -#define TDENGINE_MGMT_MNODE_H +#ifndef TDENGINE_MPEER_H +#define TDENGINE_MPEER_H #ifdef __cplusplus extern "C" { @@ -33,15 +33,15 @@ enum _TSDB_MN_ROLE { TSDB_MN_ROLE_MASTER }; -int32_t mgmtInitMnodes(); -void mgmtCleanupMnodes(); +int32_t mpeerInit(); +void mpeerCleanup(); -bool mgmtInServerStatus(); -bool mgmtIsMaster(); +bool mpeerInServerStatus(); +bool mpeerIsMaster(); -bool mgmtCheckRedirect(void *handle); -void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); -void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); +bool mpeerCheckRedirect(void *handle); +void mpeerGetPrivateIpList(SRpcIpSet *ipSet); +void mpeerGetPublicIpList(SRpcIpSet *ipSet); #ifdef __cplusplus } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index d6a9447e3d..85b31a2659 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COLUMNS 256 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns -#define TSDB_DNODE_NAME_LEN 64 +#define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 @@ -233,7 +233,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); /* * this is defined in CMakeList.txt */ -//#define TSDB_REPLICA_MAX_NUM 3 +#define TSDB_REPLICA_MAX_NUM 3 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f5168a2c9e..09d306ffad 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -579,7 +579,7 @@ typedef struct { typedef struct { uint32_t version; int32_t dnodeId; - char dnodeName[TSDB_DNODE_NAME_LEN]; + char dnodeName[TSDB_NODE_NAME_LEN + 1]; uint32_t privateIp; uint32_t publicIp; uint32_t lastReboot; // time stamp for last reboot diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 9ecc2c6458..0f18de3253 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -24,8 +24,8 @@ #include "mgmtDb.h" #include "tcluster.h" #include "tgrant.h" +#include "mpeer.h" #include "mgmtShell.h" -#include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtTable.h" @@ -678,7 +678,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 2f4abc5cfe..1d32ded088 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -19,8 +19,8 @@ #include "tbalance.h" #include "tcluster.h" #include "mnode.h" +#include "mpeer.h" #include "mgmtDClient.h" -#include "mgmtMnode.h" #include "mgmtShell.h" #include "mgmtDServer.h" #include "mgmtUser.h" @@ -141,7 +141,7 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { } void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; + if (mpeerCheckRedirect(rpcMsg->handle)) return; SDMStatusMsg *pStatus = rpcMsg->pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); @@ -221,7 +221,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodePrivateIpList(&pRsp->ipList); + mpeerGetPrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index e04630d745..cb3c9fae1e 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -22,11 +22,11 @@ #include "taccount.h" #include "tbalance.h" #include "tcluster.h" +#include "tgrant.h" +#include "mpeer.h" #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDServer.h" -#include "tgrant.h" -#include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" @@ -117,8 +117,8 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitMnodes() < 0) { - mError("failed to init mnodes"); + if (mpeerInit() < 0) { + mError("failed to init mpeers"); return -1; } @@ -135,7 +135,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (mgmtIsMaster()) { + if (mpeerIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -147,7 +147,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); grantCleanUp(); - mgmtCleanupMnodes(); + mpeerCleanup(); balanceCleanUp(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 1e2a4e9066..c349969019 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,7 +17,7 @@ #include "os.h" #include "taoserror.h" #include "trpc.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -28,7 +28,7 @@ static SMnodeObj tsMnodeObj = {0}; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -int32_t mgmtInitMnodes() { +int32_t mpeerInit() { mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); @@ -38,16 +38,14 @@ int32_t mgmtInitMnodes() { tsMnodeObj.createdTime = taosGetTimestampMs(); tsMnodeObj.role = TSDB_MN_ROLE_MASTER; tsMnodeObj.status = TSDB_MN_STATUS_SERVING; - tsMnodeObj.numOfMnodes = 1; - sprintf(tsMnodeObj.mnodeName, "%d", tsMnodeObj.mnodeId); - + return TSDB_CODE_SUCCESS; } -void mgmtCleanupMnodes() {} -bool mgmtInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } -bool mgmtIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } -bool mgmtCheckRedirect(void *thandle) { return false; } +void mpeerCleanup() {} +bool mpeerInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } +bool mpeerIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } +bool mpeerCheckRedirect(void *thandle) { return false; } static int32_t mgmtGetMnodesNum() { return 1; @@ -190,14 +188,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { +void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; ipSet->ip[0] = htonl(tsMnodeObj.privateIp); } -void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { +void mpeerGetPublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 2b22fae47a..36481b81b0 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -19,7 +19,7 @@ #include "taccount.h" #include "tcluster.h" #include "mgmtDb.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" @@ -681,7 +681,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { @@ -705,7 +705,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { @@ -729,7 +729,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 880c6d0c10..eb20225e95 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -27,7 +27,7 @@ #include "mgmtDb.h" #include "tcluster.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -142,14 +142,14 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } - if (mgmtCheckRedirect(rpcMsg->handle)) { + if (mpeerCheckRedirect(rpcMsg->handle)) { // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect()); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER); rpcFreeCont(rpcMsg->pCont); return; } - if (!mgmtInServerStatus()) { + if (!mpeerInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -337,9 +337,9 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { } if (pMsg->usePublicIp) { - mgmtGetMnodePublicIpList(&pHBRsp->ipList); + mpeerGetPublicIpList(&pHBRsp->ipList); } else { - mgmtGetMnodePrivateIpList(&pHBRsp->ipList); + mpeerGetPrivateIpList(&pHBRsp->ipList); } /* @@ -423,9 +423,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { pConnectRsp->superAuth = pUser->superAuth; if (pMsg->usePublicIp) { - mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + mpeerGetPublicIpList(&pConnectRsp->ipList); } else { - mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); + mpeerGetPrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index b536bb5ac9..8dfe835f32 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -35,7 +35,7 @@ #include "tcluster.h" #include "mgmtDServer.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -1650,8 +1650,6 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_ } static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; - SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; pCfg->dnode = htonl(pCfg->dnode); pCfg->vnode = htonl(pCfg->vnode); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index fe7d40e120..8e969b3c9c 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -20,7 +20,7 @@ #include "tutil.h" #include "taccount.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -314,8 +314,6 @@ SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) { } static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pUser = pMsg->pUser; @@ -333,8 +331,6 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { } static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pOperUser = pMsg->pUser; @@ -427,8 +423,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pOperUser = pMsg->pUser; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 6b27fbbc83..98ae422904 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -24,7 +24,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDServer.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -663,8 +663,6 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { } static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; - SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); From b9c4fc22654f6d7474348ed93f6a885a05b50459 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 00:42:21 +0800 Subject: [PATCH 02/36] change go driver to git submodule. use following commands to retreive: git clone https://github.com/taosdata/TDengine git checkout develop git pull git submodule update --init --recursive --- .gitmodules | 3 + src/connector/go | 1 + src/connector/go/src/taosSql/connection.go | 368 ------------------ src/connector/go/src/taosSql/connector.go | 49 --- src/connector/go/src/taosSql/const.go | 51 --- src/connector/go/src/taosSql/driver.go | 43 --- src/connector/go/src/taosSql/dsn.go | 202 ---------- src/connector/go/src/taosSql/result.go | 200 ---------- src/connector/go/src/taosSql/rows.go | 296 --------------- src/connector/go/src/taosSql/statement.go | 158 -------- src/connector/go/src/taosSql/taosLog.go | 120 ------ src/connector/go/src/taosSql/taosSqlCgo.go | 84 ---- src/connector/go/src/taosSql/utils.go | 422 --------------------- 13 files changed, 4 insertions(+), 1993 deletions(-) create mode 100644 .gitmodules create mode 160000 src/connector/go delete mode 100755 src/connector/go/src/taosSql/connection.go delete mode 100755 src/connector/go/src/taosSql/connector.go delete mode 100755 src/connector/go/src/taosSql/const.go delete mode 100755 src/connector/go/src/taosSql/driver.go delete mode 100755 src/connector/go/src/taosSql/dsn.go delete mode 100755 src/connector/go/src/taosSql/result.go delete mode 100755 src/connector/go/src/taosSql/rows.go delete mode 100755 src/connector/go/src/taosSql/statement.go delete mode 100755 src/connector/go/src/taosSql/taosLog.go delete mode 100755 src/connector/go/src/taosSql/taosSqlCgo.go delete mode 100755 src/connector/go/src/taosSql/utils.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..35aea3658b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/connector/go"] + path = src/connector/go + url = https://github.com/taosdata/driver-go diff --git a/src/connector/go b/src/connector/go new file mode 160000 index 0000000000..8c58c512b6 --- /dev/null +++ b/src/connector/go @@ -0,0 +1 @@ +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/go/src/taosSql/connection.go b/src/connector/go/src/taosSql/connection.go deleted file mode 100755 index 2a3f5a9acd..0000000000 --- a/src/connector/go/src/taosSql/connection.go +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import "C" -import ( - "context" - "errors" - "database/sql/driver" - "unsafe" - "strconv" - "strings" - "time" -) - -type taosConn struct { - taos unsafe.Pointer - affectedRows int - insertId int - cfg *config - status statusFlag - parseTime bool - reset bool // set when the Go SQL package calls ResetSession -} - -type taosSqlResult struct { - affectedRows int64 - insertId int64 -} - -func (res *taosSqlResult) LastInsertId() (int64, error) { - return res.insertId, nil -} - -func (res *taosSqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil -} - -func (mc *taosConn) Begin() (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) Close() (err error) { - if mc.taos == nil { - return errConnNoExist - } - mc.taos_close() - return nil -} - -func (mc *taosConn) Prepare(query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt := &taosSqlStmt{ - mc: mc, - pSql: query, - } - - // find ? count and save to stmt.paramCount - stmt.paramCount = strings.Count(query, "?") - - //fmt.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - taosLog.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - - return stmt, nil -} - -func (mc *taosConn) interpolateParams(query string, args []driver.Value) (string, error) { - // Number of ? should be same to len(args) - if strings.Count(query, "?") != len(args) { - return "", driver.ErrSkip - } - - buf := make([]byte, defaultBufSize) - buf = buf[:0] // clear buf - argPos := 0 - - for i := 0; i < len(query); i++ { - q := strings.IndexByte(query[i:], '?') - if q == -1 { - buf = append(buf, query[i:]...) - break - } - buf = append(buf, query[i:i+q]...) - i += q - - arg := args[argPos] - argPos++ - - if arg == nil { - buf = append(buf, "NULL"...) - continue - } - - switch v := arg.(type) { - case int64: - buf = strconv.AppendInt(buf, v, 10) - case uint64: - // Handle uint64 explicitly because our custom ConvertValue emits unsigned values - buf = strconv.AppendUint(buf, v, 10) - case float64: - buf = strconv.AppendFloat(buf, v, 'g', -1, 64) - case bool: - if v { - buf = append(buf, '1') - } else { - buf = append(buf, '0') - } - case time.Time: - if v.IsZero() { - buf = append(buf, "'0000-00-00'"...) - } else { - v := v.In(mc.cfg.loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) - } - buf = append(buf, '\'') - } - case []byte: - if v == nil { - buf = append(buf, "NULL"...) - } else { - buf = append(buf, "_binary'"...) - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeBytesBackslash(buf, v) - } else { - buf = escapeBytesQuotes(buf, v) - } - buf = append(buf, '\'') - } - case string: - //buf = append(buf, '\'') - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeStringBackslash(buf, v) - } else { - buf = escapeStringQuotes(buf, v) - } - //buf = append(buf, '\'') - default: - return "", driver.ErrSkip - } - - //if len(buf)+4 > mc.maxAllowedPacket { - if len(buf)+4 > maxTaosSqlLen { - return "", driver.ErrSkip - } - } - if argPos != len(args) { - return "", driver.ErrSkip - } - return string(buf), nil -} - -func (mc *taosConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - mc.affectedRows = 0 - mc.insertId = 0 - _, err := mc.taosQuery(query) - if err == nil { - return &taosSqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err - } - - return nil, err -} - -func (mc *taosConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return mc.query(query, args) -} - -func (mc *taosConn) query(query string, args []driver.Value) (*textRows, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - num_fields, err := mc.taosQuery(query) - if err == nil { - // Read Result - rows := new(textRows) - rows.mc = mc - - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -// Ping implements driver.Pinger interface -func (mc *taosConn) Ping(ctx context.Context) (err error) { - if mc.taos != nil { - return nil - } - return errInvalidConn -} - -// BeginTx implements driver.ConnBeginTx interface -func (mc *taosConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - rows, err := mc.query(query, dargs) - if err != nil { - return nil, err - } - - return rows, err -} - -func (mc *taosConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return mc.Exec(query, dargs) -} - -func (mc *taosConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt, err := mc.Prepare(query) - if err != nil { - return nil, err - } - - return stmt, nil -} - -func (stmt *taosSqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - dargs, err := namedValueToValue(args) - - if err != nil { - return nil, err - } - - rows, err := stmt.query(dargs) - if err != nil { - return nil, err - } - return rows, err -} - -func (stmt *taosSqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return stmt.Exec(dargs) -} - -func (mc *taosConn) CheckNamedValue(nv *driver.NamedValue) (err error) { - nv.Value, err = converter{}.ConvertValue(nv.Value) - return -} - -// ResetSession implements driver.SessionResetter. -// (From Go 1.10) -func (mc *taosConn) ResetSession(ctx context.Context) error { - if mc.taos == nil { - return driver.ErrBadConn - } - mc.reset = true - return nil -} diff --git a/src/connector/go/src/taosSql/connector.go b/src/connector/go/src/taosSql/connector.go deleted file mode 100755 index 55715b949e..0000000000 --- a/src/connector/go/src/taosSql/connector.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql/driver" -) - -type connector struct { - cfg *config -} - -// Connect implements driver.Connector interface. -// Connect returns a connection to the database. -func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { - var err error - // New taosConn - mc := &taosConn{ - cfg: c.cfg, - parseTime: c.cfg.parseTime, - } - - // Connect to Server - mc.taos, err = mc.taosConnect(mc.cfg.addr, mc.cfg.user, mc.cfg.passwd, mc.cfg.dbName, mc.cfg.port) - if err != nil { - return nil, err - } - - return mc, nil -} - -// Driver implements driver.Connector interface. -// Driver returns &taosSQLDriver{}. -func (c *connector) Driver() driver.Driver { - return &taosSQLDriver{} -} diff --git a/src/connector/go/src/taosSql/const.go b/src/connector/go/src/taosSql/const.go deleted file mode 100755 index 89214a0d45..0000000000 --- a/src/connector/go/src/taosSql/const.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -const ( - timeFormat = "2006-01-02 15:04:05" - maxTaosSqlLen = 65380 - defaultBufSize = maxTaosSqlLen + 32 -) - -type fieldType byte - -type fieldFlag uint16 - -const ( - flagNotNULL fieldFlag = 1 << iota -) - -type statusFlag uint16 - -const ( - statusInTrans statusFlag = 1 << iota - statusInAutocommit - statusReserved // Not in documentation - statusMoreResultsExists - statusNoGoodIndexUsed - statusNoIndexUsed - statusCursorExists - statusLastRowSent - statusDbDropped - statusNoBackslashEscapes - statusMetadataChanged - statusQueryWasSlow - statusPsOutParams - statusInTransReadonly - statusSessionStateChanged -) - diff --git a/src/connector/go/src/taosSql/driver.go b/src/connector/go/src/taosSql/driver.go deleted file mode 100755 index 6839979c61..0000000000 --- a/src/connector/go/src/taosSql/driver.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql" - "database/sql/driver" -) - -// taosSqlDriver is exported to make the driver directly accessible. -// In general the driver is used via the database/sql package. -type taosSQLDriver struct{} - -// Open new Connection. -// the DSN string is formatted -func (d taosSQLDriver) Open(dsn string) (driver.Conn, error) { - cfg, err := parseDSN(dsn) - if err != nil { - return nil, err - } - c := &connector{ - cfg: cfg, - } - return c.Connect(context.Background()) -} - -func init() { - sql.Register("taosSql", &taosSQLDriver{}) - taosLogInit() -} diff --git a/src/connector/go/src/taosSql/dsn.go b/src/connector/go/src/taosSql/dsn.go deleted file mode 100755 index c68ae1725c..0000000000 --- a/src/connector/go/src/taosSql/dsn.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "errors" - "net/url" - "strconv" - "strings" - "time" -) - -var ( - errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") - errInvalidDSNPort = errors.New("invalid DSN: network port is not a valid number") - errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") -) - -// Config is a configuration parsed from a DSN string. -// If a new Config is created instead of being parsed from a DSN string, -// the NewConfig function should be used, which sets default values. -type config struct { - user string // Username - passwd string // Password (requires User) - net string // Network type - addr string // Network address (requires Net) - port int - dbName string // Database name - params map[string]string // Connection parameters - loc *time.Location // Location for time.Time values - columnsWithAlias bool // Prepend table alias to column names - interpolateParams bool // Interpolate placeholders into query string - parseTime bool // Parse time values to time.Time -} - -// NewConfig creates a new Config and sets default values. -func newConfig() *config { - return &config{ - loc: time.UTC, - interpolateParams: true, - parseTime: true, - } -} - -// ParseDSN parses the DSN string to a Config -func parseDSN(dsn string) (cfg *config, err error) { - taosLog.Println("input dsn:", dsn) - - // New config with some default values - cfg = newConfig() - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - strs := strings.Split(dsn[k+1:i-1], ":") - if len(strs) == 1 { - return nil, errInvalidDSNAddr - } - cfg.addr = strs[0] - cfg.port, err = strconv.Atoi(strs[1]) - if err != nil { - return nil, errInvalidDSNPort - } - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbName = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - taosLog.Printf("cfg info: %+v", cfg) - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // time.Time parsing - case "parseTime": - var isBool bool - cfg.parseTime, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return -} diff --git a/src/connector/go/src/taosSql/result.go b/src/connector/go/src/taosSql/result.go deleted file mode 100755 index 2367560c31..0000000000 --- a/src/connector/go/src/taosSql/result.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql/driver" - "errors" - "fmt" - "io" - "strconv" - "time" - "unsafe" -) - -/****************************************************************************** -* Result * -******************************************************************************/ -// Read Packets as Field Packets until EOF-Packet or an Error appears -func (mc *taosConn) readColumns(count int) ([]taosSqlField, error) { - - columns := make([]taosSqlField, count) - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return nil, errors.New("invalid result") - } - - pFields := (*C.struct_taosField)(C.taos_fetch_fields(result)) - - // TODO: Optimized rewriting !!!! - fields := (*[1 << 30]C.struct_taosField)(unsafe.Pointer(pFields)) - - for i := 0; i < count; i++ { - //columns[i].tableName = ms.taos. - //fmt.Println(reflect.TypeOf(fields[i].name)) - var charray []byte - for j := range fields[i].name { - //fmt.Println("fields[i].name[j]: ", fields[i].name[j]) - if fields[i].name[j] != 0 { - charray = append(charray, byte(fields[i].name[j])) - } else { - break - } - } - columns[i].name = string(charray) - columns[i].length = (uint32)(fields[i].bytes) - columns[i].fieldType = fieldType(fields[i]._type) - columns[i].flags = 0 - // columns[i].decimals = 0 - //columns[i].charSet = 0 - } - return columns, nil -} - -func (rows *taosSqlRows) readRow(dest []driver.Value) error { - mc := rows.mc - - if rows.rs.done || mc == nil { - return io.EOF - } - - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return errors.New(C.GoString(C.taos_errstr(mc.taos))) - } - - //var row *unsafe.Pointer - row := C.taos_fetch_row(result) - if row == nil { - rows.rs.done = true - C.taos_free_result(result) - rows.mc = nil - return io.EOF - } - - // because sizeof(void*) == sizeof(int*) == 8 - // notes: sizeof(int) == 8 in go, but sizeof(int) == 4 in C. - for i := range dest { - currentRow := (unsafe.Pointer)(uintptr(*((*int)(unsafe.Pointer(uintptr(unsafe.Pointer(row)) + uintptr(i)*unsafe.Sizeof(int(0))))))) - - if currentRow == nil { - dest[i] = nil - continue - } - - switch rows.rs.columns[i].fieldType { - case C.TSDB_DATA_TYPE_BOOL: - if (*((*byte)(currentRow))) != 0 { - dest[i] = true - } else { - dest[i] = false - } - break - - case C.TSDB_DATA_TYPE_TINYINT: - dest[i] = (int)(*((*byte)(currentRow))) - break - - case C.TSDB_DATA_TYPE_SMALLINT: - dest[i] = (int16)(*((*int16)(currentRow))) - break - - case C.TSDB_DATA_TYPE_INT: - dest[i] = (int)(*((*int32)(currentRow))) // notes int32 of go <----> int of C - break - - case C.TSDB_DATA_TYPE_BIGINT: - dest[i] = (int64)(*((*int64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_FLOAT: - dest[i] = (*((*float32)(currentRow))) - break - - case C.TSDB_DATA_TYPE_DOUBLE: - dest[i] = (*((*float64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_BINARY, C.TSDB_DATA_TYPE_NCHAR: - charLen := rows.rs.columns[i].length - var index uint32 - binaryVal := make([]byte, charLen) - for index = 0; index < charLen; index++ { - binaryVal[index] = *((*byte)(unsafe.Pointer(uintptr(currentRow) + uintptr(index)))) - } - dest[i] = string(binaryVal[:]) - break - - case C.TSDB_DATA_TYPE_TIMESTAMP: - if mc.cfg.parseTime == true { - timestamp := (int64)(*((*int64)(currentRow))) - dest[i] = timestampConvertToString(timestamp, int(C.taos_result_precision(result))) - } else { - dest[i] = (int64)(*((*int64)(currentRow))) - } - break - - default: - fmt.Println("default fieldType: set dest[] to nil") - dest[i] = nil - break - } - } - - return nil -} - -// Read result as Field format until all rows or an Error appears -// call this func in conn mode -func (rows *textRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -// call thsi func in stmt mode -func (rows *binaryRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -func timestampConvertToString(timestamp int64, precision int) string { - var decimal, sVal, nsVal int64 - if precision == 0 { - decimal = timestamp % 1000 - sVal = timestamp / 1000 - nsVal = decimal * 1000 - } else { - decimal = timestamp % 1000000 - sVal = timestamp / 1000000 - nsVal = decimal * 1000000 - } - - date_time := time.Unix(sVal, nsVal) - - //const base_format = "2006-01-02 15:04:05" - str_time := date_time.Format(timeFormat) - - return (str_time + "." + strconv.Itoa(int(decimal))) -} diff --git a/src/connector/go/src/taosSql/rows.go b/src/connector/go/src/taosSql/rows.go deleted file mode 100755 index 5040dca06d..0000000000 --- a/src/connector/go/src/taosSql/rows.go +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql" - "database/sql/driver" - "io" - "math" - "reflect" -) - -type taosSqlField struct { - tableName string - name string - length uint32 - flags fieldFlag // indicate whether this field can is null - fieldType fieldType - decimals byte - charSet uint8 -} - -type resultSet struct { - columns []taosSqlField - columnNames []string - done bool -} - -type taosSqlRows struct { - mc *taosConn - rs resultSet -} - -type binaryRows struct { - taosSqlRows -} - -type textRows struct { - taosSqlRows -} - -func (rows *taosSqlRows) Columns() []string { - if rows.rs.columnNames != nil { - return rows.rs.columnNames - } - - columns := make([]string, len(rows.rs.columns)) - if rows.mc != nil && rows.mc.cfg.columnsWithAlias { - for i := range columns { - if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.rs.columns[i].name - } else { - columns[i] = rows.rs.columns[i].name - } - } - } else { - for i := range columns { - columns[i] = rows.rs.columns[i].name - } - } - - rows.rs.columnNames = columns - - return columns -} - -func (rows *taosSqlRows) ColumnTypeDatabaseTypeName(i int) string { - return rows.rs.columns[i].typeDatabaseName() -} - -func (rows *taosSqlRows) ColumnTypeLength(i int) (length int64, ok bool) { - return int64(rows.rs.columns[i].length), true -} - -func (rows *taosSqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { - return rows.rs.columns[i].flags&flagNotNULL == 0, true -} - -func (rows *taosSqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { - column := rows.rs.columns[i] - decimals := int64(column.decimals) - - switch column.fieldType { - case C.TSDB_DATA_TYPE_FLOAT: - fallthrough - case C.TSDB_DATA_TYPE_DOUBLE: - if decimals == 0x1f { - return math.MaxInt64, math.MaxInt64, true - } - return math.MaxInt64, decimals, true - } - - return 0, 0, false -} - -func (rows *taosSqlRows) ColumnTypeScanType(i int) reflect.Type { - return rows.rs.columns[i].scanType() -} - -func (rows *taosSqlRows) Close() error { - if rows.mc != nil { - result := C.taos_use_result(rows.mc.taos) - if result != nil { - C.taos_free_result(result) - } - rows.mc = nil - } - return nil -} - -func (rows *taosSqlRows) HasNextResultSet() (b bool) { - if rows.mc == nil { - return false - } - return rows.mc.status&statusMoreResultsExists != 0 -} - -func (rows *taosSqlRows) nextResultSet() (int, error) { - if rows.mc == nil { - return 0, io.EOF - } - - // Remove unread packets from stream - if !rows.rs.done { - rows.rs.done = true - } - - if !rows.HasNextResultSet() { - rows.mc = nil - return 0, io.EOF - } - rows.rs = resultSet{} - return 0,nil -} - -func (rows *taosSqlRows) nextNotEmptyResultSet() (int, error) { - for { - resLen, err := rows.nextResultSet() - if err != nil { - return 0, err - } - - if resLen > 0 { - return resLen, nil - } - - rows.rs.done = true - } -} - -func (rows *binaryRows) NextResultSet() error { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// stmt.Query return binary rows, and get row from this func -func (rows *binaryRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (rows *textRows) NextResultSet() (err error) { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// db.Query return text rows, and get row from this func -func (rows *textRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (mf *taosSqlField) typeDatabaseName() string { - //fmt.Println("######## (mf *taosSqlField) typeDatabaseName() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return "BOOL" - - case C.TSDB_DATA_TYPE_TINYINT: - return "TINYINT" - - case C.TSDB_DATA_TYPE_SMALLINT: - return "SMALLINT" - - case C.TSDB_DATA_TYPE_INT: - return "INT" - - case C.TSDB_DATA_TYPE_BIGINT: - return "BIGINT" - - case C.TSDB_DATA_TYPE_FLOAT: - return "FLOAT" - - case C.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE" - - case C.TSDB_DATA_TYPE_BINARY: - return "BINARY" - - case C.TSDB_DATA_TYPE_NCHAR: - return "NCHAR" - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP" - - default: - return "" - } -} - -var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullTime = reflect.TypeOf(NullTime{}) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) -) - -func (mf *taosSqlField) scanType() reflect.Type { - //fmt.Println("######## (mf *taosSqlField) scanType() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_TINYINT: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_SMALLINT: - return scanTypeInt16 - - case C.TSDB_DATA_TYPE_INT: - return scanTypeInt32 - - case C.TSDB_DATA_TYPE_BIGINT: - return scanTypeInt64 - - case C.TSDB_DATA_TYPE_FLOAT: - return scanTypeFloat32 - - case C.TSDB_DATA_TYPE_DOUBLE: - return scanTypeFloat64 - - case C.TSDB_DATA_TYPE_BINARY: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_NCHAR: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return scanTypeNullTime - - default: - return scanTypeUnknown - } -} diff --git a/src/connector/go/src/taosSql/statement.go b/src/connector/go/src/taosSql/statement.go deleted file mode 100755 index 3aa3e01564..0000000000 --- a/src/connector/go/src/taosSql/statement.go +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "database/sql/driver" - "fmt" - "reflect" -) - -type taosSqlStmt struct { - mc *taosConn - id uint32 - pSql string - paramCount int -} - -func (stmt *taosSqlStmt) Close() error { - return nil -} - -func (stmt *taosSqlStmt) NumInput() int { - return stmt.paramCount -} - -func (stmt *taosSqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.mc.Exec(stmt.pSql, args) -} - -func (stmt *taosSqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.query(args) -} - -func (stmt *taosSqlStmt) query(args []driver.Value) (*binaryRows, error) { - mc := stmt.mc - if mc == nil || mc.taos == nil { - return nil, errInvalidConn - } - - querySql := stmt.pSql - - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(stmt.pSql, args) - if err != nil { - return nil, err - } - querySql = prepared - } - - num_fields, err := mc.taosQuery(querySql) - if err == nil { - // Read Result - rows := new(binaryRows) - rows.mc = mc - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -type converter struct{} - -// ConvertValue mirrors the reference/default converter in database/sql/driver -// with _one_ exception. We support uint64 with their high bit and the default -// implementation does not. This function should be kept in sync with -// database/sql/driver defaultConverter.ConvertValue() except for that -// deliberate difference. -func (c converter) ConvertValue(v interface{}) (driver.Value, error) { - - if driver.IsValue(v) { - return v, nil - } - - if vr, ok := v.(driver.Valuer); ok { - sv, err := callValuerValue(vr) - if err != nil { - return nil, err - } - if !driver.IsValue(sv) { - return nil, fmt.Errorf("non-Value type %T returned from Value", sv) - } - - return sv, nil - } - - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Ptr: - // indirect pointers - if rv.IsNil() { - return nil, nil - } else { - return c.ConvertValue(rv.Elem().Interface()) - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return rv.Uint(), nil - case reflect.Float32, reflect.Float64: - return rv.Float(), nil - case reflect.Bool: - return rv.Bool(), nil - case reflect.Slice: - ek := rv.Type().Elem().Kind() - if ek == reflect.Uint8 { - return rv.Bytes(), nil - } - return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) - case reflect.String: - return rv.String(), nil - } - return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) -} - -var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() - -// callValuerValue returns vr.Value(), with one exception: -// If vr.Value is an auto-generated method on a pointer type and the -// pointer is nil, it would panic at runtime in the panicwrap -// method. Treat it like nil instead. -// -// This is so people can implement driver.Value on value types and -// still use nil pointers to those types to mean nil/NULL, just like -// string/*string. -// -// This is an exact copy of the same-named unexported function from the -// database/sql package. -func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { - if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && - rv.IsNil() && - rv.Type().Elem().Implements(valuerReflectType) { - return nil, nil - } - return vr.Value() -} diff --git a/src/connector/go/src/taosSql/taosLog.go b/src/connector/go/src/taosSql/taosLog.go deleted file mode 100755 index 63af5e34c1..0000000000 --- a/src/connector/go/src/taosSql/taosLog.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "bufio" - "errors" - "fmt" - "io" - "log" - "os" - "strings" -) - -// Various errors the driver might return. -var ( - errInvalidConn = errors.New("invalid connection") - errConnNoExist = errors.New("no existent connection ") -) - -var taosLog *log.Logger - -// SetLogger is used to set the logger for critical errors. -// The initial logger -func taosLogInit() { - cfgName := "/etc/taos/taos.cfg" - logNameDefault := "/var/log/taos/taosgo.log" - var logName string - - // get log path from cfg file - cfgFile, err := os.OpenFile(cfgName, os.O_RDONLY, 0644) - defer cfgFile.Close() - if err != nil { - fmt.Println(err) - logName = logNameDefault - } else { - logName, err = getLogNameFromCfg(cfgFile) - if err != nil { - fmt.Println(err) - logName = logNameDefault - } - } - - logFile, err := os.OpenFile(logName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - taosLog = log.New(logFile, "", log.LstdFlags) - taosLog.SetPrefix("TAOS DRIVER ") - taosLog.SetFlags(log.LstdFlags|log.Lshortfile) -} - -func getLogNameFromCfg(f *os.File) (string, error) { - // Create file buf, *Reader - r := bufio.NewReader(f) - for { - //read one line, return to slice b - b, _, err := r.ReadLine() - if err != nil { - if err == io.EOF { - break - } - panic(err) - } - - // Remove space of left and right - s := strings.TrimSpace(string(b)) - if strings.Index(s, "#") == 0 { - // comment line - continue - } - - if len(s) == 0 { - continue - } - - var ns string - // If there is a comment on the right of the line, must be remove - index := strings.Index(s, "#") - if index > 0 { - // Gets the string to the left of the comment to determine whether it is empty - ns = s[:index] - if len(ns) == 0 { - continue - } - } else { - ns = s; - } - - ss := strings.Fields(ns) - if strings.Compare("logDir", ss[0]) != 0 { - continue - } - - if len(ss) < 2 { - break - } - - // Add a filename after the path - logName := ss[1] + "/taosgo.log" - return logName,nil - } - - return "", errors.New("no config log path, use default") -} - diff --git a/src/connector/go/src/taosSql/taosSqlCgo.go b/src/connector/go/src/taosSql/taosSqlCgo.go deleted file mode 100755 index cc3aaa1658..0000000000 --- a/src/connector/go/src/taosSql/taosSqlCgo.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "errors" - "unsafe" -) - -func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) { - cuser := C.CString(user) - cpass := C.CString(pass) - cip := C.CString(ip) - cdb := C.CString(db) - defer C.free(unsafe.Pointer(cip)) - defer C.free(unsafe.Pointer(cuser)) - defer C.free(unsafe.Pointer(cpass)) - defer C.free(unsafe.Pointer(cdb)) - - taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.ushort)(port)) - if taosObj == nil { - return nil, errors.New("taos_connect() fail!") - } - - return (unsafe.Pointer)(taosObj), nil -} - -func (mc *taosConn) taosQuery(sqlstr string) (int, error) { - //taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - - csqlstr := C.CString(sqlstr) - defer C.free(unsafe.Pointer(csqlstr)) - code := int(C.taos_query(mc.taos, csqlstr)) - - if 0 != code { - mc.taos_error() - errStr := C.GoString(C.taos_errstr(mc.taos)) - taosLog.Println("taos_query() failed:", errStr) - taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - return 0, errors.New(errStr) - } - - // read result and save into mc struct - num_fields := int(C.taos_field_count(mc.taos)) - if 0 == num_fields { // there are no select and show kinds of commands - mc.affectedRows = int(C.taos_affected_rows(mc.taos)) - mc.insertId = 0 - } - - return num_fields, nil -} - -func (mc *taosConn) taos_close() { - C.taos_close(mc.taos) -} - -func (mc *taosConn) taos_error() { - // free local resouce: allocated memory/metric-meta refcnt - //var pRes unsafe.Pointer - pRes := C.taos_use_result(mc.taos) - C.taos_free_result(pRes) -} diff --git a/src/connector/go/src/taosSql/utils.go b/src/connector/go/src/taosSql/utils.go deleted file mode 100755 index a104322fcc..0000000000 --- a/src/connector/go/src/taosSql/utils.go +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#include -#cgo LDFLAGS: -L/usr/lib -ltaos -void taosSetAllocMode(int mode, const char* path, _Bool autoDump); -void taosDumpMemoryLeak(); -*/ -import "C" - - -import ( - "database/sql/driver" - "errors" - "fmt" - "sync/atomic" - "time" - "unsafe" -) - -// Returns the bool value of the input. -// The 2nd return value indicates if the input was a valid bool value -func readBool(input string) (value bool, valid bool) { - switch input { - case "1", "true", "TRUE", "True": - return true, true - case "0", "false", "FALSE", "False": - return false, true - } - - // Not a valid bool value - return -} - -/****************************************************************************** -* Time related utils * -******************************************************************************/ - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { - case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return - } - t, err = time.Parse(timeFormat[:len(str)], str) - default: - err = fmt.Errorf("invalid time string: %s", str) - return - } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return -} - -// zeroDateTime is used in formatBinaryDateTime to avoid an allocation -// if the DATE or DATETIME has the zero value. -// It must never be changed. -// The current behavior depends on database/sql copying the result. -var zeroDateTime = []byte("0000-00-00 00:00:00.000000") - -const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" -const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" - -/****************************************************************************** -* Convert from and to bytes * -******************************************************************************/ - -func uint64ToBytes(n uint64) []byte { - return []byte{ - byte(n), - byte(n >> 8), - byte(n >> 16), - byte(n >> 24), - byte(n >> 32), - byte(n >> 40), - byte(n >> 48), - byte(n >> 56), - } -} - -func uint64ToString(n uint64) []byte { - var a [20]byte - i := 20 - - // U+0030 = 0 - // ... - // U+0039 = 9 - - var q uint64 - for n >= 10 { - i-- - q = n / 10 - a[i] = uint8(n-q*10) + 0x30 - n = q - } - - i-- - a[i] = uint8(n) + 0x30 - - return a[i:] -} - -// treats string value as unsigned integer representation -func stringToInt(b []byte) int { - val := 0 - for i := range b { - val *= 10 - val += int(b[i] - 0x30) - } - return val -} - -// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. -// If cap(buf) is not enough, reallocate new buffer. -func reserveBuffer(buf []byte, appendSize int) []byte { - newSize := len(buf) + appendSize - if cap(buf) < newSize { - // Grow buffer exponentially - newBuf := make([]byte, len(buf)*2+appendSize) - copy(newBuf, buf) - buf = newBuf - } - return buf[:newSize] -} - -// escapeBytesBackslash escapes []byte with backslashes (\) -// This escapes the contents of a string (provided as []byte) by adding backslashes before special -// characters, and turning others into specific escape sequences, such as -// turning newlines into \n and null bytes into \0. -func escapeBytesBackslash(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - for _, c := range v { - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - case '\'': - buf[pos] = '\\' - buf[pos+1] = '\'' - pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - return buf[:pos] -} - -// escapeStringBackslash is similar to escapeBytesBackslash but for string. -func escapeStringBackslash(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - //case '\'': - // buf[pos] = '\\' - // buf[pos+1] = '\'' - // pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeBytesQuotes escapes apostrophes in []byte by doubling them up. -// This escapes the contents of a string by doubling up any apostrophes that -// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in -// effect on the server. -func escapeBytesQuotes(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for _, c := range v { - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeStringQuotes is similar to escapeBytesQuotes but for string. -func escapeStringQuotes(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -/****************************************************************************** -* Sync utils * -******************************************************************************/ - -// noCopy may be embedded into structs which must not be copied -// after the first use. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 -// for details. -type noCopy struct{} - -// Lock is a no-op used by -copylocks checker from `go vet`. -func (*noCopy) Lock() {} - -// atomicBool is a wrapper around uint32 for usage as a boolean value with -// atomic access. -type atomicBool struct { - _noCopy noCopy - value uint32 -} - -// IsSet returns whether the current boolean value is true -func (ab *atomicBool) IsSet() bool { - return atomic.LoadUint32(&ab.value) > 0 -} - -// Set sets the value of the bool regardless of the previous value -func (ab *atomicBool) Set(value bool) { - if value { - atomic.StoreUint32(&ab.value, 1) - } else { - atomic.StoreUint32(&ab.value, 0) - } -} - -// TrySet sets the value of the bool and returns whether the value changed -func (ab *atomicBool) TrySet(value bool) bool { - if value { - return atomic.SwapUint32(&ab.value, 1) == 0 - } - return atomic.SwapUint32(&ab.value, 0) > 0 -} - -// atomicError is a wrapper for atomically accessed error values -type atomicError struct { - _noCopy noCopy - value atomic.Value -} - -// Set sets the error value regardless of the previous value. -// The value must not be nil -func (ae *atomicError) Set(value error) { - ae.value.Store(value) -} - -// Value returns the current error value -func (ae *atomicError) Value() error { - if v := ae.value.Load(); v != nil { - // this will panic if the value doesn't implement the error interface - return v.(error) - } - return nil -} - -func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { - dargs := make([]driver.Value, len(named)) - for n, param := range named { - if len(param.Name) > 0 { - // TODO: support the use of Named Parameters #561 - return nil, errors.New("taosSql: driver does not support the use of Named Parameters") - } - dargs[n] = param.Value - } - return dargs, nil -} - - -/****************************************************************************** -* Utils for C memory issues debugging * -******************************************************************************/ -func SetAllocMode(mode int32, path string) { - cpath := C.CString(path) - defer C.free(unsafe.Pointer(cpath)) - C.taosSetAllocMode(C.int(mode), cpath, false) -} - -func DumpMemoryLeak() { - C.taosDumpMemoryLeak() -} From 9d13fd243bcd9c7fa0ece51b2b7bcb4939697f19 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 04:15:36 +0800 Subject: [PATCH 03/36] Upgrade Python to 3 instead of 2 for testcase development. [TD-139] --- tests/pytest/simpletest.sh | 2 +- tests/pytest/test.py | 1 - tests/pytest/util/log.py | 14 +++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index aab36884f3..0a905beaa1 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1 +1 @@ -sudo python2 ./test.py -f insert/basic.py +sudo python3 ./test.py -f insert/basic.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py index b88e444665..0a524b44fa 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -73,7 +73,6 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdLog.info("CBD LN78: %s" % (fileName)) tdCases.runOneLinux(conn, fileName) conn.close() else: diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 926e582448..e570af85f5 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -22,27 +22,27 @@ class TDLog: self.path = "" def info(self, info): - print "%s %s" % (datetime.datetime.now(), info) + print("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print "%s sleep %d seconds" % (datetime.datetime.now(), sec) + print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print "\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print "\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info) + print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print "\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print "\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) def printNoPrefix(self, info): - print "\033[1;36m%s\033[0m" % (info) + print("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog() From e7f46e4a1ca81a0e090e83a0eda617a74705b6ab Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 15:13:08 +0800 Subject: [PATCH 04/36] add simplest python testcase for 2.0 [TD-128] --- .gitignore | 1 - .travis.yml | 46 ++++- tests/pytest/insert/__init__.py | 0 tests/pytest/insert/basic.py | 53 +++++ tests/pytest/simpletest.sh | 1 + tests/pytest/test.py | 87 +++++++++ tests/pytest/util/__init__.py | 0 tests/pytest/util/cases.py | 103 ++++++++++ tests/pytest/util/dnodes.py | 332 ++++++++++++++++++++++++++++++++ tests/pytest/util/log.py | 48 +++++ tests/pytest/util/sql.py | 135 +++++++++++++ 11 files changed, 801 insertions(+), 5 deletions(-) create mode 100644 tests/pytest/insert/__init__.py create mode 100644 tests/pytest/insert/basic.py create mode 100755 tests/pytest/simpletest.sh create mode 100644 tests/pytest/test.py create mode 100644 tests/pytest/util/__init__.py create mode 100644 tests/pytest/util/cases.py create mode 100644 tests/pytest/util/dnodes.py create mode 100644 tests/pytest/util/log.py create mode 100644 tests/pytest/util/sql.py diff --git a/.gitignore b/.gitignore index f2c1cb75b3..e91d6739a5 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ tests/test/ tests/taoshebei/ tests/taoscsv/ tests/taosdalipu/ -tests/pytest/ tests/jenkins/ tests/hdfs/ *.iml diff --git a/.travis.yml b/.travis.yml index 63c62d1a8a..221a1f1a30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,15 @@ os: - linux # - osx +before_install: + - |- + case $TRAVIS_OS_NAME in + linux) + sudo apt -y update + sudo apt -y install python-pip python3-pip python-setuptools python3-setuptools + ;; + esac + addons: coverity_scan: @@ -50,6 +59,15 @@ script: - |- case $TRAVIS_OS_NAME in linux) + # Color setting + RED='\033[0;31m' + GREEN='\033[1;32m' + GREEN_DARK='\033[0;32m' + GREEN_UNDERLINE='\033[4;32m' + NC='\033[0m' + + sudo make install + cd ../tests/script sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt @@ -57,16 +75,32 @@ script: if [ "$total_success" -gt "0" ]; then total_success=`expr $total_success - 1` + echo -e "${GREEN} ### Total $total_success TSIM case(s) succeed! ### ${NC}" fi - echo "Total $total_success success" - total_failed=`grep failed out.txt | wc -l` - echo "Total $total_failed failed" - if [ "$total_failed" -ne "0" ]; then + echo -e "${RED} ### Total $total_failed TSIM case(s) failed! ### ${NC}" exit $total_failed fi + + pip install --user ../../src/connector/python/linux/python2/ + pip3 install --user ../../src/connector/python/linux/python3/ + + cd ../pytest + sudo ./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt + total_py_success=`grep 'successfully executed' pytest-out.txt | wc -l` + + if [ "$total_py_success" -gt "0" ]; then + echo -e "${GREEN} ### Total $total_py_success python case(s) succeed! ### ${NC}" + fi + + total_py_failed=`grep 'failed' pytest-out.txt | wc -l` + if [ "$total_py_failed" -ne "0" ]; then + echo -e "${RED} ### Total $total_py_failed python case(s) failed! ### ${NC}" + exit $total_py_failed + fi + ;; esac @@ -81,6 +115,10 @@ matrix: - build-essential - cmake - net-tools + - python-pip + - python-setuptools + - python3-pip + - python3-setuptools # - os: osx # addons: diff --git a/tests/pytest/insert/__init__.py b/tests/pytest/insert/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py new file mode 100644 index 0000000000..c6dbd76de4 --- /dev/null +++ b/tests/pytest/insert/basic.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + tdSql.execute('show databases') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + tdSql.execute('create table tb (ts timestamp, speed int)') + + insertRows = 10 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + tdSql.execute('insert into tb values (now + %dm, %d)' % (i, i)) + +# tdLog.info("insert earlier data") +# tdSql.execute('insert into tb values (now - 5m , 10)') +# tdSql.execute('insert into tb values (now - 6m , 10)') +# tdSql.execute('insert into tb values (now - 7m , 10)') +# tdSql.execute('insert into tb values (now - 8m , 10)') + +# tdSql.query("select * from tb") +# tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh new file mode 100755 index 0000000000..aab36884f3 --- /dev/null +++ b/tests/pytest/simpletest.sh @@ -0,0 +1 @@ +sudo python2 ./test.py -f insert/basic.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py new file mode 100644 index 0000000000..b88e444665 --- /dev/null +++ b/tests/pytest/test.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +from util.log import * +from util.dnodes import * +from util.cases import * + +import taos + +# add testcase here: +from insert.basic import * + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + masterIp = "" + testCluster = False + opts, args = getopt.getopt(sys.argv[1:], 'f:p:m:sch', [ + 'file=', 'path=', 'master', 'stop', 'cluster', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-f Name of test case file written by Python') + tdLog.printNoPrefix('-p Deploy Path for Simulator') + tdLog.printNoPrefix('-m Master Ip for Simulator') + tdLog.printNoPrefix('-c Test Cluster Flag') + tdLog.printNoPrefix('-s stop All dnodes') + sys.exit(0) + if key in ['-f', '--file']: + fileName = value + if key in ['-p', '--path']: + deployPath = value + if key in ['-m', '--master']: + masterIp = value + if key in ['-c', '--cluster']: + testCluster = True + if key in ['-s', '--stop']: + cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -9 taosd" + os.system(cmd) + tdLog.exit('stop All dnodes') + + if masterIp == "": + tdDnodes.init(deployPath) + if testCluster: + tdLog.notice("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) + else: + tdLog.notice("Procedures for testing self-deployment") + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + conn = taos.connect( + host='192.168.0.1', + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) + else: + tdLog.info("CBD LN78: %s" % (fileName)) + tdCases.runOneLinux(conn, fileName) + conn.close() + else: + tdLog.notice("Procedures for tdengine deployed in %s" % (masterIp)) + conn = taos.connect(host=masterIp, config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllWindows(conn) + else: + tdCases.runOneWindows(conn, fileName) + conn.close() + diff --git a/tests/pytest/util/__init__.py b/tests/pytest/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py new file mode 100644 index 0000000000..320c9d974f --- /dev/null +++ b/tests/pytest/util/cases.py @@ -0,0 +1,103 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +from util.log import * + + +class TDCase: + def __init__(self, name, case): + self.name = name + self.case = case + + +class TDCases: + def __init__(self): + self.linuxCases = [] + self.windowsCases = [] + self.clusterCases = [] + + def addWindows(self, name, case): + self.windowsCases.append(TDCase(name, case)) + + def addLinux(self, name, case): + self.linuxCases.append(TDCase(name, case)) + + def addCluster(self, name, case): + self.clusterCases.append(TDCase(name, case)) + + def runAllLinux(self, conn): + tdLog.notice("run total %d cases" % (len(self.linuxCases))) + for case in self.linuxCases: + case.case.init(conn) + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.linuxCases))) + + def runOneLinux(self, conn, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.linuxCases: + if case.name.find(fileName) != -1: + case.case.init(conn) + case.case.run() + case.case.stop() + time.sleep(5) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + def runAllWindows(self, conn): + tdLog.notice("run total %d cases" % (len(self.windowsCases))) + for case in self.windowsCases: + case.case.init(conn) + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.windowsCases))) + + def runOneWindows(self, conn, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.windowsCases: + if case.name.find(fileName) != -1: + case.case.init(conn) + case.case.run() + case.case.stop() + time.sleep(2) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + def runAllCluster(self): + tdLog.notice("run total %d cases" % (len(self.clusterCases))) + for case in self.clusterCases: + case.case.init() + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.clusterCases))) + + def runOneCluster(self, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.clusterCases: + if case.name.find(fileName) != -1: + case.case.init() + case.case.run() + case.case.stop() + time.sleep(2) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + +tdCases = TDCases() diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py new file mode 100644 index 0000000000..2be4f94802 --- /dev/null +++ b/tests/pytest/util/dnodes.py @@ -0,0 +1,332 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import os.path +from util.log import * + + +class TDSimClient: + def init(self, path): + self.path = path + + def getCfgDir(self): + return self.cfgDir + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def deploy(self): + self.logDir = "%s/sim/psim/log" % (self.path,) + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "GBK") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) + + +class TDDnode: + def __init__(self, index): + self.index = index + self.running = 0 + self.deployed = 0 + + def init(self, path): + self.path = path + + def deploy(self): + self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index) + + cmd = "rm -rf " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.startIP() + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) + self.cfg("dataDir", self.dataDir) + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("mgmtEqualVnodeNum", "0") + self.cfg("clog", "1") + self.cfg("statusInterval", "1") + self.cfg("numOfTotalVnodes", "64") + self.cfg("numOfMPeers", "3") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("monitor", "0") + self.cfg("maxVnodeConnections", "30000") + self.cfg("maxMgmtConnections", "30000") + self.cfg("maxMeterConnections", "30000") + self.cfg("maxShellConns", "30000") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "UTF-8") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("dDebugFlag", "135") + self.cfg("mDebugFlag", "135") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("httpDebugFlag", "135") + self.cfg("monitorDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + self.deployed = 1 + tdLog.debug( + "dnode:%d is deployed and configured by %s" % + (self.index, self.cfgPath)) + + def start(self): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + binPath = os.path.realpath(binPath) + binPath += "/build/bin/" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + print(cmd) + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + + tdLog.debug("wait 2 seconds for the dnode:%d to start." % (self.index)) + time.sleep(2) + + def stop(self): + if self.running != 0: + cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigint taosd" % ( + self.cfgDir) + if os.system(cmd) != 0: + tdLog.exit(cmd) + tdLog.debug("dnode:%d is stopped by kill -SIGINT" % (self.index)) + tdLog.debug( + "wait 2 seconds for the dnode:%d to stop." % + (self.index)) + time.sleep(2) + + def forcestop(self): + if self.running != 0: + cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" % ( + self.cfgDir) + if os.system(cmd) != 0: + tdLog.exit(cmd) + tdLog.debug("dnode:%d is stopped by kill -9" % (self.index)) + tdLog.debug( + "wait 2 seconds for the dnode:%d to stop." % + (self.index)) + time.sleep(2) + + def startIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def stopIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( + self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def getDnodeRootDir(self, index): + dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + return dnodeRootDir + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim/psim" % (self.path) + return dnodesRootDir + + +class TDDnodes: + def __init__(self): + self.dnodes = [] + self.dnodes.append(TDDnode(1)) + self.dnodes.append(TDDnode(2)) + self.dnodes.append(TDDnode(3)) + self.dnodes.append(TDDnode(4)) + self.dnodes.append(TDDnode(5)) + self.dnodes.append(TDDnode(6)) + self.dnodes.append(TDDnode(7)) + self.dnodes.append(TDDnode(8)) + self.dnodes.append(TDDnode(9)) + self.dnodes.append(TDDnode(10)) + + def init(self, path): + cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" + os.system(cmd) + + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug("binPath %s" % (binPath)) + binPath = os.path.realpath(binPath) + tdLog.debug("binPath real path %s" % (binPath)) + + # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) + # tdLog.debug(cmd) + # os.system(cmd) + + # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + if path == "": + # self.path = os.path.expanduser('~') + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + + for i in range(len(self.dnodes)): + self.dnodes[i].init(self.path) + + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.deploy() + + def deploy(self, index): + self.check(index) + self.dnodes[index - 1].deploy() + + def cfg(self, index, option, value): + self.check(index) + self.dnodes[index - 1].cfg(option, value) + + def start(self, index): + self.check(index) + self.dnodes[index - 1].start() + + def stop(self, index): + self.check(index) + self.dnodes[index - 1].stop() + + def forcestop(self, index): + self.check(index) + self.dnodes[index - 1].forcestop() + + def startIP(self, index): + self.check(index) + self.dnodes[index - 1].startIP() + + def stopIP(self, index): + self.check(index) + self.dnodes[index - 1].stopIP() + + def check(self, index): + if index < 1 or index > 10: + tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + + def stopAll(self): + tdLog.debug("stop all dnodes") + for i in range(len(self.dnodes)): + self.dnodes[i].stop() + + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + cmd = "ps -ef | grep -w taosd | grep 'dnode' | grep -v grep | awk '{print $2}' && sudo pkill -sigkill taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim" % (self.path) + return dnodesRootDir + + def getSimCfgPath(self): + return self.sim.getCfgDir() + + +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py new file mode 100644 index 0000000000..926e582448 --- /dev/null +++ b/tests/pytest/util/log.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + + +class TDLog: + def __init__(self): + self.path = "" + + def info(self, info): + print "%s %s" % (datetime.datetime.now(), info) + + def sleep(self, sec): + print "%s sleep %d seconds" % (datetime.datetime.now(), sec) + time.sleep(sec) + + def debug(self, err): + print "\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err) + + def success(self, info): + print "\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info) + + def notice(self, err): + print "\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err) + + def exit(self, err): + print "\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err) + sys.exit(1) + + def printNoPrefix(self, info): + print "\033[1;36m%s\033[0m" % (info) + + +tdLog = TDLog() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py new file mode 100644 index 0000000000..b4ac845bc8 --- /dev/null +++ b/tests/pytest/util/sql.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +from util.log import * + + +class TDSql: + def __init__(self): + self.queryRows = 0 + self.queryCols = 0 + self.affectedRows = 0 + + def init(self, cursor): + self.cursor = cursor + + def close(self): + self.cursor.close() + + def prepare(self): + tdLog.info("prepare database:db") + self.cursor.execute('reset query cache') + self.cursor.execute('drop database if exists db') + self.cursor.execute('create database db') + self.cursor.execute('use db') + + def error(self, sql): + expectErrNotOccured = True + try: + self.cursor.execute(sql) + except BaseException: + expectErrNotOccured = False + if expectErrNotOccured: + tdLog.exit("sql:%.40s, expect error not occured" % (sql)) + else: + tdLog.info("sql:%.40s, expect error occured" % (sql)) + + def query(self, sql): + self.sql = sql + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + # if self.queryRows == 1 and self.queryCols == 1: + # tdLog.info("sql:%s, rows:%d cols:%d data:%s" % (self.sql, self.queryRows, self.queryCols, self.queryResult[0][0])) + # else: + # tdLog.info("sql:%s, rows:%d cols:%d" % (self.sql, self.queryRows, self.queryCols)) + return self.queryRows + + def checkRows(self, expectRows): + if self.queryRows != expectRows: + tdLog.exit( + "sql:%.40s, queryRows:%d != expect:%d" % + (self.sql, self.queryRows, expectRows)) + tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % + (self.sql, self.queryRows, expectRows)) + + def checkData(self, row, col, data): + if row < 0: + tdLog.exit( + "sql:%.40s, row:%d is smaller than zero" % + (self.sql, row)) + if col < 0: + tdLog.exit( + "sql:%.40s, col:%d is smaller than zero" % + (self.sql, col)) + if row >= self.queryRows: + tdLog.exit( + "sql:%.40s, row:%d is larger than queryRows:%d" % + (self.sql, row, self.queryRows)) + if col >= self.queryCols: + tdLog.exit( + "sql:%.40s, col:%d is larger than queryRows:%d" % + (self.sql, col, self.queryCols)) + if self.queryResult[row][col] != data: + tdLog.exit( + "sql:%.40s row:%d col:%d data:%s != expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + (self.sql, row, col, self.queryResult[row][col], data)) + + def getData(self, row, col): + if row < 0: + tdLog.exit( + "sql:%.40s, row:%d is smaller than zero" % + (self.sql, row)) + if col < 0: + tdLog.exit( + "sql:%.40s, col:%d is smaller than zero" % + (self.sql, col)) + if row >= self.queryRows: + tdLog.exit( + "sql:%.40s, row:%d is larger than queryRows:%d" % + (self.sql, row, self.queryRows)) + if col >= self.queryCols: + tdLog.exit( + "sql:%.40s, col:%d is larger than queryRows:%d" % + (self.sql, col, self.queryCols)) + return self.queryResult[row][col] + + def executeTimes(self, sql, times): + for i in range(times): + try: + return self.cursor.execute(sql) + except BaseException: + time.sleep(1) + continue + + def execute(self, sql): + self.sql = sql + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + + def checkAffectedRows(self, expectAffectedRows): + if self.affectedRows != expectAffectedRows: + tdLog.exit("sql:%.40s, affectedRows:%d != expect:%d" % + (self.sql, self.affectedRows, expectAffectedRows)) + tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % + (self.sql, self.affectedRows, expectAffectedRows)) + + +tdSql = TDSql() From 32bdf91387ac27fdd6cb576d7409deae9cbdbe64 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 16:30:08 +0800 Subject: [PATCH 05/36] add one more parameter in walCfg keep the wal file when it is closed --- src/vnode/main/src/vnodeMain.c | 1 + src/vnode/wal/src/walMain.c | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 210063ecf5..ea7a003d3d 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -368,6 +368,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->walCfg.wals = (int8_t)wals; + pVnode->walCfg.keep = 0; int32_t arbitratorIp = -1; num = fscanf(fp, "%s %u", option[0], &arbitratorIp); diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index e799bfd63a..edca4e371c 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -36,6 +36,7 @@ typedef struct { int fd; + int keep; int level; int max; // maximum number of wal files uint32_t id; // increase continuously @@ -61,6 +62,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { pWal->id = 0; pWal->num = 0; pWal->level = pCfg->commitLog; + pWal->keep = pCfg->keep; strcpy(pWal->path, path); pthread_mutex_init(&pWal->mutex, NULL); @@ -82,18 +84,21 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { void walClose(void *handle) { if (handle == NULL) return; - SWal *pWal = handle; - + SWal *pWal = handle; close(pWal->fd); - // remove all files in the directory - for (int i=0; inum; ++i) { - sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i); - if (remove(pWal->name) <0) { - wError("wal:%s, failed to remove", pWal->name); - } else { - wTrace("wal:%s, it is removed", pWal->name); + if (pWal->keep == 0) { + // remove all files in the directory + for (int i=0; inum; ++i) { + sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i); + if (remove(pWal->name) <0) { + wError("wal:%s, failed to remove", pWal->name); + } else { + wTrace("wal:%s, it is removed", pWal->name); + } } + } else { + wTrace("wal:%s, it is closed and kept", pWal->name); } pthread_mutex_destroy(&pWal->mutex); From 0743d310d312982eee64a7b60d024a8f1d8b6179 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 16:34:40 +0800 Subject: [PATCH 06/36] add keep in walCfg --- src/inc/twal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index e90e2e54ce..06dc2e881b 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -36,12 +36,13 @@ typedef struct { typedef struct { int8_t commitLog; // commitLog int8_t wals; // number of WAL files; + int8_t keep; // keep the wal file when closed } SWalCfg; typedef void* twalh; // WAL HANDLE typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); -twalh walOpen(const char *path, const SWalCfg *pCfg); +twalh walOpen(const char *path, const SWalCfg *pCfg); void walClose(twalh); int walRenew(twalh); int walWrite(twalh, SWalHead *); From 71607ccb8bfb43e98b4b1d8e51759b1accdcd5e5 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 11 Apr 2020 16:39:11 +0800 Subject: [PATCH 07/36] [TD-52] update status message --- src/dnode/inc/dnodeMClient.h | 1 + src/dnode/src/dnodeMClient.c | 227 ++++++++++++++++++------- src/inc/mnode.h | 2 + src/inc/mpeer.h | 24 +-- src/inc/taosmsg.h | 15 +- src/mnode/src/mgmtDb.c | 2 - src/mnode/src/mgmtDnode.c | 4 +- src/mnode/src/mgmtMnode.c | 135 +++++++++------ src/mnode/src/mgmtProfile.c | 9 +- src/mnode/src/mgmtShell.c | 2 +- src/{plugins/http => util}/inc/cJSON.h | 0 src/{plugins/http => util}/src/cJSON.c | 0 12 files changed, 281 insertions(+), 140 deletions(-) rename src/{plugins/http => util}/inc/cJSON.h (100%) rename src/{plugins/http => util}/src/cJSON.c (100%) diff --git a/src/dnode/inc/dnodeMClient.h b/src/dnode/inc/dnodeMClient.h index cab9ea9be4..ba63894631 100644 --- a/src/dnode/inc/dnodeMClient.h +++ b/src/dnode/inc/dnodeMClient.h @@ -24,6 +24,7 @@ int32_t dnodeInitMClient(); void dnodeCleanupMClient(); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); uint32_t dnodeGetMnodeMasteIp(); +void * dnodeGetMpeerInfos(); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 5630872318..c09583bd65 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -15,33 +15,46 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "cJSON.h" #include "taosmsg.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" +#include "tsync.h" #include "dnode.h" #include "dnodeMClient.h" #include "dnodeModule.h" #include "dnodeMgmt.h" +#define MPEER_CONTENT_LEN 2000 + static bool dnodeReadMnodeIpList(); static void dnodeSaveMnodeIpList(); static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void *tsDnodeMClientRpc = NULL; -static SRpcIpSet tsDnodeMnodeIpList = {0}; +static SRpcIpSet tsMpeerIpList = {0}; +static SDMNodeInfos tsMpeerInfos = {0}; int32_t dnodeInitMClient() { if (!dnodeReadMnodeIpList()) { - dTrace("failed to read mnode iplist, set it from cfg file"); - memset(&tsDnodeMnodeIpList, 0, sizeof(SRpcIpSet)); - tsDnodeMnodeIpList.port = tsMnodeDnodePort; - tsDnodeMnodeIpList.numOfIps = 1; - tsDnodeMnodeIpList.ip[0] = inet_addr(tsMasterIp); + memset(&tsMpeerIpList, 0, sizeof(SRpcIpSet)); + memset(&tsMpeerInfos, 0, sizeof(SDMNodeInfos)); + tsMpeerIpList.port = tsMnodeDnodePort; + tsMpeerIpList.numOfIps = 1; + tsMpeerIpList.ip[0] = inet_addr(tsMasterIp); if (tsSecondIp[0]) { - tsDnodeMnodeIpList.numOfIps = 2; - tsDnodeMnodeIpList.ip[1] = inet_addr(tsSecondIp); + tsMpeerIpList.numOfIps = 2; + tsMpeerIpList.ip[1] = inet_addr(tsSecondIp); + } + } else { + SRpcIpSet mgmtIpSet = {0}; + tsMpeerIpList.inUse = tsMpeerInfos.inUse; + tsMpeerIpList.numOfIps = tsMpeerInfos.nodeNum; + tsMpeerIpList.port = tsMpeerInfos.nodeInfos[0].nodePort; + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + tsMpeerIpList.ip[i] = tsMpeerInfos.nodeInfos[i].nodeIp; } } @@ -96,23 +109,31 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { } SDMStatusRsp *pStatusRsp = pMsg->pCont; - if (pStatusRsp->ipList.numOfIps <= 0) { - dError("status msg is invalid, num of ips is %d", pStatusRsp->ipList.numOfIps); + SDMNodeInfos *mpeers = &pStatusRsp->mpeers; + if (mpeers->nodeNum <= 0) { + dError("status msg is invalid, num of ips is %d", mpeers->nodeNum); return; } - pStatusRsp->ipList.port = htons(pStatusRsp->ipList.port); - for (int32_t i = 0; i < pStatusRsp->ipList.numOfIps; ++i) { - pStatusRsp->ipList.ip[i] = htonl(pStatusRsp->ipList.ip[i]); + SRpcIpSet mgmtIpSet = {0}; + mgmtIpSet.inUse = mpeers->inUse; + mgmtIpSet.numOfIps = mpeers->nodeNum; + mgmtIpSet.port = htons(mpeers->nodeInfos[0].nodePort); + for (int32_t i = 0; i < mpeers->nodeNum; i++) { + mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp); } - //dTrace("status msg is received, result:%s", tstrerror(pMsg->code)); - - if (memcmp(&(pStatusRsp->ipList), &tsDnodeMnodeIpList, sizeof(SRpcIpSet)) != 0) { - dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", pStatusRsp->ipList.numOfIps, pStatusRsp->ipList.inUse); - memcpy(&tsDnodeMnodeIpList, &pStatusRsp->ipList, sizeof(SRpcIpSet)); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; ++i) { - dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); + if (memcmp(&mgmtIpSet, &tsMpeerIpList, sizeof(SRpcIpSet)) != 0) { + memcpy(&tsMpeerIpList, &mgmtIpSet, sizeof(SRpcIpSet)); + memcpy(&tsMpeerInfos, mpeers, sizeof(SDMNodeInfos)); + dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + for (int32_t i = 0; i < mpeers->nodeNum; i++) { + tsMpeerInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); + tsMpeerInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); + tsMpeerInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); + dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, + taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, + tsMpeerInfos.nodeInfos[i].nodeName); } dnodeSaveMnodeIpList(); } @@ -129,70 +150,148 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { if (tsDnodeMClientRpc) { - rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg); + rpcSendRequest(tsDnodeMClientRpc, &tsMpeerIpList, rpcMsg); } } static bool dnodeReadMnodeIpList() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/iplist", tsDnodeDir); - + sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir); FILE *fp = fopen(ipFile, "r"); - if (!fp) return false; - - char option[32] = {0}; - int32_t value = 0; - int32_t num = 0; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "inUse") != 0) return false; - tsDnodeMnodeIpList.inUse = (int8_t)value;; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "numOfIps") != 0) return false; - tsDnodeMnodeIpList.numOfIps = (int8_t)value; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "port") != 0) return false; - tsDnodeMnodeIpList.port = (uint16_t)value; - - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strncmp(option, "ip", 2) != 0) return false; - tsDnodeMnodeIpList.ip[i] = (uint32_t)value; + if (!fp) { + dTrace("failed to read mnode mgmtIpList.json, file not exist"); + return false; } - fclose(fp); - dPrint("read mnode iplist successed"); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); - } + bool ret = false; + int maxLen = 2000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + dError("failed to read mnode mgmtIpList.json, content is null"); + return false; + } - return true; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read mnode mgmtIpList.json, invalid json format"); + goto PARSE_OVER; + } + + cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); + if (!inUse || inUse->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, inUse not found"); + goto PARSE_OVER; + } + tsMpeerInfos.inUse = inUse->valueint; + + cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); + if (!nodeNum || nodeNum->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodeNum not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeNum = nodeNum->valueint; + + cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + dError("failed to read mnode mgmtIpList.json, nodeInfos not found"); + goto PARSE_OVER; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != tsMpeerInfos.nodeNum) { + dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched"); + goto PARSE_OVER; + } + + for (int i = 0; i < size; ++i) { + cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodeId not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodeId = nodeId->valueint; + + cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); + if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { + dError("failed to read mnode mgmtIpList.json, nodeIp not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); + + cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort"); + if (!nodePort || nodePort->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodePort not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; + + cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); + if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { + dError("failed to read mnode mgmtIpList.json, nodeName not found"); + goto PARSE_OVER; + } + strncpy(tsMpeerInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); + } + + ret = true; + + dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, + taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, + tsMpeerInfos.nodeInfos[i].nodeName); + } + +PARSE_OVER: + free(content); + fclose(fp); + return ret; } static void dnodeSaveMnodeIpList() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/iplist", tsDnodeDir); - + sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir); FILE *fp = fopen(ipFile, "w"); if (!fp) return; - fprintf(fp, "inUse %d\n", tsDnodeMnodeIpList.inUse); - fprintf(fp, "numOfIps %d\n", tsDnodeMnodeIpList.numOfIps); - fprintf(fp, "port %u\n", tsDnodeMnodeIpList.port); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - fprintf(fp, "ip%d %u\n", i, tsDnodeMnodeIpList.ip[i]); + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMpeerInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMpeerInfos.nodeNum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMpeerInfos.nodeInfos[i].nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMpeerInfos.nodeInfos[i].nodeIp)); + len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMpeerInfos.nodeInfos[i].nodePort); + len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMpeerInfos.nodeInfos[i].nodeName); + if (i < tsMpeerInfos.nodeNum -1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } } - + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); fclose(fp); + free(content); + dPrint("save mnode iplist successed"); } uint32_t dnodeGetMnodeMasteIp() { - return tsDnodeMnodeIpList.ip[0]; + return tsMpeerIpList.ip[tsMpeerIpList.inUse]; +} + +void* dnodeGetMpeerInfos() { + return &tsMpeerInfos; } \ No newline at end of file diff --git a/src/inc/mnode.h b/src/inc/mnode.h index b19e91a90e..f2c072453f 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -52,8 +52,10 @@ typedef struct _mnode_obj { int32_t refCount; int8_t role; int8_t status; + uint16_t port; uint32_t privateIp; uint32_t publicIp; + char mnodeName[TSDB_NODE_NAME_LEN + 1]; } SMnodeObj; typedef struct _dnode_obj { diff --git a/src/inc/mpeer.h b/src/inc/mpeer.h index 7007f19226..157ea40119 100644 --- a/src/inc/mpeer.h +++ b/src/inc/mpeer.h @@ -20,28 +20,30 @@ extern "C" { #endif -enum _TSDB_MN_STATUS { - TSDB_MN_STATUS_OFFLINE, - TSDB_MN_STATUS_UNSYNCED, - TSDB_MN_STATUS_SYNCING, - TSDB_MN_STATUS_SERVING -}; +struct _mnode_obj; -enum _TSDB_MN_ROLE { - TSDB_MN_ROLE_UNDECIDED, - TSDB_MN_ROLE_SLAVE, - TSDB_MN_ROLE_MASTER +enum _TAOS_MN_STATUS { + TAOS_MN_STATUS_OFFLINE, + TAOS_MN_STATUS_DROPPING, + TAOS_MN_STATUS_READY }; int32_t mpeerInit(); void mpeerCleanup(); +int32_t mpeerGetMnodesNum(); +void * mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode); +void mpeerReleaseMnode(struct _mnode_obj *pMnode); bool mpeerInServerStatus(); bool mpeerIsMaster(); +bool mpeerCheckRedirect(); -bool mpeerCheckRedirect(void *handle); void mpeerGetPrivateIpList(SRpcIpSet *ipSet); void mpeerGetPublicIpList(SRpcIpSet *ipSet); +void mpeerGetMpeerInfos(void *mpeers); + +char * mpeerGetMnodeStatusStr(int32_t status); +char * mpeerGetMnodeRoleStr(int32_t role); #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 09d306ffad..5378bf4bae 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -593,7 +593,20 @@ typedef struct { } SDMStatusMsg; typedef struct { - SRpcIpSet ipList; + int32_t nodeId; + uint32_t nodeIp; + uint16_t nodePort; + char nodeName[TSDB_NODE_NAME_LEN + 1]; +} SDMNodeInfo; + +typedef struct { + int8_t inUse; + int8_t nodeNum; + SDMNodeInfo nodeInfos[TSDB_MAX_MPEERS]; +} SDMNodeInfos; + +typedef struct { + SDMNodeInfos mpeers; SDnodeState dnodeState; SVnodeAccess vnodeAccess[]; } SDMStatusRsp; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 0f18de3253..b1931347a7 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -678,8 +678,6 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { - if (mpeerCheckRedirect(pMsg->thandle)) return; - SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 1d32ded088..51ac4e842d 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -141,8 +141,6 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { } void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { - if (mpeerCheckRedirect(rpcMsg->handle)) return; - SDMStatusMsg *pStatus = rpcMsg->pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->privateIp = htonl(pStatus->privateIp); @@ -221,7 +219,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mpeerGetPrivateIpList(&pRsp->ipList); + mpeerGetMpeerInfos(&pRsp->mpeers); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index c349969019..82da454793 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,41 +17,36 @@ #include "os.h" #include "taoserror.h" #include "trpc.h" +#include "tsync.h" #include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" -#ifndef _MPEER - -static SMnodeObj tsMnodeObj = {0}; +extern int32_t mpeerInitMnodes(); +extern void mpeerCleanupMnodes(); static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -int32_t mpeerInit() { - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); +#ifndef _MPEER +static SMnodeObj tsMnodeObj = {0}; + +int32_t mpeerInitMnodes() { tsMnodeObj.mnodeId = 1; + tsMnodeObj.dnodeId = 1; tsMnodeObj.privateIp = inet_addr(tsPrivateIp); tsMnodeObj.publicIp = inet_addr(tsPublicIp); tsMnodeObj.createdTime = taosGetTimestampMs(); - tsMnodeObj.role = TSDB_MN_ROLE_MASTER; - tsMnodeObj.status = TSDB_MN_STATUS_SERVING; + tsMnodeObj.role = TAOS_SYNC_ROLE_MASTER; + tsMnodeObj.status = TAOS_MN_STATUS_READY; + tsMnodeObj.port = tsMnodeDnodePort; + sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId); return TSDB_CODE_SUCCESS; } -void mpeerCleanup() {} -bool mpeerInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } -bool mpeerIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } -bool mpeerCheckRedirect(void *thandle) { return false; } - -static int32_t mgmtGetMnodesNum() { - return 1; -} - -static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { +void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; } else { @@ -61,22 +56,74 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { return *pMnode; } -char *taosGetMnodeStatusStr(int32_t mnodeStatus) { - switch (mnodeStatus) { - case TSDB_MN_STATUS_OFFLINE: return "offline"; - case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_MN_STATUS_SYNCING: return "syncing"; - case TSDB_MN_STATUS_SERVING: return "serving"; - default: return "undefined"; +void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(tsMnodeObj.privateIp); +} + +void mpeerGetPublicIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(tsMnodeObj.publicIp); +} + +void mpeerGetMpeerInfos(void *param) { + SDMNodeInfos *mpeers = param; + mpeers->nodeNum = 1; + mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId); + mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp); + mpeers->nodeInfos[0].nodePort = htons(tsMnodeObj.port); + strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName); +} + +void mpeerCleanupDnodes() {} +int32_t mpeerGetMnodesNum() { return 1; } +void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} +bool mpeerInServerStatus() { return tsMnodeObj.status == TAOS_MN_STATUS_READY; } +bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; } +bool mpeerCheckRedirect() { return false; } + +#endif + +int32_t mpeerInit() { + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); + + return mpeerInitMnodes(); +} + +void mpeerCleanup() { + mpeerCleanupDnodes(); +} + +char *mpeerGetMnodeStatusStr(int32_t status) { + switch (status) { + case TAOS_MN_STATUS_OFFLINE: + return "offline"; + case TAOS_MN_STATUS_DROPPING: + return "dropping"; + case TAOS_MN_STATUS_READY: + return "ready"; + default: + return "undefined"; } } -char *taosGetMnodeRoleStr(int32_t mnodeRole) { - switch (mnodeRole) { - case TSDB_MN_ROLE_UNDECIDED: return "undicided"; - case TSDB_MN_ROLE_SLAVE: return "slave"; - case TSDB_MN_ROLE_MASTER: return "master"; - default: return "undefined"; +char *mpeerGetMnodeRoleStr(int32_t role) { + switch (role) { + case TAOS_SYNC_ROLE_OFFLINE: + return "offline"; + case TAOS_SYNC_ROLE_UNSYNCED: + return "unsynced"; + case TAOS_SYNC_ROLE_SLAVE: + return "slave"; + case TAOS_SYNC_ROLE_MASTER: + return "master"; + default: + return "undefined"; } } @@ -133,7 +180,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = mgmtGetMnodesNum(); + pShow->numOfRows = mpeerGetMnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; mgmtReleaseUser(pUser); @@ -149,7 +196,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi char ipstr[32]; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextMnode(pShow->pNode, (SMnodeObj **)&pMnode); + pShow->pNode = mpeerGetNextMnode(pShow->pNode, &pMnode); if (pMnode == NULL) break; cols = 0; @@ -173,11 +220,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetMnodeStatusStr(pMnode->status)); + strcpy(pWrite, mpeerGetMnodeStatusStr(pMnode->status)); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetMnodeRoleStr(pMnode->role)); + strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role)); cols++; numOfRows++; @@ -186,20 +233,4 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi pShow->numOfReads += numOfRows; return numOfRows; -} - -void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { - ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); - ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(tsMnodeObj.privateIp); -} - -void mpeerGetPublicIpList(SRpcIpSet *ipSet) { - ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); - ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(tsMnodeObj.publicIp); -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 36481b81b0..f7dec4656b 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -681,8 +681,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; @@ -705,8 +704,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; @@ -729,8 +727,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index eb20225e95..dbd7627d3f 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -142,7 +142,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } - if (mpeerCheckRedirect(rpcMsg->handle)) { + if (mpeerCheckRedirect()) { // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect()); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER); rpcFreeCont(rpcMsg->pCont); diff --git a/src/plugins/http/inc/cJSON.h b/src/util/inc/cJSON.h similarity index 100% rename from src/plugins/http/inc/cJSON.h rename to src/util/inc/cJSON.h diff --git a/src/plugins/http/src/cJSON.c b/src/util/src/cJSON.c similarity index 100% rename from src/plugins/http/src/cJSON.c rename to src/util/src/cJSON.c From a9be7a8553ab0a89c1d93d7146dfd7b0b73e66db Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 22:28:58 +0800 Subject: [PATCH 08/36] exit nicely --- src/dnode/src/dnodeRead.c | 84 +++++++++++++++++++++++----------- src/dnode/src/dnodeWrite.c | 31 +++++++++---- src/rpc/src/rpcCache.c | 3 +- src/rpc/src/rpcClient.c | 4 +- src/rpc/src/rpcServer.c | 7 ++- src/rpc/src/rpcUdp.c | 4 +- src/util/src/ihash.c | 1 - src/util/src/tqueue.c | 17 +++++-- src/util/src/tsched.c | 2 + src/vnode/main/src/vnodeMain.c | 3 +- 10 files changed, 106 insertions(+), 50 deletions(-) diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index d4365dae10..f13c18f22a 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -32,27 +32,51 @@ typedef struct { SRpcMsg rpcMsg; } SReadMsg; +typedef struct { + pthread_t thread; // thread + int32_t workerId; // worker ID +} SReadWorker; + +typedef struct { + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + SReadWorker *readWorker; +} SReadWorkerPool; + static void *dnodeProcessReadQueue(void *param); -static void dnodeHandleIdleReadWorker(); +static void dnodeHandleIdleReadWorker(SReadWorker *); // module global variable -static taos_qset readQset; -static int32_t threads; // number of query threads -static int32_t maxThreads; -static int32_t minThreads; +static SReadWorkerPool readPool; +static taos_qset readQset; int32_t dnodeInitRead() { readQset = taosOpenQset(); - minThreads = 3; - maxThreads = tsNumOfCores * tsNumOfThreadsPerCore; - if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads; + readPool.min = 2; + readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; + if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; + readPool.readWorker = (SReadWorker *) calloc(sizeof(SReadWorker), readPool.max); + + if (readPool.readWorker == NULL) return -1; + for (int i=0; i < readPool.max; ++i) { + SReadWorker *pWorker = readPool.readWorker + i; + pWorker->workerId = i; + } dPrint("dnode read is opened"); return 0; } void dnodeCleanupRead() { + + for (int i=0; i < readPool.max; ++i) { + SReadWorker *pWorker = readPool.readWorker + i; + if (pWorker->thread) + pthread_join(pWorker->thread, NULL); + } + taosCloseQset(readQset); dPrint("dnode read is closed"); } @@ -116,18 +140,25 @@ void *dnodeAllocateRqueue(void *pVnode) { taosAddIntoQset(readQset, queue, pVnode); // spawn a thread to process queue - if (threads < maxThreads) { - pthread_t thread; - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (readPool.num < readPool.max) { + do { + SReadWorker *pWorker = readPool.readWorker + readPool.num; - if (pthread_create(&thread, &thAttr, dnodeProcessReadQueue, readQset) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); - } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) { + dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + readPool.num++; + dTrace("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num); + } while (readPool.num < readPool.min); } - dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue); + dTrace("pVnode:%p, read queue:%p is allocated", pVnode, queue); return queue; } @@ -167,14 +198,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { } static void *dnodeProcessReadQueue(void *param) { - taos_qset qset = (taos_qset)param; - SReadMsg *pReadMsg; - int type; - void *pVnode; + SReadWorker *pWorker = param; + SReadMsg *pReadMsg; + int type; + void *pVnode; while (1) { - if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) { - dnodeHandleIdleReadWorker(); + if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) { + dnodeHandleIdleReadWorker(pWorker); continue; } @@ -186,11 +217,12 @@ static void *dnodeProcessReadQueue(void *param) { return NULL; } -static void dnodeHandleIdleReadWorker() { +static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) { int32_t num = taosGetQueueNumber(readQset); - if (num == 0 || (num <= minThreads && threads > minThreads)) { - threads--; + if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) { + readPool.num--; + dTrace("read worker:%d is released, total:%d", pWorker->workerId, readPool.num); pthread_exit(NULL); } else { usleep(100); diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 3c598ca360..b56e0d8ad7 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -28,6 +28,7 @@ #include "vnode.h" typedef struct { + taos_qall qall; taos_qset qset; // queue set pthread_t thread; // thread int32_t workerId; // worker ID @@ -65,6 +66,14 @@ int32_t dnodeInitWrite() { } void dnodeCleanupWrite() { + + for (int32_t i = 0; i < wWorkerPool.max; ++i) { + SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + if (pWorker->thread) { + pthread_join(pWorker->thread, NULL); + } + } + free(wWorkerPool.writeWorker); dPrint("dnode write is closed"); } @@ -113,6 +122,7 @@ void *dnodeAllocateWqueue(void *pVnode) { if (pWorker->qset == NULL) return NULL; taosAddIntoQset(pWorker->qset, queue, pVnode); + pWorker->qall = taosAllocateQall(); wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; pthread_attr_t thAttr; @@ -122,13 +132,17 @@ void *dnodeAllocateWqueue(void *pVnode) { if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) { dError("failed to create thread to process read queue, reason:%s", strerror(errno)); taosCloseQset(pWorker->qset); + } else { + dTrace("write worker:%d is launched", pWorker->workerId); } + + pthread_attr_destroy(&thAttr); } else { taosAddIntoQset(pWorker->qset, queue, pVnode); wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; } - dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue); + dTrace("pVnode:%p, write queue:%p is allocated", pVnode, queue); return queue; } @@ -160,17 +174,14 @@ void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code) { static void *dnodeProcessWriteQueue(void *param) { SWriteWorker *pWorker = (SWriteWorker *)param; - taos_qall qall; SWriteMsg *pWrite; SWalHead *pHead; int32_t numOfMsgs; int type; void *pVnode, *item; - qall = taosAllocateQall(); - while (1) { - numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, qall, &pVnode); + numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs <=0) { dnodeHandleIdleWorker(pWorker); // thread exit if no queues anymore continue; @@ -178,7 +189,7 @@ static void *dnodeProcessWriteQueue(void *param) { for (int32_t i = 0; i < numOfMsgs; ++i) { pWrite = NULL; - taosGetQitem(qall, &type, &item); + taosGetQitem(pWorker->qall, &type, &item); if (type == TAOS_QTYPE_RPC) { pWrite = (SWriteMsg *)item; pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead)); @@ -196,9 +207,9 @@ static void *dnodeProcessWriteQueue(void *param) { walFsync(vnodeGetWal(pVnode)); // browse all items, and process them one by one - taosResetQitems(qall); + taosResetQitems(pWorker->qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, &type, &item); + taosGetQitem(pWorker->qall, &type, &item); if (type == TAOS_QTYPE_RPC) { pWrite = (SWriteMsg *)item; dnodeSendRpcWriteRsp(pVnode, item, pWrite->rpcMsg.code); @@ -209,8 +220,6 @@ static void *dnodeProcessWriteQueue(void *param) { } } - taosFreeQall(qall); - return NULL; } @@ -221,8 +230,10 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { usleep(1000); sched_yield(); } else { + taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); pWorker->qset = NULL; + dTrace("write worker:%d is released", pWorker->workerId); pthread_exit(NULL); } } diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index a397f6f845..a4863ef61d 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -103,7 +103,8 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); tfree(pCache->connHashList); - tfree(pCache->count) + tfree(pCache->count); + tfree(pCache->lockedBy); pthread_mutex_unlock(&pCache->mutex); diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index b362b1ba44..264449bbb0 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -84,7 +84,9 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) { + int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); + pthread_attr_destroy(&thattr); + if (code != 0) { tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); return NULL; } diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 1aadabc5f7..538b3059e3 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -83,6 +83,9 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, } memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pThreadObj = pServerObj->pThreadObj; for (i = 0; i < numOfThreads; ++i) { pThreadObj->processData = fp; @@ -105,8 +108,6 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, return NULL; } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); return NULL; @@ -116,8 +117,6 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pThreadObj++; } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); return NULL; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 64a4df0e73..785288f5b9 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -146,10 +146,12 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v pConn->tmrCtrl = pSet->tmrCtrl; } - if (pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn) != 0) { + int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + if (code != 0) { tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); taosCloseSocket(pConn->fd); taosCleanUpUdpConnection(pSet); + pthread_attr_destroy(&thAttr); return NULL; } diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index 30773ae8d9..2cfadad964 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -189,7 +189,6 @@ void taosCleanUpIntHash(void *handle) { free(pObj->hashList); } - memset(pObj, 0, sizeof(IHashObj)); free(pObj->lockedBy); free(pObj); } diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index 2cf94267f8..b9f1141d35 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -117,7 +117,7 @@ int taosWriteQitem(taos_queue param, int type, void *item) { queue->numOfItems++; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - pTrace("item:%p is put into queue, type:%d items:%d", item, type, queue->numOfItems); + pTrace("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems); pthread_mutex_unlock(&queue->mutex); @@ -297,14 +297,16 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand STaosQset *qset = (STaosQset *)param; STaosQnode *pNode = NULL; int code = 0; + + pthread_mutex_lock(&qset->mutex); for(int i=0; inumOfQueues; ++i) { - pthread_mutex_lock(&qset->mutex); + //pthread_mutex_lock(&qset->mutex); if (qset->current == NULL) qset->current = qset->head; STaosQueue *queue = qset->current; if (queue) qset->current = queue->next; - pthread_mutex_unlock(&qset->mutex); + //pthread_mutex_unlock(&qset->mutex); if (queue == NULL) break; pthread_mutex_lock(&queue->mutex); @@ -326,6 +328,8 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand if (pNode) break; } + pthread_mutex_unlock(&qset->mutex); + return code; } @@ -335,13 +339,15 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) { STaosQall *qall = (STaosQall *)p2; int code = 0; + pthread_mutex_lock(&qset->mutex); + for(int i=0; inumOfQueues; ++i) { - pthread_mutex_lock(&qset->mutex); + // pthread_mutex_lock(&qset->mutex); if (qset->current == NULL) qset->current = qset->head; queue = qset->current; if (queue) qset->current = queue->next; - pthread_mutex_unlock(&qset->mutex); + // pthread_mutex_unlock(&qset->mutex); if (queue == NULL) break; pthread_mutex_lock(&queue->mutex); @@ -365,6 +371,7 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) { if (code != 0) break; } + pthread_mutex_unlock(&qset->mutex); return code; } diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 56d16eeb71..11735d91b4 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -94,10 +94,12 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads); + pthread_attr_destroy(&attr); return (void *)pSched; _error: + pthread_attr_destroy(&attr); taosCleanUpScheduler(pSched); return NULL; } diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index ea7a003d3d..4d77e007ad 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -224,10 +224,11 @@ void vnodeRelease(void *pVnodeRaw) { // remove the whole directory } - dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId); free(pVnode); int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1); + dTrace("pVnode:%p vgId:%d, vnode is released, vnodes:%d", pVnode, vgId, count); + if (count <= 0) { taosCleanUpIntHash(tsDnodeVnodesHash); vnodeModuleInit = PTHREAD_ONCE_INIT; From e4956e3f03452ccbb76e54f5039dc80307cd30e2 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 10:54:10 +0800 Subject: [PATCH 09/36] remove invalid read in hash.c remove goto in tsched.c --- src/util/src/hash.c | 2 +- src/util/src/tsched.c | 111 ++++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 9cad14e8c7..03e155845e 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -298,7 +298,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { */ static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize, uint32_t hashVal) { - size_t totalSize = dataSize + sizeof(SHashNode) + keyLen; + size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null SHashNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 11735d91b4..8608b64057 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -40,68 +40,68 @@ static void *taosProcessSchedQueue(void *param); static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { - pthread_attr_t attr; - SSchedQueue * pSched = (SSchedQueue *)malloc(sizeof(SSchedQueue)); + SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { - pError("%s: no enough memory for pSched, reason: %s", label, strerror(errno)); - goto _error; + pError("%s: no enough memory for pSched", label); + return NULL; + } + + pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize); + if (pSched->queue == NULL) { + pError("%s: no enough memory for queue", label); + taosCleanUpScheduler(pSched); + return NULL; + } + + pSched->qthread = calloc(sizeof(pthread_t), numOfThreads); + if (pSched->qthread == NULL) { + pError("%s: no enough memory for qthread", label); + taosCleanUpScheduler(pSched); + return NULL; } - memset(pSched, 0, sizeof(SSchedQueue)); pSched->queueSize = queueSize; strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow pSched->label[sizeof(pSched->label)-1] = '\0'; - if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { - pError("init %s:queueMutex failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { - pError("init %s:empty semaphore failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if (tsem_init(&pSched->fullSem, 0, 0) != 0) { - pError("init %s:full semaphore failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if ((pSched->queue = (SSchedMsg *)malloc((size_t)pSched->queueSize * sizeof(SSchedMsg))) == NULL) { - pError("%s: no enough memory for queue, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - memset(pSched->queue, 0, (size_t)pSched->queueSize * sizeof(SSchedMsg)); pSched->fullSlot = 0; pSched->emptySlot = 0; - pSched->qthread = malloc(sizeof(pthread_t) * (size_t)numOfThreads); - if (pSched->qthread == NULL) { - pError("%s: no enough memory for qthread, reason: %s", pSched->label, strerror(errno)); - goto _error; + if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { + pError("init %s:queueMutex failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; } - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { + pError("init %s:empty semaphore failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; + } + + if (tsem_init(&pSched->fullSem, 0, 0) != 0) { + pError("init %s:full semaphore failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; + } for (int i = 0; i < numOfThreads; ++i) { - if (pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched) != 0) { - pError("%s: failed to create rpc thread, reason:%s", pSched->label, strerror(errno)); - goto _error; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + pthread_attr_destroy(&attr); + if (code != 0) { + pError("%s: failed to create rpc thread(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; } ++pSched->numOfThreads; } - pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads); - pthread_attr_destroy(&attr); + pTrace("%s scheduler is initialized, numOfThreads:%d", label, pSched->numOfThreads); return (void *)pSched; - -_error: - pthread_attr_destroy(&attr); - taosCleanUpScheduler(pSched); - return NULL; } void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { @@ -126,21 +126,21 @@ void *taosProcessSchedQueue(void *param) { pTrace("wait %s fullSem was interrupted", pSched->label); continue; } - pError("wait %s fullSem failed, errno:%d, reason:%s", pSched->label, errno, strerror(errno)); + pError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) - pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); msg = pSched->queue[pSched->fullSlot]; memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - pError("unlock %s queueMutex failed, reason:%s\n", pSched->label, strerror(errno)); + pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); if (tsem_post(&pSched->emptySem) != 0) - pError("post %s emptySem failed, reason:%s\n", pSched->label, strerror(errno)); + pError("post %s emptySem failed(%s)", pSched->label, strerror(errno)); if (msg.fp) (*(msg.fp))(&msg); @@ -160,22 +160,23 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { while (tsem_wait(&pSched->emptySem) != 0) { if (errno != EINTR) { - pError("wait %s emptySem failed, reason:%s", pSched->label, strerror(errno)); + pError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); break; } pTrace("wait %s emptySem was interrupted", pSched->label); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) - pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - pError("unlock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - if (tsem_post(&pSched->fullSem) != 0) pError("post %s fullSem failed, reason:%s", pSched->label, strerror(errno)); + if (tsem_post(&pSched->fullSem) != 0) + pError("post %s fullSem failed(%s)", pSched->label, strerror(errno)); return 0; } @@ -185,10 +186,12 @@ void taosCleanUpScheduler(void *param) { if (pSched == NULL) return; for (int i = 0; i < pSched->numOfThreads; ++i) { - pthread_cancel(pSched->qthread[i]); + if (pSched->qthread[i]) + pthread_cancel(pSched->qthread[i]); } for (int i = 0; i < pSched->numOfThreads; ++i) { - pthread_join(pSched->qthread[i], NULL); + if (pSched->qthread[i]) + pthread_join(pSched->qthread[i], NULL); } tsem_destroy(&pSched->emptySem); @@ -199,8 +202,8 @@ void taosCleanUpScheduler(void *param) { taosTmrStopA(&pSched->pTimer); } - free(pSched->queue); - free(pSched->qthread); + if (pSched->queue) free(pSched->queue); + if (pSched->qthread) free(pSched->qthread); free(pSched); // fix memory leak } From bd63cf2b4cc63c6ef386936722b4df7b5668ebe9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 12 Apr 2020 13:44:49 +0800 Subject: [PATCH 10/36] add general/user/basic1.sim --- tests/script/basicSuite.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 440ec5592b..c282480bbe 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -10,4 +10,6 @@ run general/db/basic3.sim run general/db/basic4.sim run general/db/basic5.sim +run general/user/basic1.sim + ################################## From 50d7e6876d3a7e281be267dc6326f6bd0e84b32e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 13:49:34 +0800 Subject: [PATCH 11/36] remove possible memory leaks --- src/rpc/src/rpcServer.c | 86 ++++++++++++++++++++++------------------- src/rpc/src/rpcUdp.c | 10 ++--- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 538b3059e3..6fe5385bba 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -65,79 +65,85 @@ static void taosProcessTcpData(void *param); static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - int i; SServerObj *pServerObj; - pthread_attr_t thattr; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)malloc(sizeof(SServerObj)); + pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); pServerObj->port = port; strcpy(pServerObj->label, label); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads); + pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); + free(pServerObj); return NULL; } - memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = 0; pThreadObj = pServerObj->pThreadObj; - for (i = 0; i < numOfThreads; ++i) { + for (int i = 0; i < numOfThreads; ++i) { pThreadObj->processData = fp; strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) { - tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno)); - return NULL; + code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + if (code < 0) { + tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + break;; } - if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); - return NULL; + code = pthread_cond_init(&(pThreadObj->fdReady), NULL); + if (code != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + break; } pThreadObj->pollFd = epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); - return NULL; + code = -1; + break; } - if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { - tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); - return NULL; + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + break; } pThreadObj->threadId = i; pThreadObj++; } - if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { - tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); - return NULL; + if (code == 0) { + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); + } } - /* - if ( pthread_create(&(pServerObj->thread), &thattr, - (void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) { - tError("%s failed to create UD accept thread, reason:%s", label, - strerror(errno)); - return NULL; - } - */ - pthread_attr_destroy(&thattr); - tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + if (code != 0) { + free(pServerObj->pThreadObj); + free(pServerObj); + pServerObj = NULL; + } else { + tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + } return (void *)pServerObj; } void taosCleanUpTcpServer(void *handle) { - int i; SThreadObj *pThreadObj; SServerObj *pServerObj = (SServerObj *)handle; @@ -146,7 +152,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pServerObj->thread); pthread_join(pServerObj->thread, NULL); - for (i = 0; i < pServerObj->numOfThreads; ++i) { + for (int i = 0; i < pServerObj->numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { @@ -161,9 +167,9 @@ void taosCleanUpTcpServer(void *handle) { pthread_mutex_destroy(&(pThreadObj->threadMutex)); } - tfree(pServerObj->pThreadObj); tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); + tfree(pServerObj->pThreadObj); tfree(pServerObj); } @@ -278,10 +284,10 @@ static void taosAcceptTcpConnection(void *arg) { sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); return; } else { - tTrace("%s TCP server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); } while (1) { @@ -289,11 +295,11 @@ static void taosAcceptTcpConnection(void *arg) { connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { - tError("%s TCP accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); continue; } - tTrace("%s TCP connection from ip:%s port:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosKeepTcpAlive(connFd); @@ -318,7 +324,7 @@ static void taosAcceptTcpConnection(void *arg) { event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pServerObj->label, strerror(errno)); + tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); tfree(pFdObj); close(connFd); continue; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 785288f5b9..e666187cf1 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -78,7 +78,6 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - pthread_attr_t thAttr; SUdpConn * pConn; SUdpConnSet * pSet; @@ -106,9 +105,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v } } - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - uint16_t ownPort; for (int i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -146,19 +142,21 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v pConn->tmrCtrl = pSet->tmrCtrl; } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + pthread_attr_destroy(&thAttr); if (code != 0) { tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); taosCloseSocket(pConn->fd); taosCleanUpUdpConnection(pSet); - pthread_attr_destroy(&thAttr); return NULL; } ++pSet->threads; } - pthread_attr_destroy(&thAttr); tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads); return pSet; From a5dabf7e9919851c861d3d9fa182de109733a884 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 17:05:06 +0800 Subject: [PATCH 12/36] remove the potential race condition on TCP FD --- src/rpc/src/rpcServer.c | 47 ++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 6fe5385bba..7430157bcf 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -145,7 +145,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void taosCleanUpTcpServer(void *handle) { SThreadObj *pThreadObj; - SServerObj *pServerObj = (SServerObj *)handle; + SServerObj *pServerObj = handle; if (pServerObj == NULL) return; @@ -174,15 +174,15 @@ void taosCleanUpTcpServer(void *handle) { } void taosCloseTcpServerConnection(void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; - + SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; + pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; + SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -354,14 +354,25 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pThreadObj = pFdObj->pThreadObj; - if (pThreadObj == NULL) { - tError("FdObj double clean up!!!"); - return; - } - epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } + close(pFdObj->fd); + epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); pthread_mutex_lock(&pThreadObj->threadMutex); @@ -384,24 +395,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - // notify the upper layer, so it will clean the associated context - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); - - memset(pFdObj, 0, sizeof(SFdObj)); - + pFdObj, pThreadObj->numOfFds); tfree(pFdObj); } From 8c22eed1bbfe2f0317c63be88ef0420087ba422e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 18:10:06 +0800 Subject: [PATCH 13/36] remove a potential race condition --- src/rpc/src/rpcClient.c | 26 ++++++++++++-------------- src/rpc/test/rclient.c | 2 ++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index 264449bbb0..a3105ef516 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -188,10 +188,19 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pTcp = pFdObj->pTcp; - if (pTcp == NULL) { - tError("double free TcpFdObj!!!!"); - return; + + if (pFdObj->thandle) { + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pTcp->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pTcp->processData))(&recvInfo); } epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -216,19 +225,8 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo); tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); - memset(pFdObj, 0, sizeof(STcpFd)); tfree(pFdObj); } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index f000ab91a2..732d7eb81c 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -204,6 +204,8 @@ int main(int argc, char *argv[]) { tPrint("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tPrint("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + getchar(); + taosCloseLogger(); return 0; From 5e58bd5c5f5702d82ce07507b01e5d3f8a646381 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 22:38:19 +0800 Subject: [PATCH 14/36] remove potential race condition on FdObj --- src/rpc/src/rpcClient.c | 54 +++++++++++++++++--------------- src/rpc/src/rpcMain.c | 2 +- src/rpc/src/rpcServer.c | 68 ++++++++++++++++++++++------------------- 3 files changed, 67 insertions(+), 57 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index a3105ef516..7ca24f6229 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -65,18 +65,18 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pTcp->shandle = shandle; if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno)); + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); return NULL; } if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); return NULL; } pTcp->pollFd = epoll_create(10); // size does not matter if (pTcp->pollFd < 0) { - tError("%s failed to create TCP epoll", label); + tError("%s failed to create TCP client epoll", label); return NULL; } @@ -87,11 +87,11 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); pthread_attr_destroy(&thattr); if (code != 0) { - tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); return NULL; } - tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port); + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); return pTcp; } @@ -181,15 +181,9 @@ int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - STcpClient *pTcp; +static void taosReportBrokenLink(STcpFd *pFdObj) { SRecvInfo recvInfo; - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - pTcp = pFdObj->pTcp; + STcpClient *pTcp = pFdObj->pTcp; if (pFdObj->thandle) { recvInfo.msg = NULL; @@ -202,6 +196,15 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { recvInfo.connType = RPC_CONN_TCP; (*(pTcp->processData))(&recvInfo); } +} + +static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { + + if (pFdObj == NULL) return; + if (pFdObj->signature != pFdObj) return; + + pFdObj->signature = NULL; + STcpClient *pTcp = pFdObj->pTcp; epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); close(pFdObj->fd); @@ -211,7 +214,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pTcp->numOfFds--; if (pTcp->numOfFds < 0) - tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj); + tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -225,7 +228,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); tfree(pFdObj); } @@ -250,29 +253,29 @@ static void *taosReadTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP error happened on FD\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP FD hang up\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d", pTcp->label, headLen); - taosCleanUpTcpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); if (NULL == buffer) { - tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, msgLen); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -281,9 +284,10 @@ static void *taosReadTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen); + tError("%s %p, read error, leftLen:%d retLen:%d", + pTcp->label, pFdObj->thandle, leftLen, retLen); tfree(buffer); - taosCleanUpTcpFdObj(pFdObj); + taosReportBrokenLink(pFdObj); continue; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index bec9621e3b..8280264764 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -817,7 +817,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; tTrace("%s %p, link is broken", pRpc->label, pConn); - pConn->chandle = NULL; + // pConn->chandle = NULL; if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 7430157bcf..37576fa0f6 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -177,7 +177,6 @@ void taosCloseTcpServerConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } @@ -191,6 +190,25 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void #define maxEvents 10 +static void taosReportBrokenLink(SFdObj *pFdObj) { + + SThreadObj *pThreadObj = pFdObj->pThreadObj; + + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } +} + static void taosProcessTcpData(void *param) { SThreadObj * pThreadObj; int i, fdNum; @@ -214,29 +232,29 @@ static void taosProcessTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP thread:%d, error happened on FD", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP thread:%d, FD hang up", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = malloc(msgLen + tsRpcOverhead); if ( NULL == buffer) { - tError("%s TCP malloc(size:%d) fail\n", pThreadObj->label, msgLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -245,8 +263,9 @@ static void taosProcessTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pThreadObj->label, leftLen, retLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, leftLen:%d retLen:%d", + pThreadObj->label, pFdObj->thandle, leftLen, retLen); + taosReportBrokenLink(pFdObj); tfree(buffer); continue; } @@ -339,7 +358,7 @@ static void taosAcceptTcpConnection(void *arg) { pthread_cond_signal(&pThreadObj->fdReady); pthread_mutex_unlock(&(pThreadObj->threadMutex)); - tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); // pick up next thread for next connection @@ -349,27 +368,12 @@ static void taosAcceptTcpConnection(void *arg) { } static void taosCleanUpFdObj(SFdObj *pFdObj) { - SThreadObj *pThreadObj; if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; pFdObj->signature = NULL; - pThreadObj = pFdObj->pThreadObj; - - // notify the upper layer, so it will clean the associated context - if (pFdObj->thandle) { - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pThreadObj->processData))(&recvInfo); - } + SThreadObj *pThreadObj = pFdObj->pThreadObj; close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -379,7 +383,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId); + tError("%s %p, TCP thread:%d, number of FDs is negative!!!", + pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); // remove from the FdObject list @@ -395,8 +400,9 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tfree(pFdObj); } From d2e0f7de02140879668527edb9b62af4adea6cc4 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 12 Apr 2020 22:53:40 +0800 Subject: [PATCH 15/36] fix valgrind warning --- src/vnode/tsdb/src/tsdbMain.c | 10 +++++++++- src/vnode/tsdb/src/tsdbMeta.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 6873c69c2a..ee9413fc12 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -813,6 +813,13 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) return iters; } +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + free(pMemTable); + } +} + // Commit to file static void *tsdbCommitData(void *arg) { // TODO @@ -859,7 +866,8 @@ static void *tsdbCommitData(void *arg) { // TODO: free the skiplist for (int i = 0; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { // Here has memory leak + if (pTable && pTable->imem) { + tsdbFreeMemTable(pTable->imem); pTable->imem = NULL; } } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0eb6dde1d0..0ab276c120 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -34,7 +34,7 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) { *contLen = tsdbEstimateTableEncodeSize(pTable); if (*contLen < 0) return NULL; - void *ret = malloc(*contLen); + void *ret = calloc(1, *contLen); if (ret == NULL) return NULL; void *ptr = ret; From 593e7bb79ac360f61d1486471df339fb77095185 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 13 Apr 2020 09:27:53 +0800 Subject: [PATCH 16/36] [modify webdoc] --- documentation/webdocs/markdowndocs/administrator-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index cbccc03cbb..289f2dbf91 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -105,7 +105,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 **shellActivityTimer** - 默认值：3 -系统在服务端保持结果集的最长时间，范围[1-120]。 +系统在服务端保持结果集的最长时间，单位：秒，范围[1-120]。 **maxUsers** - 默认值：10,000 @@ -138,7 +138,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 系统（服务端和客户端）运行日志开关： - 131 仅输出错误和警告信息 -- 135 输入错误（ERROR）、警告（WARN）、信息（Info） +- 135 输出错误（ERROR）、警告（WARN）、信息（Info） 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： From 65f0ff6d89e47bb6d5dcddfbd382a3d8dacfdd2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 13 Apr 2020 10:18:01 +0800 Subject: [PATCH 17/36] change tscPrint to tscTrace to avoid print debug info to shell console. --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ebac676b77..1d6f7d67e7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,7 +191,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); + tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { From 704d8da049519cfdc6e1da01e251e36e4254ddaf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:03:47 +0800 Subject: [PATCH 18/36] re-organize TCP connection code --- src/rpc/inc/rpcServer.h | 34 --- src/rpc/inc/{rpcClient.h => rpcTcp.h} | 12 +- src/rpc/src/rpcClient.c | 315 ------------------- src/rpc/src/rpcMain.c | 15 +- src/rpc/src/{rpcServer.c => rpcTcp.c} | 424 ++++++++++++-------------- 5 files changed, 212 insertions(+), 588 deletions(-) delete mode 100644 src/rpc/inc/rpcServer.h rename src/rpc/inc/{rpcClient.h => rpcTcp.h} (74%) delete mode 100644 src/rpc/src/rpcClient.c mode change 100755 => 100644 src/rpc/src/rpcMain.c rename src/rpc/src/{rpcServer.c => rpcTcp.c} (55%) diff --git a/src/rpc/inc/rpcServer.h b/src/rpc/inc/rpcServer.h deleted file mode 100644 index 6b238733a4..0000000000 --- a/src/rpc/inc/rpcServer.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_server_header_ -#define _rpc_server_header_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosdef.h" - -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosCleanUpTcpServer(void *param); -void taosCloseTcpServerConnection(void *param); -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/rpc/inc/rpcClient.h b/src/rpc/inc/rpcTcp.h similarity index 74% rename from src/rpc/inc/rpcClient.h rename to src/rpc/inc/rpcTcp.h index c87ae79312..16972dbc7e 100644 --- a/src/rpc/inc/rpcClient.h +++ b/src/rpc/inc/rpcTcp.h @@ -13,20 +13,22 @@ * along with this program. If not, see . */ -#ifndef _rpc_client_header_ -#define _rpc_client_header_ +#ifndef _rpc_tcp_header_ +#define _rpc_tcp_header_ #ifdef __cplusplus extern "C" { #endif -#include "taosdef.h" +void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle); void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port); -void taosCloseTcpClientConnection(void *chandle); -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); + +void taosCloseTcpConnection(void *chandle); +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); #ifdef __cplusplus } diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c deleted file mode 100644 index 7ca24f6229..0000000000 --- a/src/rpc/src/rpcClient.c +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosmsg.h" -#include "tlog.h" -#include "tsocket.h" -#include "tutil.h" -#include "rpcClient.h" -#include "rpcHead.h" - -#ifndef EPOLLWAKEUP -#define EPOLLWAKEUP (1u << 29) -#endif - -typedef struct _tcp_fd { - void *signature; - int fd; // TCP socket FD - void * thandle; - uint32_t ip; - char ipstr[20]; - uint16_t port; - struct _tcp_client *pTcp; - struct _tcp_fd * prev, *next; -} STcpFd; - -typedef struct _tcp_client { - pthread_t thread; - STcpFd * pHead; - pthread_mutex_t mutex; - pthread_cond_t fdReady; - int pollFd; - int numOfFds; - char label[12]; - char ipstr[20]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pRecv); -} STcpClient; - -#define maxTcpEvents 100 - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj); -static void *taosReadTcpData(void *param); - -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - STcpClient *pTcp; - pthread_attr_t thattr; - - pTcp = (STcpClient *)malloc(sizeof(STcpClient)); - memset(pTcp, 0, sizeof(STcpClient)); - strcpy(pTcp->label, label); - strcpy(pTcp->ipstr, ip); - pTcp->shandle = shandle; - - if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); - return NULL; - } - - if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); - return NULL; - } - - pTcp->pollFd = epoll_create(10); // size does not matter - if (pTcp->pollFd < 0) { - tError("%s failed to create TCP client epoll", label); - return NULL; - } - - pTcp->processData = fp; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); - pthread_attr_destroy(&thattr); - if (code != 0) { - tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); - return NULL; - } - - tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); - - return pTcp; -} - -void taosCleanUpTcpClient(void *chandle) { - STcpClient *pTcp = (STcpClient *)chandle; - if (pTcp == NULL) return; - - while (pTcp->pHead) { - taosCleanUpTcpFdObj(pTcp->pHead); - pTcp->pHead = pTcp->pHead->next; - } - - close(pTcp->pollFd); - - pthread_cancel(pTcp->thread); - pthread_join(pTcp->thread, NULL); - - // tTrace (":%s, all connections are cleaned up", pTcp->label); - - tfree(pTcp); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { - STcpClient * pTcp = (STcpClient *)shandle; - STcpFd * pFdObj; - struct epoll_event event; - struct in_addr destIp; - int fd; - - fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr); - if (fd <= 0) return NULL; - - pFdObj = (STcpFd *)malloc(sizeof(STcpFd)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pTcp->label); - tclose(fd); - return NULL; - } - - memset(pFdObj, 0, sizeof(STcpFd)); - pFdObj->fd = fd; - strcpy(pFdObj->ipstr, ip); - inet_aton(ip, &destIp); - pFdObj->ip = destIp.s_addr; - pFdObj->port = port; - pFdObj->pTcp = pTcp; - pFdObj->thandle = thandle; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno)); - tfree(pFdObj); - tclose(fd); - return NULL; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pTcp->mutex)); - pFdObj->next = pTcp->pHead; - if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj; - pTcp->pHead = pFdObj; - pTcp->numOfFds++; - pthread_cond_signal(&pTcp->fdReady); - pthread_mutex_unlock(&(pTcp->mutex)); - - tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds); - - return pFdObj; -} - -void taosCloseTcpClientConnection(void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (pFdObj == NULL) return; - - taosCleanUpTcpFdObj(pFdObj); -} - -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (chandle == NULL) return -1; - - return (int)send(pFdObj->fd, data, (size_t)len, 0); -} - -static void taosReportBrokenLink(STcpFd *pFdObj) { - SRecvInfo recvInfo; - STcpClient *pTcp = pFdObj->pTcp; - - if (pFdObj->thandle) { - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pTcp->processData))(&recvInfo); - } -} - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - STcpClient *pTcp = pFdObj->pTcp; - - epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - close(pFdObj->fd); - - pthread_mutex_lock(&pTcp->mutex); - - pTcp->numOfFds--; - - if (pTcp->numOfFds < 0) - tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); - - if (pFdObj->prev) { - (pFdObj->prev)->next = pFdObj->next; - } else { - pTcp->pHead = pFdObj->next; - } - - if (pFdObj->next) { - (pFdObj->next)->prev = pFdObj->prev; - } - - pthread_mutex_unlock(&pTcp->mutex); - - tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); - - tfree(pFdObj); -} - -static void *taosReadTcpData(void *param) { - STcpClient *pTcp = (STcpClient *)param; - int i, fdNum; - STcpFd *pFdObj; - struct epoll_event events[maxTcpEvents]; - SRecvInfo recvInfo; - SRpcHead rpcHead; - - while (1) { - pthread_mutex_lock(&pTcp->mutex); - if (pTcp->numOfFds < 1) pthread_cond_wait(&pTcp->fdReady, &pTcp->mutex); - pthread_mutex_unlock(&pTcp->mutex); - - fdNum = epoll_wait(pTcp->pollFd, events, maxTcpEvents, -1); - if (fdNum < 0) continue; - - for (i = 0; i < fdNum; ++i) { - pFdObj = events[i].data.ptr; - - if (events[i].events & EPOLLERR) { - tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - if (events[i].events & EPOLLHUP) { - tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); - if (headLen != sizeof(SRpcHead)) { - tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); - taosReportBrokenLink(pFdObj); - continue; - } - - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); - if (NULL == buffer) { - tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); - taosReportBrokenLink(pFdObj); - continue; - } - - char *msg = buffer + tsRpcOverhead; - int32_t leftLen = msgLen - headLen; - int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); - - if (leftLen != retLen) { - tError("%s %p, read error, leftLen:%d retLen:%d", - pTcp->label, pFdObj->thandle, leftLen, retLen); - tfree(buffer); - taosReportBrokenLink(pFdObj); - continue; - } - - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); - - memcpy(msg, &rpcHead, sizeof(SRpcHead)); - recvInfo.msg = msg; - recvInfo.msgLen = msgLen; - recvInfo.ip = pFdObj->ip; - recvInfo.port = pFdObj->port; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = pFdObj; - recvInfo.connType = RPC_CONN_TCP; - - pFdObj->thandle = (*(pTcp->processData))(&recvInfo); - - if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj); - } - } - - return NULL; -} - - diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c old mode 100755 new mode 100644 index 8280264764..5048d5db14 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -27,8 +27,7 @@ #include "taosmsg.h" #include "rpcUdp.h" #include "rpcCache.h" -#include "rpcClient.h" -#include "rpcServer.h" +#include "rpcTcp.h" #include "rpcHead.h" #include "trpc.h" #include "hash.h" @@ -67,7 +66,7 @@ typedef struct { void *udphandle;// returned handle from UDP initialization void *pCache; // connection cache pthread_mutex_t mutex; - struct _RpcConn *connList; // connection list + struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { @@ -88,7 +87,7 @@ typedef struct { char msg[0]; // RpcHead starts from here } SRpcReqContext; -typedef struct _RpcConn { +typedef struct SRpcConn { int sid; // session ID uint32_t ownId; // own link ID uint32_t peerId; // peer link ID @@ -156,8 +155,8 @@ void (*taosCleanUpConn[])(void *thandle) = { int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { taosSendUdpData, taosSendUdpData, - taosSendTcpServerData, - taosSendTcpClientData + taosSendTcpData, + taosSendTcpData }; void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = { @@ -170,8 +169,8 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = void (*taosCloseConn[])(void *chandle) = { NULL, NULL, - taosCloseTcpServerConnection, - taosCloseTcpClientConnection + taosCloseTcpConnection, + taosCloseTcpConnection }; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType); diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcTcp.c similarity index 55% rename from src/rpc/src/rpcServer.c rename to src/rpc/src/rpcTcp.c index 37576fa0f6..27b81deda5 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcTcp.c @@ -18,30 +18,30 @@ #include "tlog.h" #include "tsocket.h" #include "tutil.h" -#include "rpcServer.h" #include "rpcHead.h" +#include "rpcTcp.h" -#define TAOS_IPv4ADDR_LEN 16 #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) #endif -typedef struct _fd_obj { - void *signature; - int fd; // TCP socket FD - void * thandle; // handle from upper layer, like TAOS - char ipstr[TAOS_IPv4ADDR_LEN]; - unsigned int ip; - uint16_t port; - struct _thread_obj *pThreadObj; - struct _fd_obj * prev, *next; +typedef struct SFdObj { + void *signature; + int fd; // TCP socket FD + void *thandle; // handle from upper layer, like TAOS + uint32_t ip; + uint16_t port; + struct SThreadObj *pThreadObj; + struct SFdObj *prev; + struct SFdObj *next; } SFdObj; -typedef struct _thread_obj { +typedef struct SThreadObj { pthread_t thread; SFdObj * pHead; - pthread_mutex_t threadMutex; + pthread_mutex_t mutex; pthread_cond_t fdReady; + char ipstr[TSDB_IPv4ADDR_LEN]; int pollFd; int numOfFds; int threadId; @@ -51,7 +51,7 @@ typedef struct _thread_obj { } SThreadObj; typedef struct { - char ip[40]; + char ip[TSDB_IPv4ADDR_LEN]; uint16_t port; char label[12]; int numOfThreads; @@ -60,13 +60,15 @@ typedef struct { pthread_t thread; } SServerObj; -static void taosCleanUpFdObj(SFdObj *pFdObj); -static void taosProcessTcpData(void *param); -static void taosAcceptTcpConnection(void *arg); +static void *taosProcessTcpData(void *param); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd); +static void taosFreeFdObj(SFdObj *pFdObj); +static void taosReportBrokenLink(SFdObj *pFdObj); +static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; + SServerObj *pServerObj; + SThreadObj *pThreadObj; pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); @@ -88,7 +90,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + code = pthread_mutex_init(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break;; @@ -110,7 +112,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); @@ -144,8 +146,8 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, } void taosCleanUpTcpServer(void *handle) { - SThreadObj *pThreadObj; SServerObj *pServerObj = handle; + SThreadObj *pThreadObj; if (pServerObj == NULL) return; @@ -156,7 +158,7 @@ void taosCleanUpTcpServer(void *handle) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { - taosCleanUpFdObj(pThreadObj->pHead); + taosFreeFdObj(pThreadObj->pHead); pThreadObj->pHead = pThreadObj->pHead; } @@ -164,7 +166,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pThreadObj->thread); pthread_join(pThreadObj->thread, NULL); pthread_cond_destroy(&(pThreadObj->fdReady)); - pthread_mutex_destroy(&(pThreadObj->threadMutex)); + pthread_mutex_destroy(&(pThreadObj->mutex)); } tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); @@ -173,14 +175,146 @@ void taosCleanUpTcpServer(void *handle) { tfree(pServerObj); } -void taosCloseTcpServerConnection(void *chandle) { +static void taosAcceptTcpConnection(void *arg) { + int connFd = -1; + struct sockaddr_in caddr; + int sockFd; + int threadId = 0; + SThreadObj *pThreadObj; + SServerObj *pServerObj; + + pServerObj = (SServerObj *)arg; + + sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + if (sockFd < 0) return; + + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + + while (1) { + socklen_t addrlen = sizeof(caddr); + connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen); + + if (connFd < 0) { + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); + continue; + } + + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port); + taosKeepTcpAlive(connFd); + + // pick up the thread to handle this connection + pThreadObj = pServerObj->pThreadObj + threadId; + + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); + if (pFdObj) { + pFdObj->ip = caddr.sin_addr.s_addr; + pFdObj->port = caddr.sin_port; + tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); + } else { + close(connFd); + tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno)); + } + + // pick up next thread for next connection + threadId++; + threadId = threadId % pServerObj->numOfThreads; + } +} + +void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { + SThreadObj *pThreadObj; + pthread_attr_t thattr; + + pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); + memset(pThreadObj, 0, sizeof(SThreadObj)); + strcpy(pThreadObj->label, label); + strcpy(pThreadObj->ipstr, ip); + pThreadObj->shandle = shandle; + + if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); + return NULL; + } + + if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + return NULL; + } + + pThreadObj->pollFd = epoll_create(10); // size does not matter + if (pThreadObj->pollFd < 0) { + tError("%s failed to create TCP client epoll", label); + return NULL; + } + + pThreadObj->processData = fp; + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); + return NULL; + } + + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); + + return pThreadObj; +} + +void taosCleanUpTcpClient(void *chandle) { + SThreadObj *pThreadObj = chandle; + if (pThreadObj == NULL) return; + + while (pThreadObj->pHead) { + taosFreeFdObj(pThreadObj->pHead); + pThreadObj->pHead = pThreadObj->pHead->next; + } + + close(pThreadObj->pollFd); + + pthread_cancel(pThreadObj->thread); + pthread_join(pThreadObj->thread, NULL); + + tTrace (":%s, all connections are cleaned up", pThreadObj->label); + + tfree(pThreadObj); +} + +void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { + SThreadObj * pThreadObj = shandle; + struct in_addr destIp; + + int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr); + if (fd <= 0) return NULL; + + inet_aton(ip, &destIp); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); + + if (pFdObj) { + pFdObj->thandle = thandle; + pFdObj->port = port; + pFdObj->ip = destIp.s_addr; + tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d", + pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds); + } else { + close(fd); + tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); + } + + return pFdObj; +} + +void taosCloseTcpConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - taosCleanUpFdObj(pFdObj); + taosFreeFdObj(pFdObj); } -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -188,8 +322,6 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -#define maxEvents 10 - static void taosReportBrokenLink(SFdObj *pFdObj) { SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -209,26 +341,26 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } } -static void taosProcessTcpData(void *param) { - SThreadObj * pThreadObj; - int i, fdNum; - SFdObj * pFdObj; +#define maxEvents 10 + +static void *taosProcessTcpData(void *param) { + SThreadObj *pThreadObj = param; + SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - pThreadObj = (SThreadObj *)param; SRpcHead rpcHead; while (1) { - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); if (pThreadObj->numOfFds < 1) { - pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->threadMutex); + pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex); } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); - fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); + int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); if (fdNum < 0) continue; - for (i = 0; i < fdNum; ++i) { + for (int i = 0; i < fdNum; ++i) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { @@ -270,7 +402,7 @@ static void taosProcessTcpData(void *param) { continue; } - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); + // tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen); memcpy(msg, &rpcHead, sizeof(SRpcHead)); recvInfo.msg = msg; @@ -283,91 +415,43 @@ static void taosProcessTcpData(void *param) { recvInfo.connType = RPC_CONN_TCP; pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo); - if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj); + if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } } + + return NULL; } -static void taosAcceptTcpConnection(void *arg) { - int connFd = -1; - struct sockaddr_in clientAddr; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) { struct epoll_event event; - pServerObj = (SServerObj *)arg; + SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + if (pFdObj == NULL) return NULL; - sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + pFdObj->fd = fd; + pFdObj->pThreadObj = pThreadObj; + pFdObj->signature = pFdObj; - if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; + event.data.ptr = pFdObj; + if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + tfree(pFdObj); + return NULL; } - while (1) { - socklen_t addrlen = sizeof(clientAddr); - connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); + // notify the data process, add into the FdObj list + pthread_mutex_lock(&(pThreadObj->mutex)); + pFdObj->next = pThreadObj->pHead; + if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; + pThreadObj->pHead = pFdObj; + pThreadObj->numOfFds++; + pthread_cond_signal(&pThreadObj->fdReady); + pthread_mutex_unlock(&(pThreadObj->mutex)); - if (connFd < 0) { - tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); - continue; - } - - tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); - taosKeepTcpAlive(connFd); - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - strcpy(pFdObj->ipstr, inet_ntoa(clientAddr.sin_addr)); - pFdObj->ip = clientAddr.sin_addr.s_addr; - pFdObj->port = htons(clientAddr.sin_port); - pFdObj->pThreadObj = pThreadObj; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, - pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } + return pFdObj; } -static void taosCleanUpFdObj(SFdObj *pFdObj) { +static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -378,7 +462,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); pThreadObj->numOfFds--; @@ -398,7 +482,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); @@ -406,116 +490,4 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { tfree(pFdObj); } -#if 0 -static void taosAcceptUDConnection(void *arg) { - int connFd = -1; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; - struct epoll_event event; - - pServerObj = (SServerObj *)arg; - sockFd = taosOpenUDServerSocket(pServerObj->ip, pServerObj->port); - - if (sockFd < 0) { - tError("%s failed to open UD socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s UD server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - } - - while (1) { - connFd = accept(sockFd, NULL, NULL); - - if (connFd < 0) { - tError("%s UD accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); - continue; - } - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - pFdObj->pThreadObj = pThreadObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add UD FD for epoll, error:%s", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } -} -#endif - -#if 0 -void taosListTcpConnection(void *handle, char *buffer) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; - SFdObj * pFdObj; - int i, numOfFds, numOfConns; - char * msg; - - pServerObj = (SServerObj *)handle; - buffer[0] = 0; - msg = buffer; - numOfConns = 0; - - pThreadObj = pServerObj->pThreadObj; - - for (i = 0; i < pServerObj->numOfThreads; ++i) { - numOfFds = 0; - sprintf(msg, "TCP:%s Thread:%d number of connections:%d\n", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - msg = msg + strlen(msg); - pFdObj = pThreadObj->pHead; - while (pFdObj) { - sprintf(msg, " ip:%s port:%hu\n", pFdObj->ipstr, pFdObj->port); - msg = msg + strlen(msg); - numOfFds++; - numOfConns++; - pFdObj = pFdObj->next; - } - - if (numOfFds != pThreadObj->numOfFds) - tError("TCP:%s thread:%d BIG error, numOfFds:%d actual numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds, numOfFds); - - pThreadObj++; - } - - sprintf(msg, "TCP:%s total connections:%d\n", pServerObj->label, numOfConns); - - return; -} -#endif - From e01081f2755596a1822bd4ea1d2cda189bb40468 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:50:30 +0800 Subject: [PATCH 19/36] rename some structures --- src/rpc/src/rpcCache.c | 10 +++++----- src/rpc/src/rpcHaship.c | 14 +++++++------- src/rpc/src/rpcUdp.c | 24 ++++++++++++------------ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index a4863ef61d..3370b31277 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -22,18 +22,18 @@ #include "tutil.h" #include "rpcCache.h" -typedef struct _c_hash_t { +typedef struct SConnHash { uint32_t ip; uint16_t port; char connType; - struct _c_hash_t *prev; - struct _c_hash_t *next; - void * data; + struct SConnHash *prev; + struct SConnHash *next; + void *data; uint64_t time; } SConnHash; typedef struct { - SConnHash ** connHashList; + SConnHash **connHashList; mpool_h connHashMemPool; int maxSessions; int total; diff --git a/src/rpc/src/rpcHaship.c b/src/rpc/src/rpcHaship.c index c904dba4a1..43f2d138de 100644 --- a/src/rpc/src/rpcHaship.c +++ b/src/rpc/src/rpcHaship.c @@ -17,13 +17,13 @@ #include "tlog.h" #include "tmempool.h" -typedef struct _ip_hash_t { +typedef struct SIpHash { uint32_t ip; uint16_t port; int hash; - struct _ip_hash_t *prev; - struct _ip_hash_t *next; - void * data; + struct SIpHash *prev; + struct SIpHash *next; + void *data; } SIpHash; typedef struct { @@ -47,7 +47,7 @@ int rpcHashIp(void *handle, uint32_t ip, uint16_t port) { void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -70,7 +70,7 @@ void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -102,7 +102,7 @@ void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index e666187cf1..1e5ac3c6b5 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -32,7 +32,7 @@ int tsUdpDelay = 0; typedef struct { - void * signature; + void *signature; int index; int fd; uint16_t port; // peer port @@ -53,23 +53,23 @@ typedef struct { int server; char ip[16]; // local IP uint16_t port; // local Port - void * shandle; // handle passed by upper layer during server initialization + void *shandle; // handle passed by upper layer during server initialization int threads; char label[12]; - void * tmrCtrl; + void *tmrCtrl; void *(*fp)(SRecvInfo *pPacket); SUdpConn udpConn[]; } SUdpConnSet; typedef struct { - void * signature; + void *signature; uint32_t ip; // dest IP uint16_t port; // dest Port - SUdpConn * pConn; + SUdpConn *pConn; struct sockaddr_in destAdd; - void * msgHdr; + void *msgHdr; int totalLen; - void * timer; + void *timer; int emptyNum; } SUdpBuf; @@ -78,8 +78,8 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn * pConn; - SUdpConnSet * pSet; + SUdpConn *pConn; + SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); pSet = (SUdpConnSet *)malloc((size_t)size); @@ -164,7 +164,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn * pConn; + SUdpConn *pConn; if (pSet == NULL) return; @@ -205,10 +205,10 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por } static void *taosRecvUdpData(void *param) { + SUdpConn *pConn = param; struct sockaddr_in sourceAdd; int dataLen; unsigned int addLen; - SUdpConn * pConn = (SUdpConn *)param; uint16_t port; int minSize = sizeof(SRpcHead); SRecvInfo recvInfo; @@ -274,7 +274,7 @@ static void *taosRecvUdpData(void *param) { int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { SUdpConn *pConn = (SUdpConn *)chandle; - SUdpBuf * pBuf; + SUdpBuf *pBuf; if (pConn == NULL || pConn->signature != pConn) return -1; From f2b41206ce816b3e751b89fca931058aaa271bb4 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 19:25:08 +0800 Subject: [PATCH 20/36] rename a structure --- src/dnode/src/dnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index b56e0d8ad7..7eb4add831 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -41,7 +41,7 @@ typedef struct { SRpcMsg rpcMsg; } SWriteMsg; -typedef struct _wworker_pool { +typedef struct { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic SWriteWorker *writeWorker; From 34a3ac2436bab39f40261381d5aafa0d66ba0a94 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 13 Apr 2020 19:25:38 +0800 Subject: [PATCH 21/36] [TD-52] make sdb use wal --- src/dnode/src/dnodeMClient.c | 95 +- src/mnode/CMakeLists.txt | 2 +- src/mnode/inc/mgmtSdb.h | 19 +- src/mnode/src/mgmtDb.c | 19 +- src/mnode/src/mgmtMain.c | 8 +- src/mnode/src/mgmtSdb.c | 984 +++++-------- src/mnode/src/mgmtTable.c | 68 +- src/mnode/src/mgmtUser.c | 29 +- src/mnode/src/mgmtVgroup.c | 17 +- src/util/inc/cJSON.h | 3 + src/util/src/cJSON.c | 3 + tests/tsim/inc/cJSON.h | 267 ---- tests/tsim/src/cJSON.c | 2702 ---------------------------------- 13 files changed, 531 insertions(+), 3685 deletions(-) delete mode 100644 tests/tsim/inc/cJSON.h delete mode 100644 tests/tsim/src/cJSON.c diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index c09583bd65..f9794d040b 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -34,27 +34,26 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void *tsDnodeMClientRpc = NULL; -static SRpcIpSet tsMpeerIpList = {0}; -static SDMNodeInfos tsMpeerInfos = {0}; +static SRpcIpSet tsMnodeIpList = {0}; +static SDMNodeInfos tsMnodeInfos = {0}; int32_t dnodeInitMClient() { if (!dnodeReadMnodeIpList()) { - memset(&tsMpeerIpList, 0, sizeof(SRpcIpSet)); - memset(&tsMpeerInfos, 0, sizeof(SDMNodeInfos)); - tsMpeerIpList.port = tsMnodeDnodePort; - tsMpeerIpList.numOfIps = 1; - tsMpeerIpList.ip[0] = inet_addr(tsMasterIp); + memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet)); + memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos)); + tsMnodeIpList.port = tsMnodeDnodePort; + tsMnodeIpList.numOfIps = 1; + tsMnodeIpList.ip[0] = inet_addr(tsMasterIp); if (tsSecondIp[0]) { - tsMpeerIpList.numOfIps = 2; - tsMpeerIpList.ip[1] = inet_addr(tsSecondIp); + tsMnodeIpList.numOfIps = 2; + tsMnodeIpList.ip[1] = inet_addr(tsSecondIp); } } else { - SRpcIpSet mgmtIpSet = {0}; - tsMpeerIpList.inUse = tsMpeerInfos.inUse; - tsMpeerIpList.numOfIps = tsMpeerInfos.nodeNum; - tsMpeerIpList.port = tsMpeerInfos.nodeInfos[0].nodePort; - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - tsMpeerIpList.ip[i] = tsMpeerInfos.nodeInfos[i].nodeIp; + tsMnodeIpList.inUse = tsMnodeInfos.inUse; + tsMnodeIpList.numOfIps = tsMnodeInfos.nodeNum; + tsMnodeIpList.port = tsMnodeInfos.nodeInfos[0].nodePort; + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + tsMnodeIpList.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; } } @@ -123,17 +122,17 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp); } - if (memcmp(&mgmtIpSet, &tsMpeerIpList, sizeof(SRpcIpSet)) != 0) { - memcpy(&tsMpeerIpList, &mgmtIpSet, sizeof(SRpcIpSet)); - memcpy(&tsMpeerInfos, mpeers, sizeof(SDMNodeInfos)); - dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0) { + memcpy(&tsMnodeIpList, &mgmtIpSet, sizeof(SRpcIpSet)); + memcpy(&tsMnodeInfos, mpeers, sizeof(SDMNodeInfos)); + dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); for (int32_t i = 0; i < mpeers->nodeNum; i++) { - tsMpeerInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); - tsMpeerInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); - tsMpeerInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); - dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, - taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, - tsMpeerInfos.nodeInfos[i].nodeName); + tsMnodeInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); + tsMnodeInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); + tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); + dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, + taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, + tsMnodeInfos.nodeInfos[i].nodeName); } dnodeSaveMnodeIpList(); } @@ -150,7 +149,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { if (tsDnodeMClientRpc) { - rpcSendRequest(tsDnodeMClientRpc, &tsMpeerIpList, rpcMsg); + rpcSendRequest(tsDnodeMClientRpc, &tsMnodeIpList, rpcMsg); } } @@ -185,14 +184,14 @@ static bool dnodeReadMnodeIpList() { dError("failed to read mnode mgmtIpList.json, inUse not found"); goto PARSE_OVER; } - tsMpeerInfos.inUse = inUse->valueint; + tsMnodeInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnode mgmtIpList.json, nodeNum not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeNum = nodeNum->valueint; + tsMnodeInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -201,7 +200,7 @@ static bool dnodeReadMnodeIpList() { } int size = cJSON_GetArraySize(nodeInfos); - if (size != tsMpeerInfos.nodeNum) { + if (size != tsMnodeInfos.nodeNum) { dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched"); goto PARSE_OVER; } @@ -215,37 +214,37 @@ static bool dnodeReadMnodeIpList() { dError("failed to read mnode mgmtIpList.json, nodeId not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodeId = nodeId->valueint; + tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { dError("failed to read mnode mgmtIpList.json, nodeIp not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); + tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort"); if (!nodePort || nodePort->type != cJSON_Number) { dError("failed to read mnode mgmtIpList.json, nodePort not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; + tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { dError("failed to read mnode mgmtIpList.json, nodeName not found"); goto PARSE_OVER; } - strncpy(tsMpeerInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); + strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); } ret = true; - dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, - taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, - tsMpeerInfos.nodeInfos[i].nodeName); + dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, + taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, + tsMnodeInfos.nodeInfos[i].nodeName); } PARSE_OVER: @@ -265,15 +264,15 @@ static void dnodeSaveMnodeIpList() { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMpeerInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMpeerInfos.nodeNum); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMnodeInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMpeerInfos.nodeInfos[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMpeerInfos.nodeInfos[i].nodeIp)); - len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMpeerInfos.nodeInfos[i].nodePort); - len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMpeerInfos.nodeInfos[i].nodeName); - if (i < tsMpeerInfos.nodeNum -1) { + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp)); + len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort); + len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName); + if (i < tsMnodeInfos.nodeNum -1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -289,9 +288,9 @@ static void dnodeSaveMnodeIpList() { } uint32_t dnodeGetMnodeMasteIp() { - return tsMpeerIpList.ip[tsMpeerIpList.inUse]; + return tsMnodeIpList.ip[tsMnodeIpList.inUse]; } void* dnodeGetMpeerInfos() { - return &tsMpeerInfos; + return &tsMnodeInfos; } \ No newline at end of file diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index b830695f52..2e975f089c 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,5 +13,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) + #TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) ENDIF () \ No newline at end of file diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index ecf6887b42..95f3e6d39b 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,6 +20,18 @@ extern "C" { #endif +typedef enum { + SDB_TABLE_MNODE = 0, + SDB_TABLE_DNODE = 1, + SDB_TABLE_ACCOUNT = 2, + SDB_TABLE_USER = 3, + SDB_TABLE_DB = 4, + SDB_TABLE_VGROUP = 5, + SDB_TABLE_STABLE = 6, + SDB_TABLE_CTABLE = 7, + SDB_TABLE_MAX = 8 +} ESdbTable; + typedef enum { SDB_KEY_TYPE_STRING, SDB_KEY_TYPE_AUTO @@ -34,8 +46,6 @@ typedef struct { ESdbOperType type; void * table; void * pObj; - int64_t version; - int32_t maxRowSize; int32_t rowSize; void * rowData; } SSdbOperDesc; @@ -45,6 +55,7 @@ typedef struct { int32_t hashSessions; int32_t maxRowSize; int32_t refCountPos; + ESdbTable tableId; ESdbKeyType keyType; int32_t (*insertFp)(SSdbOperDesc *pOper); int32_t (*deleteFp)(SSdbOperDesc *pOper); @@ -52,8 +63,12 @@ typedef struct { int32_t (*encodeFp)(SSdbOperDesc *pOper); int32_t (*decodeFp)(SSdbOperDesc *pDesc); int32_t (*destroyFp)(SSdbOperDesc *pDesc); + int32_t (*updateAllFp)(); } SSdbTableDesc; +int32_t sdbInit(); +void sdbCleanUp(); + void * sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index b1931347a7..2de9656e3e 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -88,14 +88,9 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { SDbObj *pDb = pOper->pObj; - - if (pOper->maxRowSize < tsDbUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pDb, tsDbUpdateSize); - pOper->rowSize = tsDbUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { @@ -107,11 +102,16 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtDbActionUpdateAll() { + return 0; +} + int32_t mgmtInitDbs() { SDbObj tObj; tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_DB, .tableName = "dbs", .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, @@ -123,6 +123,7 @@ int32_t mgmtInitDbs() { .encodeFp = mgmtDbActionEncode, .decodeFp = mgmtDbActionDecode, .destroyFp = mgmtDbActionDestroy, + .updateAllFp = mgmtDbActionUpdateAll }; tsDbSdb = sdbOpenTable(&tableDesc); @@ -136,7 +137,7 @@ int32_t mgmtInitDbs() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DB, mgmtGetDbMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mgmtRetrieveDbs); - + mTrace("db data is initialized"); return 0; } diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index cb3c9fae1e..a074060f52 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -105,7 +105,12 @@ int32_t mgmtStartSystem() { } if (mgmtInitTables() < 0) { - mError("failed to init meters"); + mError("failed to init tables"); + return -1; + } + + if (sdbInit() < 0) { + mError("failed to init sdb"); return -1; } @@ -158,6 +163,7 @@ void mgmtCleanUpSystem() { clusterCleanUp(); mgmtCleanUpUsers(); acctCleanUp(); + sdbCleanUp(); taosTmrCleanUp(tsMgmtTmr); mPrint("mgmt is cleaned up"); } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index f3fc7d07f5..6a229e716c 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -22,37 +22,30 @@ #include "tlog.h" #include "trpc.h" #include "tutil.h" +#include "twal.h" +#include "tsync.h" #include "hashint.h" #include "hashstr.h" - #include "mgmtSdb.h" -#define abs(x) (((x) < 0) ? -(x) : (x)) -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF - typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[2]; - TSCKSUM checkSum; -} SSdbHeader; + int32_t code; + int64_t version; + void * pSync; + void * pWal; + sem_t sem; + pthread_mutex_t mutex; +} SSdbSync; typedef struct _SSdbTable { - SSdbHeader header; - char tableName[TSDB_DB_NAME_LEN]; - char fileName[TSDB_FILENAME_LEN]; + char tableName[TSDB_DB_NAME_LEN + 1]; + ESdbTable tableId; ESdbKeyType keyType; - int32_t tableId; int32_t hashSessions; int32_t maxRowSize; int32_t refCountPos; int32_t autoIndex; - int32_t fd; int64_t numOfRows; - int64_t version; - int64_t fileSize; void * iHandle; int32_t (*insertFp)(SSdbOperDesc *pDesc); int32_t (*deleteFp)(SSdbOperDesc *pOper); @@ -60,55 +53,48 @@ typedef struct _SSdbTable { int32_t (*decodeFp)(SSdbOperDesc *pOper); int32_t (*encodeFp)(SSdbOperDesc *pOper); int32_t (*destroyFp)(SSdbOperDesc *pOper); + int32_t (*updateAllFp)(); pthread_mutex_t mutex; } SSdbTable; typedef struct { - int64_t version; - int64_t offset; int32_t rowSize; void * row; } SRowMeta; -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t version; - char data[]; -} SRowHead; - typedef enum { - SDB_FORWARD_TYPE_INSERT, - SDB_FORWARD_TYPE_DELETE, - SDB_FORWARD_TYPE_UPDATE -} ESdbForwardType; + SDB_ACTION_INSERT, + SDB_ACTION_DELETE, + SDB_ACTION_UPDATE +} ESdbActionType; -typedef struct { - ESdbForwardType type; - int32_t tableId; - int64_t version; - int32_t rowSize; - void * rowData; -} SForwardMsg; - -extern char version[]; -const int16_t sdbFileVersion = 2; -int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; - -static SSdbTable *sdbTableList[10] = {0}; -static int32_t sdbNumOfTables = 0; -static uint64_t sdbVersion = 0; +static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0}; +static int32_t tsSdbNumOfTables = 0; +static SSdbSync * tsSdbSync; static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; -static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; -static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; +static int sdbProcessWrite(void *param, void *data, int type); -uint64_t sdbGetVersion() { return sdbVersion; } -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } +uint64_t sdbGetVersion() { return tsSdbSync->version; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + +static char *sdbGetActionStr(int32_t action) { + switch (action) { + case SDB_ACTION_INSERT: + return "insert"; + case SDB_ACTION_DELETE: + return "delete"; + case SDB_ACTION_UPDATE: + return "update"; + } + return "invalid"; +} static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { static char str[16]; @@ -123,300 +109,94 @@ static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { } } -static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(forwardMsg); - } else { - return 0; - } +static void *sdbGetTableFromId(int32_t tableId) { + return tsSdbTableList[tableId]; } -static void sdbFinishCommit(SSdbTable *pTable) { - uint32_t sdbEcommit = SDB_ENDCOMMIT; - off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->fileSize); - twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->fileSize += sizeof(sdbEcommit); -} +// static void mpeerConfirmForward(void *ahandle, void *param, int32_t code) { +// sem_post(&tsSdbSync->sem); +// mPrint("mpeerConfirmForward"); +// } -static int32_t sdbOpenSdbFile(SSdbTable *pTable) { - struct stat fstat, ofstat; - uint64_t size; - char * dirc = NULL; - char * basec = NULL; - union { - char cversion[64]; - uint64_t iversion; - } swVersion; - - memcpy(swVersion.cversion, version, sizeof(uint64_t)); - - // check sdb.db and .sdb.db status - char fn[TSDB_FILENAME_LEN] = "\0"; - dirc = strdup(pTable->fileName); - basec = strdup(pTable->fileName); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - tfree(dirc); - tfree(basec); - if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fileName, &fstat) == 0) { - remove(fn); - } else { - remove(pTable->fileName); - rename(fn, pTable->fileName); - } - } - - pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pTable->fd < 0) { - sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); - return -1; - } - - pTable->fileSize = 0; - stat(pTable->fileName, &fstat); - size = sizeof(pTable->header); - - if (fstat.st_size == 0) { - pTable->header.swVersion = swVersion.iversion; - pTable->header.sdbFileVersion = sdbFileVersion; - if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - twrite(pTable->fd, &(pTable->header), size); - pTable->fileSize += size; - sdbFinishCommit(pTable); - } else { - uint32_t sdbEcommit = 0; - off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); - while (offset > 0) { - read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - if (sdbEcommit == SDB_ENDCOMMIT) { - ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); - break; - } - offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); - } - lseek(pTable->fd, 0, SEEK_SET); - - ssize_t tsize = read(pTable->fd, &(pTable->header), size); - if (tsize < size) { - sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - - if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); - } - - if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - - pTable->fileSize += size; - // skip end commit symbol - lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->fileSize += sizeof(sdbEcommit); - } - - pTable->numOfRows = 0; - - return pTable->fd; -} - -static int32_t sdbInitTableByFile(SSdbTable *pTable) { - sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); - if (sdbOpenSdbFile(pTable) < 0) { - sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); - return -1; - } +static int32_t sdbForwardDbReqToPeer(SWalHead *pHead) { + // int32_t code = syncForwardToPeer(NULL, pHead, NULL); + // if (code < 0) { + // return code; + // } - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); + // sem_wait(&tsSdbSync->sem); + // return tsSdbSync->code; + return TSDB_CODE_SUCCESS; +} + +int32_t sdbInit() { + tsSdbSync = calloc(1, sizeof(SSdbSync)); + sem_init(&tsSdbSync->sem, 0, 0); + pthread_mutex_init(&tsSdbSync->mutex, NULL); + + SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; + tsSdbSync->pWal = walOpen(tsMnodeDir, &walCfg); + if (tsSdbSync->pWal == NULL) { + sdbError("failed to open sdb in %s", tsMnodeDir); return -1; } - int32_t numOfChanged = 0; - int32_t maxAutoIndex = 0; - while (1) { - memset(rowHead, 0, total_size); + sdbTrace("open sdb file for read"); + walRestore(tsSdbSync->pWal, tsSdbSync, sdbProcessWrite); - int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); - tfree(rowHead); - return -1; + int32_t totalRows = 0; + int32_t numOfTables = 0; + for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) { + SSdbTable *pTable = sdbGetTableFromId(tableId); + if (pTable == NULL) continue; + if (pTable->updateAllFp) { + (*pTable->updateAllFp)(); } - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->fileSize++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, - pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); - pTable->fileSize += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->fileSize not end of the file - sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, - rowHead->rowSize); - break; - } - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); - pTable->fileSize += real_size; - continue; - } - - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - pTable->version = MAX(pTable->version, abs(rowHead->version)); - - void *pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { - if (rowHead->version < 0) { - sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, - sdbGetkeyStr(pTable, rowHead->data)); - } else { - SRowMeta rowMeta; - rowMeta.version = rowHead->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .rowSize = rowHead->rowSize - }; - int32_t code = (*pTable->decodeFp)(&oper); - if (code == TSDB_CODE_SUCCESS) { - rowMeta.row = oper.pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } - } - } else { - if (rowHead->version < 0) { - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - pTable->numOfRows--; - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - SRowMeta rowMeta; - rowMeta.version = rowHead->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .rowSize = rowHead->rowSize, - .pObj = pMetaRow - }; - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - - int32_t code = (*pTable->decodeFp)(&oper); - if (code == TSDB_CODE_SUCCESS) { - rowMeta.row = oper.pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } - } - numOfChanged++; - } - - pTable->fileSize += real_size; - pTable->fileSize += 4; - lseek(pTable->fd, 4, SEEK_CUR); + totalRows += pTable->numOfRows; + numOfTables++; + sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows); } - void *pNode = NULL; - while (1) { - SRowMeta * pMeta; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - SSdbOperDesc oper = { - .pObj = pMeta->row, - .table = pTable, - .version = pMeta->version, - }; - - sdbIncRef(pTable, oper.pObj); - int32_t code = (*pTable->insertFp)(&oper); - if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); - } - } - - sdbVersion += pTable->version; - - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { - pTable->autoIndex = maxAutoIndex; - } - - tfree(rowHead); - return 0; + sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbSync->version, totalRows, numOfTables); + return TSDB_CODE_SUCCESS; } -void *sdbOpenTable(SSdbTableDesc *pDesc) { - SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); - if (pTable == NULL) return NULL; - - pTable->keyType = pDesc->keyType; - pTable->hashSessions = pDesc->hashSessions; - pTable->maxRowSize = pDesc->maxRowSize; - pTable->refCountPos = pDesc->refCountPos; - pTable->insertFp = pDesc->insertFp; - pTable->deleteFp = pDesc->deleteFp; - pTable->updateFp = pDesc->updateFp; - pTable->encodeFp = pDesc->encodeFp; - pTable->decodeFp = pDesc->decodeFp; - pTable->destroyFp = pDesc->destroyFp; - strcpy(pTable->tableName, pDesc->tableName); - sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); - - if (sdbInitIndexFp[pTable->keyType] != NULL) { - pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); +void sdbCleanUp() { + if (tsSdbSync) { + sem_destroy(&tsSdbSync->sem); + pthread_mutex_destroy(&tsSdbSync->mutex); + walClose(tsSdbSync->pWal); + tsSdbSync = NULL; } +} - pthread_mutex_init(&pTable->mutex, NULL); +void sdbIncRef(void *handle, void *pRow) { + if (pRow) { + SSdbTable *pTable = handle; + int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + atomic_add_fetch_32(pRefCount, 1); + //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + //} + } +} - if (sdbInitTableByFile(pTable) < 0) return NULL; - - pTable->tableId = sdbNumOfTables++; - sdbTableList[pTable->tableId] = pTable; - - sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); - - return pTable; +void sdbDecRef(void *handle, void *pRow) { + if (pRow) { + SSdbTable *pTable = handle; + int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + //} + int8_t* updateEnd = pRow + pTable->refCountPos - 1; + if (refCount <= 0 && *updateEnd) { + sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + SSdbOperDesc oper = {.pObj = pRow}; + (*pTable->destroyFp)(&oper); + } + } } static SRowMeta *sdbGetRowMeta(void *handle, void *key) { @@ -430,34 +210,6 @@ static SRowMeta *sdbGetRowMeta(void *handle, void *key) { return pMeta; } -void sdbIncRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - atomic_add_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - } - } -} - -void sdbDecRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - } - int8_t* updateEnd = pRow + pTable->refCountPos - 1; - if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - SSdbOperDesc oper = {.pObj = pRow}; - (*pTable->destroyFp)(&oper); - } - } -} - void *sdbGetRow(void *handle, void *key) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; @@ -476,12 +228,167 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } +static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + SRowMeta rowMeta; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + + pthread_mutex_lock(&pTable->mutex); + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + sdbIncRef(pTable, pOper->pObj); + pTable->numOfRows++; + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->insertFp)(pOper); + return TSDB_CODE_SUCCESS; +} + +static int32_t sdbDeleteLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + pthread_mutex_lock(&pTable->mutex); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + pTable->numOfRows--; + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->deleteFp)(pOper); + int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; + *updateEnd = 1; + sdbDecRef(pTable, pOper->pObj); + + return TSDB_CODE_SUCCESS; +} + +static int32_t sdbUpdateLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->updateFp)(pOper); + return TSDB_CODE_SUCCESS; +} + +static int sdbProcessWrite(void *param, void *data, int type) { + SWalHead *pHead = data; + int32_t code = 0; + int32_t tableId = pHead->msgType / 10; + int32_t action = pHead->msgType % 10; + + SSdbTable *pTable = sdbGetTableFromId(tableId); + assert(pTable != NULL); + + if (pHead->version == 0) { + // from mgmt, update version + pthread_mutex_lock(&tsSdbSync->mutex); + tsSdbSync->version++; + pHead->version = tsSdbSync->version; + + code = sdbForwardDbReqToPeer(pHead); + if (code != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + return code; + } + + code = walWrite(tsSdbSync->pWal, pHead); + pthread_mutex_unlock(&tsSdbSync->mutex); + + if (code < 0) { + sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + } else { + sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), + sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + walFsync(tsSdbSync->pWal); + free(pHead); + + return code; + } else { + // for data from WAL or forward, version may be smaller + pthread_mutex_lock(&tsSdbSync->mutex); + + if (pHead->version <= tsSdbSync->version) { + pthread_mutex_unlock(&tsSdbSync->mutex); + return TSDB_CODE_SUCCESS; + } else if (pHead->version != tsSdbSync->version + 1) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, + pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + tsSdbSync->version); + return TSDB_CODE_OTHERS; + } else { + tsSdbSync->version = pHead->version; + sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + + + code = -1; + if (action == SDB_ACTION_INSERT) { + SSdbOperDesc oper = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + + code = sdbInsertLocal(pTable, &oper); + } else if (action == SDB_ACTION_DELETE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper = { + .table = pTable, + .pObj = rowMeta->row, + }; + + code = sdbDeleteLocal(pTable, &oper); + } else if (action == SDB_ACTION_UPDATE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper1 = { + .table = pTable, + .pObj = rowMeta->row, + }; + sdbDeleteLocal(pTable, &oper1); + + SSdbOperDesc oper2 = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper2); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + code = sdbInsertLocal(pTable, &oper2); + } + + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } +} + int32_t sdbInsertRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; - if (pTable == NULL) { - sdbError("sdb tables is null"); - return TSDB_CODE_OTHERS; - } + if (pTable == NULL) return -1; if (sdbGetRow(pTable, pOper->pObj)) { sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); @@ -489,87 +396,33 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } - pOper->maxRowSize = pTable->maxRowSize; - pthread_mutex_lock(&pTable->mutex); - - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_INSERT, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, - }; - - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); - pthread_mutex_unlock(&pTable->mutex); - return TSDB_CODE_OTHERS; - } - } - - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - pthread_mutex_unlock(&pTable->mutex); - sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); - return -1; - } - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + pthread_mutex_lock(&pTable->mutex); *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; // let vgId increase from 2 if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; } - } - pTable->version++; - sdbVersion++; - - pOper->rowData = rowHead->data; - (*pTable->encodeFp)(pOper); - rowHead->rowSize = pOper->rowSize; - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); - pTable->version--; - sdbVersion--; pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; } - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - tfree(rowHead); - - // update in SDB layer - SRowMeta rowMeta; - rowMeta.version = pTable->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = pOper->rowSize; - rowMeta.row = pOper->pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); - sdbIncRef(pTable, pOper->pObj); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + int32_t size = sizeof(SWalHead) + pTable->maxRowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->len = pOper->rowSize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; - pTable->numOfRows++; + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; + } - pthread_mutex_unlock(&pTable->mutex); - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - (*pTable->insertFp)(pOper); - - return 0; + return sdbInsertLocal(pTable, pOper); } // row here can be object or null-terminated string @@ -586,204 +439,61 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - pthread_mutex_lock(&pTable->mutex); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_DELETE, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pMeta->rowSize, - .rowData = pMeta->row, - }; + int32_t rowSize = 0; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->pObj) + 1; + break; + case SDB_KEY_TYPE_AUTO: + rowSize = sizeof(uint64_t); + break; + default: + return -1; + } - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to delete record", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - } + int32_t size = sizeof(SWalHead) + rowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->len = rowSize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; + memcpy(pHead->cont, pOper->pObj, rowSize); - int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - - pTable->version++; - sdbVersion++; - - int32_t rowSize = 0; - switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: - rowSize = strlen((char *)pOper->pObj) + 1; - break; - case SDB_KEY_TYPE_AUTO: - rowSize = sizeof(uint64_t); - break; - default: - return -1; - } - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->version = -(pTable->version); - memcpy(rowHead->data, pOper->pObj, rowSize); - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); - pTable->version--; - sdbVersion--; - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - - tfree(rowHead); - - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); - - pTable->numOfRows--; - - pthread_mutex_unlock(&pTable->mutex); - - (*pTable->deleteFp)(pOper); - int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; - *updateEnd = 1; - sdbDecRef(pTable, pOper->pObj); - return 0; + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; + } + + return sdbDeleteLocal(pTable, pOper); } -// row here can be the object or the string info (encoded string) int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; } - void *pMetaRow = pMeta->row; + void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - pthread_mutex_lock(&pTable->mutex); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_UPDATE, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, - }; - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to update record", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - } + int32_t size = sizeof(SWalHead) + pTable->maxRowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - pthread_mutex_unlock(&pTable->mutex); - sdbError("table:%s, failed to allocate row head memory", pTable->tableName); - return -1; + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; } - if (pMetaRow != pOper->pObj) { - memcpy(rowHead->data, pOper->rowData, pOper->rowSize); - rowHead->rowSize = pOper->rowSize; - } else { - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .maxRowSize = pTable->maxRowSize, - .pObj = pOper->pObj - }; - (*pTable->encodeFp)(&oper); - rowHead->rowSize = oper.rowSize; - } - - pTable->version++; - sdbVersion++; - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); - pTable->version--; - sdbVersion--; - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - pMeta->version = pTable->version; - pMeta->offset = pTable->fileSize; - pMeta->rowSize = rowHead->rowSize; - - pthread_mutex_unlock(&pTable->mutex); - - (*pTable->updateFp)(pOper); // update in upper layer - - tfree(rowHead); - - return 0; -} - -void sdbCloseTable(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - void * pNode = NULL; - - if (pTable == NULL) return; - - while (1) { - SRowMeta * pMeta; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - SSdbOperDesc oper = { - .pObj = pMeta->row, - .table = pTable, - .version = pMeta->version, - }; - - (*pTable->destroyFp)(&oper); - } - - if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - - if (pTable->fd) tclose(pTable->fd); - - pthread_mutex_destroy(&pTable->mutex); - - sdbNumOfTables--; - sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); - - tfree(pTable); + return sdbUpdateLocal(pTable, pOper); } void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { @@ -801,3 +511,63 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { return pNode; } + +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; + + strcpy(pTable->tableName, pDesc->tableName); + pTable->keyType = pDesc->keyType; + pTable->tableId = pDesc->tableId; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->refCountPos = pDesc->refCountPos; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + pTable->updateAllFp = pDesc->updateAllFp; + + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } + + pthread_mutex_init(&pTable->mutex, NULL); + + tsSdbNumOfTables++; + tsSdbTableList[pTable->tableId] = pTable; + return pTable; +} + +void sdbCloseTable(void *handle) { + SSdbTable *pTable = (SSdbTable *)handle; + if (pTable == NULL) return; + + tsSdbNumOfTables--; + tsSdbTableList[pTable->tableId] = NULL; + + void *pNode = NULL; + while (1) { + SRowMeta *pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + }; + + (*pTable->destroyFp)(&oper); + } + + if (sdbCleanUpIndexFp[pTable->keyType]) { + (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); + } + + pthread_mutex_destroy(&pTable->mutex); + + sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables); + tfree(pTable); +} diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 8dfe835f32..19b747b745 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -174,6 +174,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { } static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; SChildTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); @@ -182,7 +183,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { pOper->rowSize = tsChildTableUpdateSize; } else { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) { + if (maxRowSize < tsChildTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); @@ -224,35 +225,11 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtInitChildTables() { +static int32_t mgmtChildTableActionUpdateAll() { void *pNode = NULL; void *pLastNode = NULL; SChildTableObj *pTable = NULL; - SChildTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - - SSdbTableDesc tableDesc = { - .tableName = "ctables", - .hashSessions = tsMaxTables, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, - .insertFp = mgmtChildTableActionInsert, - .deleteFp = mgmtChildTableActionDelete, - .updateFp = mgmtChildTableActionUpdate, - .encodeFp = mgmtChildTableActionEncode, - .decodeFp = mgmtChildTableActionDecode, - .destroyFp = mgmtChildTableActionDestroy, - }; - - tsChildTableSdb = sdbOpenTable(&tableDesc); - if (tsChildTableSdb == NULL) { - mError("failed to init child table data"); - return -1; - } - - pNode = NULL; while (1) { pLastNode = pNode; mgmtDecTableRef(pTable); @@ -328,6 +305,35 @@ static int32_t mgmtInitChildTables() { } } + return 0; +} + +static int32_t mgmtInitChildTables() { + SChildTableObj tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_CTABLE, + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + .updateAllFp = mgmtChildTableActionUpdateAll + }; + + tsChildTableSdb = sdbOpenTable(&tableDesc); + if (tsChildTableSdb == NULL) { + mError("failed to init child table data"); + return -1; + } + mTrace("child table is initialized"); return 0; } @@ -374,12 +380,14 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { } static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; + SSuperTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { + if (maxRowSize < tsSuperTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } @@ -411,11 +419,16 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtSuperTableActionUpdateAll() { + return 0; +} + static int32_t mgmtInitSuperTables() { SSuperTableObj tObj; tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, @@ -427,6 +440,7 @@ static int32_t mgmtInitSuperTables() { .encodeFp = mgmtSuperTableActionEncode, .decodeFp = mgmtSuperTableActionDecode, .destroyFp = mgmtSuperTableActionDestroy, + .updateAllFp = mgmtSuperTableActionUpdateAll }; tsSuperTableSdb = sdbOpenTable(&tableDesc); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 8e969b3c9c..b216e8f36d 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -70,14 +70,9 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { SUserObj *pUser = pOper->pObj; - - if (pOper->maxRowSize < tsUserUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pUser, tsUserUpdateSize); - pOper->rowSize = tsUserUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { @@ -89,11 +84,22 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtUserActionUpdateAll() { + SAcctObj *pAcct = acctGetAcct("root"); + mgmtCreateUser(pAcct, "root", "taosdata"); + mgmtCreateUser(pAcct, "monitor", tsInternalPass); + mgmtCreateUser(pAcct, "_root", tsInternalPass); + acctReleaseAcct(pAcct); + + return 0; +} + int32_t mgmtInitUsers() { SUserObj tObj; tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_USER, .tableName = "users", .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, @@ -105,6 +111,7 @@ int32_t mgmtInitUsers() { .encodeFp = mgmtUserActionEncode, .decodeFp = mgmtUserActionDecode, .destroyFp = mgmtUserActionDestroy, + .updateAllFp = mgmtUserActionUpdateAll }; tsUserSdb = sdbOpenTable(&tableDesc); @@ -113,12 +120,6 @@ int32_t mgmtInitUsers() { return -1; } - SAcctObj *pAcct = acctGetAcct("root"); - mgmtCreateUser(pAcct, "root", "taosdata"); - mgmtCreateUser(pAcct, "monitor", tsInternalPass); - mgmtCreateUser(pAcct, "_root", tsInternalPass); - acctReleaseAcct(pAcct); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 98ae422904..09fdffb68e 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -138,13 +138,9 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { SVgObj *pVgroup = pOper->pObj; - if (pOper->maxRowSize < tsVgUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); - pOper->rowSize = tsVgUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { @@ -156,11 +152,16 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtVgroupActionUpdateAll() { + return 0; +} + int32_t mgmtInitVgroups() { SVgObj tObj; tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_VGROUP, .tableName = "vgroups", .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, @@ -172,6 +173,7 @@ int32_t mgmtInitVgroups() { .encodeFp = mgmtVgroupActionEncode, .decodeFp = mgmtVgroupActionDecode, .destroyFp = mgmtVgroupActionDestroy, + .updateAllFp = mgmtVgroupActionUpdateAll, }; tsVgroupSdb = sdbOpenTable(&tableDesc); @@ -187,6 +189,7 @@ int32_t mgmtInitVgroups() { mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); + return 0; } diff --git a/src/util/inc/cJSON.h b/src/util/inc/cJSON.h index 31c6d19e78..cdd5faa523 100644 --- a/src/util/inc/cJSON.h +++ b/src/util/inc/cJSON.h @@ -71,6 +71,9 @@ typedef struct cJSON /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ char *string; + + //Keep the original string of number + char numberstring[13]; } cJSON; typedef struct cJSON_Hooks diff --git a/src/util/src/cJSON.c b/src/util/src/cJSON.c index fa836f9871..ecf0e05b42 100644 --- a/src/util/src/cJSON.c +++ b/src/util/src/cJSON.c @@ -289,6 +289,9 @@ loop_end: item->type = cJSON_Number; input_buffer->offset += (size_t)(after_end - number_c_string); + + strncpy(item->numberstring, (const char *)number_c_string, 12); + return true; } diff --git a/tests/tsim/inc/cJSON.h b/tests/tsim/inc/cJSON.h deleted file mode 100644 index cdd5faa523..0000000000 --- a/tests/tsim/inc/cJSON.h +++ /dev/null @@ -1,267 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef cJSON__h -#define cJSON__h - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* project version */ -#define CJSON_VERSION_MAJOR 1 -#define CJSON_VERSION_MINOR 5 -#define CJSON_VERSION_PATCH 9 - -#include -#include - -/* cJSON Types: */ -#define cJSON_Invalid (0) -#define cJSON_False (1 << 0) -#define cJSON_True (1 << 1) -#define cJSON_NULL (1 << 2) -#define cJSON_Number (1 << 3) -#define cJSON_String (1 << 4) -#define cJSON_Array (1 << 5) -#define cJSON_Object (1 << 6) -#define cJSON_Raw (1 << 7) /* raw json */ - -#define cJSON_IsReference 256 -#define cJSON_StringIsConst 512 - -/* The cJSON structure: */ -typedef struct cJSON -{ - /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ - struct cJSON *next; - struct cJSON *prev; - /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ - struct cJSON *child; - - /* The type of the item, as above. */ - int type; - - /* The item's string, if type==cJSON_String and type == cJSON_Raw */ - char *valuestring; - /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ - int64_t valueint; - /* The item's number, if type==cJSON_Number */ - double valuedouble; - - /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ - char *string; - - //Keep the original string of number - char numberstring[13]; -} cJSON; - -typedef struct cJSON_Hooks -{ - void *(*malloc_fn)(size_t sz); - void (*free_fn)(void *ptr); -} cJSON_Hooks; - -typedef int cJSON_bool; - -#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) -#define __WINDOWS__ -#endif -#ifdef __WINDOWS__ - -/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options: - -CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols -CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) -CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol - -For *nix builds that support visibility attribute, you can define similar behavior by - -setting default visibility to hidden by adding --fvisibility=hidden (for gcc) -or --xldscope=hidden (for sun cc) -to CFLAGS - -then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does - -*/ - -/* export symbols by default, this is necessary for copy pasting the C and header file */ -#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_EXPORT_SYMBOLS -#endif - -#if defined(CJSON_HIDE_SYMBOLS) -#define CJSON_PUBLIC(type) type __stdcall -#elif defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall -#elif defined(CJSON_IMPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall -#endif -#else /* !WIN32 */ -#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) -#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type -#else -#define CJSON_PUBLIC(type) type -#endif -#endif - -/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. - * This is to prevent stack overflows. */ -#ifndef CJSON_NESTING_LIMIT -#define CJSON_NESTING_LIMIT 1000 -#endif - -/* returns the version of cJSON as a string */ -CJSON_PUBLIC(const char*) cJSON_Version(void); - -/* Supply malloc, realloc and free functions to cJSON */ -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); - -/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ -/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); -/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ -/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); - -/* Render a cJSON entity to text for transfer/storage. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); -/* Render a cJSON entity to text for transfer/storage without any formatting. */ -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); -/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); -/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ -/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); -/* Delete a cJSON entity and all subentities. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *c); - -/* Returns the number of items in an array (or object). */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); -/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); -/* Get item "string" from object. Case insensitive. */ -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); -/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); - -/* These functions check the type of an item */ -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); - -/* These calls create a cJSON item of the appropriate type. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); -/* raw json */ -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); - -/* These utilities create an Array of count items. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count); - -/* Append item to the specified array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); -/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. - * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before - * writing to `item->string` */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); -/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); - -/* Remove/Detatch items from Arrays/Objects. */ -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); - -/* Update array items. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); - -/* Duplicate a cJSON item */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); -/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will -need to be released. With recurse!=0, it will duplicate any children connected to the item. -The item->next and ->prev pointers are always zero on return from Duplicate. */ -/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. - * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); - - -CJSON_PUBLIC(void) cJSON_Minify(char *json); - -/* Macros for creating things quickly. */ -#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) -#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) -#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) -#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) -#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) -#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) -#define cJSON_AddRawToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateRaw(s)) - -/* When assigning an integer value, it needs to be propagated to valuedouble too. */ -#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) -/* helper for the cJSON_SetNumberValue macro */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); -#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) - -/* Macro for iterating over an array or object */ -#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) - -/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ -CJSON_PUBLIC(void *) cJSON_malloc(size_t size); -CJSON_PUBLIC(void) cJSON_free(void *object); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/tsim/src/cJSON.c b/tests/tsim/src/cJSON.c deleted file mode 100644 index ecf0e05b42..0000000000 --- a/tests/tsim/src/cJSON.c +++ /dev/null @@ -1,2702 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -/* cJSON */ -/* JSON parser in C. */ - -#ifdef __GNUC__ -#pragma GCC visibility push(default) -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __GNUC__ -#pragma GCC visibility pop -#endif - -#include "cJSON.h" - -/* define our own boolean type */ -#define true ((cJSON_bool)1) -#define false ((cJSON_bool)0) - -typedef struct { - const unsigned char *json; - size_t position; -} error; -static error global_error = { NULL, 0 }; - -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) -{ - return (const char*) (global_error.json + global_error.position); -} - -/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 5) || (CJSON_VERSION_PATCH != 9) - #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. -#endif - -CJSON_PUBLIC(const char*) cJSON_Version(void) -{ - static char version[15]; - sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); - - return version; -} - -/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ -static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) -{ - if ((string1 == NULL) || (string2 == NULL)) - { - return 1; - } - - if (string1 == string2) - { - return 0; - } - - for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) - { - if (*string1 == '\0') - { - return 0; - } - } - - return tolower(*string1) - tolower(*string2); -} - -typedef struct internal_hooks -{ - void *(*allocate)(size_t size); - void (*deallocate)(void *pointer); - void *(*reallocate)(void *pointer, size_t size); -} internal_hooks; - -static internal_hooks global_hooks = { malloc, free, realloc }; - -static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) -{ - size_t length = 0; - unsigned char *copy = NULL; - - if (string == NULL) - { - return NULL; - } - - length = strlen((const char*)string) + sizeof(""); - if (!(copy = (unsigned char*)hooks->allocate(length))) - { - return NULL; - } - memcpy(copy, string, length); - - return copy; -} - -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) -{ - if (hooks == NULL) - { - /* Reset hooks */ - global_hooks.allocate = malloc; - global_hooks.deallocate = free; - global_hooks.reallocate = realloc; - return; - } - - global_hooks.allocate = malloc; - if (hooks->malloc_fn != NULL) - { - global_hooks.allocate = hooks->malloc_fn; - } - - global_hooks.deallocate = free; - if (hooks->free_fn != NULL) - { - global_hooks.deallocate = hooks->free_fn; - } - - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) - { - global_hooks.reallocate = realloc; - } -} - -/* Internal constructor. */ -static cJSON *cJSON_New_Item(const internal_hooks * const hooks) -{ - cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); - if (node) - { - memset(node, '\0', sizeof(cJSON)); - } - - return node; -} - -/* Delete a cJSON structure. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) -{ - cJSON *next = NULL; - while (item != NULL) - { - next = item->next; - if (!(item->type & cJSON_IsReference) && (item->child != NULL)) - { - cJSON_Delete(item->child); - } - if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) - { - global_hooks.deallocate(item->valuestring); - } - if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) - { - global_hooks.deallocate(item->string); - } - global_hooks.deallocate(item); - item = next; - } -} - -/* get the decimal point character of the current locale */ -static unsigned char get_decimal_point(void) -{ - struct lconv *lconv = localeconv(); - return (unsigned char) lconv->decimal_point[0]; -} - -typedef struct -{ - const unsigned char *content; - size_t length; - size_t offset; - size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ - internal_hooks hooks; -} parse_buffer; - -/* check if the given size is left to read in a given parse buffer (starting with 1) */ -#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) -#define cannot_read(buffer, size) (!can_read(buffer, size)) -/* check if the buffer can be accessed at the given index (starting with 0) */ -#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) -#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) -/* get a pointer to the buffer at the position */ -#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) - -/* Parse the input text to generate a number, and populate the result into item. */ -static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) -{ - double number = 0; - unsigned char *after_end = NULL; - unsigned char number_c_string[64]; - unsigned char decimal_point = get_decimal_point(); - size_t i = 0; - - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; - } - - /* copy the number into a temporary buffer and replace '.' with the decimal point - * of the current locale (for strtod) - * This also takes care of '\0' not necessarily being available for marking the end of the input */ - for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++) - { - switch (buffer_at_offset(input_buffer)[i]) - { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '+': - case '-': - case 'e': - case 'E': - number_c_string[i] = buffer_at_offset(input_buffer)[i]; - break; - - case '.': - number_c_string[i] = decimal_point; - break; - - default: - goto loop_end; - } - } -loop_end: - number_c_string[i] = '\0'; - - number = strtod((const char*)number_c_string, (char**)&after_end); - if (number_c_string == after_end) - { - return false; /* parse_error */ - } - - item->valuedouble = number; - - /* use saturation in case of overflow */ - if (number >= LLONG_MAX) - { - item->valueint = LLONG_MAX; - } - else if (number <= LLONG_MIN) - { - item->valueint = LLONG_MIN; - } - else - { - item->valueint = (int64_t)number; - } - - item->type = cJSON_Number; - - input_buffer->offset += (size_t)(after_end - number_c_string); - - strncpy(item->numberstring, (const char *)number_c_string, 12); - - return true; -} - -/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) -{ - if (number >= LLONG_MAX) - { - object->valueint = LLONG_MAX; - } - else if (number <= LLONG_MIN) - { - object->valueint = LLONG_MIN; - } - else - { - object->valueint = (int64_t)number; - } - - return object->valuedouble = number; -} - -typedef struct -{ - unsigned char *buffer; - size_t length; - size_t offset; - size_t depth; /* current nesting depth (for formatted printing) */ - cJSON_bool noalloc; - cJSON_bool format; /* is this print a formatted print */ - internal_hooks hooks; -} printbuffer; - -/* realloc printbuffer if necessary to have at least "needed" bytes more */ -static unsigned char* ensure(printbuffer * const p, size_t needed) -{ - unsigned char *newbuffer = NULL; - size_t newsize = 0; - - if ((p == NULL) || (p->buffer == NULL)) - { - return NULL; - } - - if ((p->length > 0) && (p->offset >= p->length)) - { - /* make sure that offset is valid */ - return NULL; - } - - if (needed > LLONG_MAX) - { - /* sizes bigger than LLONG_MAX are currently not supported */ - return NULL; - } - - needed += p->offset + 1; - if (needed <= p->length) - { - return p->buffer + p->offset; - } - - if (p->noalloc) { - return NULL; - } - - /* calculate new buffer size */ - if (needed > (LLONG_MAX / 2)) - { - /* overflow of int, use LLONG_MAX if possible */ - if (needed <= LLONG_MAX) - { - newsize = LLONG_MAX; - } - else - { - return NULL; - } - } - else - { - newsize = needed * 2; - } - - if (p->hooks.reallocate != NULL) - { - /* reallocate with realloc if available */ - newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); - if (newbuffer == NULL) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - } - else - { - /* otherwise reallocate manually */ - newbuffer = (unsigned char*)p->hooks.allocate(newsize); - if (!newbuffer) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - if (newbuffer) - { - memcpy(newbuffer, p->buffer, p->offset + 1); - } - p->hooks.deallocate(p->buffer); - } - p->length = newsize; - p->buffer = newbuffer; - - return newbuffer + p->offset; -} - -/* calculate the new length of the string in a printbuffer and update the offset */ -static void update_offset(printbuffer * const buffer) -{ - const unsigned char *buffer_pointer = NULL; - if ((buffer == NULL) || (buffer->buffer == NULL)) - { - return; - } - buffer_pointer = buffer->buffer + buffer->offset; - - buffer->offset += strlen((const char*)buffer_pointer); -} - -/* Render the number nicely from the given item into a string. */ -static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - double d = item->valuedouble; - int length = 0; - size_t i = 0; - unsigned char number_buffer[26]; /* temporary buffer to print the number into */ - unsigned char decimal_point = get_decimal_point(); - double test; - - if (output_buffer == NULL) - { - return false; - } - - /* This checks for NaN and Infinity */ - if ((d * 0) != 0) - { - length = sprintf((char*)number_buffer, "null"); - } - else - { - /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ - length = sprintf((char*)number_buffer, "%1.15g", d); - - /* Check whether the original double can be recovered */ - if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || ((double)test != d)) - { - /* If not, print with 17 decimal places of precision */ - length = sprintf((char*)number_buffer, "%1.17g", d); - } - } - - /* sprintf failed or buffer overrun occured */ - if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) - { - return false; - } - - /* reserve appropriate space in the output */ - output_pointer = ensure(output_buffer, (size_t)length); - if (output_pointer == NULL) - { - return false; - } - - /* copy the printed number to the output and replace locale - * dependent decimal point with '.' */ - for (i = 0; i < ((size_t)length); i++) - { - if (number_buffer[i] == decimal_point) - { - output_pointer[i] = '.'; - continue; - } - - output_pointer[i] = number_buffer[i]; - } - output_pointer[i] = '\0'; - - output_buffer->offset += (size_t)length; - - return true; -} - -/* parse 4 digit hexadecimal number */ -static unsigned parse_hex4(const unsigned char * const input) -{ - unsigned int h = 0; - size_t i = 0; - - for (i = 0; i < 4; i++) - { - /* parse digit */ - if ((input[i] >= '0') && (input[i] <= '9')) - { - h += (unsigned int) input[i] - '0'; - } - else if ((input[i] >= 'A') && (input[i] <= 'F')) - { - h += (unsigned int) 10 + input[i] - 'A'; - } - else if ((input[i] >= 'a') && (input[i] <= 'f')) - { - h += (unsigned int) 10 + input[i] - 'a'; - } - else /* invalid */ - { - return 0; - } - - if (i < 3) - { - /* shift left to make place for the next nibble */ - h = h << 4; - } - } - - return h; -} - -/* converts a UTF-16 literal to UTF-8 - * A literal can be one or two sequences of the form \uXXXX */ -static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) -{ - uint64_t codepoint = 0; - unsigned int first_code = 0; - const unsigned char *first_sequence = input_pointer; - unsigned char utf8_length = 0; - unsigned char utf8_position = 0; - unsigned char sequence_length = 0; - unsigned char first_byte_mark = 0; - - if ((input_end - first_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - /* get the first utf16 sequence */ - first_code = parse_hex4(first_sequence + 2); - - /* check that the code is valid */ - if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) - { - goto fail; - } - - /* UTF16 surrogate pair */ - if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) - { - const unsigned char *second_sequence = first_sequence + 6; - unsigned int second_code = 0; - sequence_length = 12; /* \uXXXX\uXXXX */ - - if ((input_end - second_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) - { - /* missing second half of the surrogate pair */ - goto fail; - } - - /* get the second utf16 sequence */ - second_code = parse_hex4(second_sequence + 2); - /* check that the code is valid */ - if ((second_code < 0xDC00) || (second_code > 0xDFFF)) - { - /* invalid second half of the surrogate pair */ - goto fail; - } - - - /* calculate the unicode codepoint from the surrogate pair */ - codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); - } - else - { - sequence_length = 6; /* \uXXXX */ - codepoint = first_code; - } - - /* encode as UTF-8 - * takes at maximum 4 bytes to encode: - * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - if (codepoint < 0x80) - { - /* normal ascii, encoding 0xxxxxxx */ - utf8_length = 1; - } - else if (codepoint < 0x800) - { - /* two bytes, encoding 110xxxxx 10xxxxxx */ - utf8_length = 2; - first_byte_mark = 0xC0; /* 11000000 */ - } - else if (codepoint < 0x10000) - { - /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ - utf8_length = 3; - first_byte_mark = 0xE0; /* 11100000 */ - } - else if (codepoint <= 0x10FFFF) - { - /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ - utf8_length = 4; - first_byte_mark = 0xF0; /* 11110000 */ - } - else - { - /* invalid unicode codepoint */ - goto fail; - } - - /* encode as utf8 */ - for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) - { - /* 10xxxxxx */ - (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); - codepoint >>= 6; - } - /* encode first byte */ - if (utf8_length > 1) - { - (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); - } - else - { - (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); - } - - *output_pointer += utf8_length; - - return sequence_length; - -fail: - return 0; -} - -/* Parse the input text into an unescaped cinput, and populate item. */ -static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) -{ - const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; - const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; - unsigned char *output_pointer = NULL; - unsigned char *output = NULL; - - /* not a string */ - if (buffer_at_offset(input_buffer)[0] != '\"') - { - goto fail; - } - - { - /* calculate approximate size of the output (overestimate) */ - size_t allocation_length = 0; - size_t skipped_bytes = 0; - while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) - { - /* is escape sequence */ - if (input_end[0] == '\\') - { - if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) - { - /* prevent buffer overflow when last input character is a backslash */ - goto fail; - } - skipped_bytes++; - input_end++; - } - input_end++; - } - if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) - { - goto fail; /* string ended unexpectedly */ - } - - /* This is at most how much we need for the output */ - allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; - output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); - if (output == NULL) - { - goto fail; /* allocation failure */ - } - } - - output_pointer = output; - /* loop through the string literal */ - while (input_pointer < input_end) - { - if (*input_pointer != '\\') - { - *output_pointer++ = *input_pointer++; - } - /* escape sequence */ - else - { - unsigned char sequence_length = 2; - if ((input_end - input_pointer) < 1) - { - goto fail; - } - - switch (input_pointer[1]) - { - case 'b': - *output_pointer++ = '\b'; - break; - case 'f': - *output_pointer++ = '\f'; - break; - case 'n': - *output_pointer++ = '\n'; - break; - case 'r': - *output_pointer++ = '\r'; - break; - case 't': - *output_pointer++ = '\t'; - break; - case '\"': - case '\\': - case '/': - *output_pointer++ = input_pointer[1]; - break; - - /* UTF-16 literal */ - case 'u': - sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); - if (sequence_length == 0) - { - /* failed to convert UTF16-literal to UTF-8 */ - goto fail; - } - break; - - default: - goto fail; - } - input_pointer += sequence_length; - } - } - - /* zero terminate the output */ - *output_pointer = '\0'; - - item->type = cJSON_String; - item->valuestring = (char*)output; - - input_buffer->offset = (size_t) (input_end - input_buffer->content); - input_buffer->offset++; - - return true; - -fail: - if (output != NULL) - { - input_buffer->hooks.deallocate(output); - } - - if (input_pointer != NULL) - { - input_buffer->offset = (size_t)(input_pointer - input_buffer->content); - } - - return false; -} - -/* Render the cstring provided to an escaped version that can be printed. */ -static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) -{ - const unsigned char *input_pointer = NULL; - unsigned char *output = NULL; - unsigned char *output_pointer = NULL; - size_t output_length = 0; - /* numbers of additional characters needed for escaping */ - size_t escape_characters = 0; - - if (output_buffer == NULL) - { - return false; - } - - /* empty string */ - if (input == NULL) - { - output = ensure(output_buffer, sizeof("\"\"")); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "\"\""); - - return true; - } - - /* set "flag" to 1 if something needs to be escaped */ - for (input_pointer = input; *input_pointer; input_pointer++) - { - switch (*input_pointer) - { - case '\"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - /* one character escape sequence */ - escape_characters++; - break; - default: - if (*input_pointer < 32) - { - /* UTF-16 escape sequence uXXXX */ - escape_characters += 5; - } - break; - } - } - output_length = (size_t)(input_pointer - input) + escape_characters; - - output = ensure(output_buffer, output_length + sizeof("\"\"")); - if (output == NULL) - { - return false; - } - - /* no characters have to be escaped */ - if (escape_characters == 0) - { - output[0] = '\"'; - memcpy(output + 1, input, output_length); - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; - } - - output[0] = '\"'; - output_pointer = output + 1; - /* copy the string */ - for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) - { - if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) - { - /* normal character, copy */ - *output_pointer = *input_pointer; - } - else - { - /* character needs to be escaped */ - *output_pointer++ = '\\'; - switch (*input_pointer) - { - case '\\': - *output_pointer = '\\'; - break; - case '\"': - *output_pointer = '\"'; - break; - case '\b': - *output_pointer = 'b'; - break; - case '\f': - *output_pointer = 'f'; - break; - case '\n': - *output_pointer = 'n'; - break; - case '\r': - *output_pointer = 'r'; - break; - case '\t': - *output_pointer = 't'; - break; - default: - /* escape and print as unicode codepoint */ - sprintf((char*)output_pointer, "u%04x", *input_pointer); - output_pointer += 4; - break; - } - } - } - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; -} - -/* Invoke print_string_ptr (which is useful) on an item. */ -static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) -{ - return print_string_ptr((unsigned char*)item->valuestring, p); -} - -/* Predeclare these prototypes. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); - -/* Utility to jump whitespace and cr/lf */ -static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) -{ - if ((buffer == NULL) || (buffer->content == NULL)) - { - return NULL; - } - - while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) - { - buffer->offset++; - } - - if (buffer->offset == buffer->length) - { - buffer->offset--; - } - - return buffer; -} - -/* Parse an object - create a new root, and populate. */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) -{ - parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; - cJSON *item = NULL; - - /* reset error position */ - global_error.json = NULL; - global_error.position = 0; - - if (value == NULL) - { - goto fail; - } - - buffer.content = (const unsigned char*)value; - buffer.length = strlen((const char*)value) + sizeof(""); - buffer.offset = 0; - buffer.hooks = global_hooks; - - item = cJSON_New_Item(&global_hooks); - if (item == NULL) /* memory fail */ - { - goto fail; - } - - if (!parse_value(item, buffer_skip_whitespace(&buffer))) - { - /* parse failure. ep is set. */ - goto fail; - } - - /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ - if (require_null_terminated) - { - buffer_skip_whitespace(&buffer); - if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') - { - goto fail; - } - } - if (return_parse_end) - { - *return_parse_end = (const char*)buffer_at_offset(&buffer); - } - - return item; - -fail: - if (item != NULL) - { - cJSON_Delete(item); - } - - if (value != NULL) - { - error local_error; - local_error.json = (const unsigned char*)value; - local_error.position = 0; - - if (buffer.offset < buffer.length) - { - local_error.position = buffer.offset; - } - else if (buffer.length > 0) - { - local_error.position = buffer.length - 1; - } - - if (return_parse_end != NULL) - { - *return_parse_end = (const char*)local_error.json + local_error.position; - } - - global_error = local_error; - } - - return NULL; -} - -/* Default options for cJSON_Parse */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) -{ - return cJSON_ParseWithOpts(value, 0, 0); -} - -#define cjson_min(a, b) ((a < b) ? a : b) - -static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) -{ - printbuffer buffer[1]; - unsigned char *printed = NULL; - - memset(buffer, 0, sizeof(buffer)); - - /* create buffer */ - buffer->buffer = (unsigned char*) hooks->allocate(256); - buffer->format = format; - buffer->hooks = *hooks; - if (buffer->buffer == NULL) - { - goto fail; - } - - /* print the value */ - if (!print_value(item, buffer)) - { - goto fail; - } - update_offset(buffer); - - /* check if reallocate is available */ - if (hooks->reallocate != NULL) - { - printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->length); - buffer->buffer = NULL; - if (printed == NULL) { - goto fail; - } - } - else /* otherwise copy the JSON over to a new buffer */ - { - printed = (unsigned char*) hooks->allocate(buffer->offset + 1); - if (printed == NULL) - { - goto fail; - } - memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); - printed[buffer->offset] = '\0'; /* just to be sure */ - - /* free the buffer */ - hooks->deallocate(buffer->buffer); - } - - return printed; - -fail: - if (buffer->buffer != NULL) - { - hooks->deallocate(buffer->buffer); - } - - if (printed != NULL) - { - hooks->deallocate(printed); - } - - return NULL; -} - -/* Render a cJSON item/entity/structure to text. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) -{ - return (char*)print(item, true, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) -{ - return (char*)print(item, false, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if (prebuffer < 0) - { - return NULL; - } - - p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); - if (!p.buffer) - { - return NULL; - } - - p.length = (size_t)prebuffer; - p.offset = 0; - p.noalloc = false; - p.format = fmt; - p.hooks = global_hooks; - - if (!print_value(item, &p)) - { - global_hooks.deallocate(p.buffer); - return NULL; - } - - return (char*)p.buffer; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if ((len < 0) || (buf == NULL)) - { - return false; - } - - p.buffer = (unsigned char*)buf; - p.length = (size_t)len; - p.offset = 0; - p.noalloc = true; - p.format = fmt; - p.hooks = global_hooks; - - return print_value(item, &p); -} - -/* Parser core - when encountering text, process appropriately. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) -{ - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; /* no input */ - } - - /* parse the different types of values */ - /* null */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) - { - item->type = cJSON_NULL; - input_buffer->offset += 4; - return true; - } - /* false */ - if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) - { - item->type = cJSON_False; - input_buffer->offset += 5; - return true; - } - /* true */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) - { - item->type = cJSON_True; - item->valueint = 1; - input_buffer->offset += 4; - return true; - } - /* string */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) - { - return parse_string(item, input_buffer); - } - /* number */ - if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) - { - return parse_number(item, input_buffer); - } - /* array */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) - { - return parse_array(item, input_buffer); - } - /* object */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) - { - return parse_object(item, input_buffer); - } - - - return false; -} - -/* Render a value to text. */ -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output = NULL; - - if ((item == NULL) || (output_buffer == NULL)) - { - return false; - } - - switch ((item->type) & 0xFF) - { - case cJSON_NULL: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "null"); - return true; - - case cJSON_False: - output = ensure(output_buffer, 6); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "false"); - return true; - - case cJSON_True: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "true"); - return true; - - case cJSON_Number: - return print_number(item, output_buffer); - - case cJSON_Raw: - { - size_t raw_length = 0; - if (item->valuestring == NULL) - { - if (!output_buffer->noalloc) - { - output_buffer->hooks.deallocate(output_buffer->buffer); - } - return false; - } - - raw_length = strlen(item->valuestring) + sizeof(""); - output = ensure(output_buffer, raw_length); - if (output == NULL) - { - return false; - } - memcpy(output, item->valuestring, raw_length); - return true; - } - - case cJSON_String: - return print_string(item, output_buffer); - - case cJSON_Array: - return print_array(item, output_buffer); - - case cJSON_Object: - return print_object(item, output_buffer); - - default: - return false; - } -} - -/* Build an array from input text. */ -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* head of the linked list */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (buffer_at_offset(input_buffer)[0] != '[') - { - /* not an array */ - goto fail; - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) - { - /* empty array */ - goto success; - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse next value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') - { - goto fail; /* expected end of array */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Array; - item->child = head; - - input_buffer->offset++; - - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an array to text */ -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_element = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output array. */ - /* opening square bracket */ - output_pointer = ensure(output_buffer, 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer = '['; - output_buffer->offset++; - output_buffer->depth++; - - while (current_element != NULL) - { - if (!print_value(current_element, output_buffer)) - { - return false; - } - update_offset(output_buffer); - if (current_element->next) - { - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ','; - if(output_buffer->format) - { - *output_pointer++ = ' '; - } - *output_pointer = '\0'; - output_buffer->offset += length; - } - current_element = current_element->next; - } - - output_pointer = ensure(output_buffer, 2); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ']'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Build an object from the text. */ -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* linked list head */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) - { - goto fail; /* not an object */ - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) - { - goto success; /* empty object */ - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse the name of the child */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_string(current_item, input_buffer)) - { - goto fail; /* faile to parse name */ - } - buffer_skip_whitespace(input_buffer); - - /* swap valuestring and string, because we parsed the name */ - current_item->string = current_item->valuestring; - current_item->valuestring = NULL; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) - { - goto fail; /* invalid object */ - } - - /* parse the value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) - { - goto fail; /* expected end of object */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Object; - item->child = head; - - input_buffer->offset++; - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an object to text. */ -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_item = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output: */ - length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer++ = '{'; - output_buffer->depth++; - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - output_buffer->offset += length; - - while (current_item) - { - if (output_buffer->format) - { - size_t i; - output_pointer = ensure(output_buffer, output_buffer->depth); - if (output_pointer == NULL) - { - return false; - } - for (i = 0; i < output_buffer->depth; i++) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += output_buffer->depth; - } - - /* print key */ - if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ':'; - if (output_buffer->format) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += length; - - /* print value */ - if (!print_value(current_item, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - /* print comma if not last */ - length = (size_t) ((output_buffer->format ? 1 : 0) + (current_item->next ? 1 : 0)); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - if (current_item->next) - { - *output_pointer++ = ','; - } - - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - *output_pointer = '\0'; - output_buffer->offset += length; - - current_item = current_item->next; - } - - output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); - if (output_pointer == NULL) - { - return false; - } - if (output_buffer->format) - { - size_t i; - for (i = 0; i < (output_buffer->depth - 1); i++) - { - *output_pointer++ = '\t'; - } - } - *output_pointer++ = '}'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Get Array size/item / object item. */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) -{ - cJSON *child = NULL; - size_t size = 0; - - if (array == NULL) - { - return 0; - } - - child = array->child; - - while(child != NULL) - { - size++; - child = child->next; - } - - /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ - - return (int)size; -} - -static cJSON* get_array_item(const cJSON *array, size_t index) -{ - cJSON *current_child = NULL; - - if (array == NULL) - { - return NULL; - } - - current_child = array->child; - while ((current_child != NULL) && (index > 0)) - { - index--; - current_child = current_child->next; - } - - return current_child; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) -{ - if (index < 0) - { - return NULL; - } - - return get_array_item(array, (size_t)index); -} - -static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) -{ - cJSON *current_element = NULL; - - if ((object == NULL) || (name == NULL)) - { - return NULL; - } - - current_element = object->child; - if (case_sensitive) - { - while ((current_element != NULL) && (strcmp(name, current_element->string) != 0)) - { - current_element = current_element->next; - } - } - else - { - while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) - { - current_element = current_element->next; - } - } - - return current_element; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, false); -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, true); -} - -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) -{ - return cJSON_GetObjectItem(object, string) ? 1 : 0; -} - -/* Utility for array list handling. */ -static void suffix_object(cJSON *prev, cJSON *item) -{ - prev->next = item; - item->prev = prev; -} - -/* Utility for handling references. */ -static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) -{ - cJSON *reference = NULL; - if (item == NULL) - { - return NULL; - } - - reference = cJSON_New_Item(hooks); - if (reference == NULL) - { - return NULL; - } - - memcpy(reference, item, sizeof(cJSON)); - reference->string = NULL; - reference->type |= cJSON_IsReference; - reference->next = reference->prev = NULL; - return reference; -} - -/* Add item to array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item) -{ - cJSON *child = NULL; - - if ((item == NULL) || (array == NULL)) - { - return; - } - - child = array->child; - - if (child == NULL) - { - /* list is empty, start new one */ - array->child = item; - } - else - { - /* append to the end */ - while (child->next) - { - child = child->next; - } - suffix_object(child, item); - } -} - -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) -{ - if (item == NULL) - { - return; - } - - /* call cJSON_AddItemToObjectCS for code reuse */ - cJSON_AddItemToObjectCS(object, (char*)cJSON_strdup((const unsigned char*)string, &global_hooks), item); - /* remove cJSON_StringIsConst flag */ - item->type &= ~cJSON_StringIsConst; -} - -#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic push -#endif -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - -/* Add an item to an object with constant string as key */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) -{ - if ((item == NULL) || (string == NULL)) - { - return; - } - if (!(item->type & cJSON_StringIsConst) && item->string) - { - global_hooks.deallocate(item->string); - } - item->string = (char*)string; - item->type |= cJSON_StringIsConst; - cJSON_AddItemToArray(object, item); -} -#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic pop -#endif - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) -{ - if (array == NULL) - { - return; - } - - cJSON_AddItemToArray(array, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) -{ - if ((object == NULL) || (string == NULL)) - { - return; - } - - cJSON_AddItemToObject(object, string, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) -{ - if ((parent == NULL) || (item == NULL)) - { - return NULL; - } - - if (item->prev != NULL) - { - /* not the first element */ - item->prev->next = item->next; - } - if (item->next != NULL) - { - /* not the last element */ - item->next->prev = item->prev; - } - - if (item == parent->child) - { - /* first element */ - parent->child = item->next; - } - /* make sure the detached item doesn't point anywhere anymore */ - item->prev = NULL; - item->next = NULL; - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) -{ - if (which < 0) - { - return NULL; - } - - return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) -{ - cJSON_Delete(cJSON_DetachItemFromArray(array, which)); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItem(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObject(object, string)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); -} - -/* Replace array/object items with new ones. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) -{ - cJSON *after_inserted = NULL; - - if (which < 0) - { - return; - } - - after_inserted = get_array_item(array, (size_t)which); - if (after_inserted == NULL) - { - cJSON_AddItemToArray(array, newitem); - return; - } - - newitem->next = after_inserted; - newitem->prev = after_inserted->prev; - after_inserted->prev = newitem; - if (after_inserted == array->child) - { - array->child = newitem; - } - else - { - newitem->prev->next = newitem; - } -} - -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) -{ - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) - { - return false; - } - - if (replacement == item) - { - return true; - } - - replacement->next = item->next; - replacement->prev = item->prev; - - if (replacement->next != NULL) - { - replacement->next->prev = replacement; - } - if (replacement->prev != NULL) - { - replacement->prev->next = replacement; - } - if (parent->child == item) - { - parent->child = replacement; - } - - item->next = NULL; - item->prev = NULL; - cJSON_Delete(item); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) -{ - if (which < 0) - { - return; - } - - cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); -} - -static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) -{ - if ((replacement == NULL) || (string == NULL)) - { - return false; - } - - /* replace the name in the replacement */ - if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) - { - cJSON_free(replacement->string); - } - replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - replacement->type &= ~cJSON_StringIsConst; - - cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, false); -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, true); -} - -/* Create basic types: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_NULL; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_True; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool b) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = b ? cJSON_True : cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Number; - item->valuedouble = num; - - /* use saturation in case of overflow */ - if (num >= LLONG_MAX) - { - item->valueint = LLONG_MAX; - } - else if (num <= LLONG_MIN) - { - item->valueint = LLONG_MIN; - } - else - { - item->valueint = (int64_t)num; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_String; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Raw; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type=cJSON_Array; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) - { - item->type = cJSON_Object; - } - - return item; -} - -/* Create Arrays: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if (!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber((double)numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0;a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (strings == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateString(strings[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p,n); - } - p = n; - } - - return a; -} - -/* Duplication */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) -{ - cJSON *newitem = NULL; - cJSON *child = NULL; - cJSON *next = NULL; - cJSON *newchild = NULL; - - /* Bail on bad ptr */ - if (!item) - { - goto fail; - } - /* Create new item */ - newitem = cJSON_New_Item(&global_hooks); - if (!newitem) - { - goto fail; - } - /* Copy over all vars */ - newitem->type = item->type & (~cJSON_IsReference); - newitem->valueint = item->valueint; - newitem->valuedouble = item->valuedouble; - if (item->valuestring) - { - newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); - if (!newitem->valuestring) - { - goto fail; - } - } - if (item->string) - { - newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); - if (!newitem->string) - { - goto fail; - } - } - /* If non-recursive, then we're done! */ - if (!recurse) - { - return newitem; - } - /* Walk the ->next chain for the child. */ - child = item->child; - while (child != NULL) - { - newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */ - if (!newchild) - { - goto fail; - } - if (next != NULL) - { - /* If newitem->child already set, then crosswire ->prev and ->next and move on */ - next->next = newchild; - newchild->prev = next; - next = newchild; - } - else - { - /* Set newitem->child and move to it */ - newitem->child = newchild; - next = newchild; - } - child = child->next; - } - - return newitem; - -fail: - if (newitem != NULL) - { - cJSON_Delete(newitem); - } - - return NULL; -} - -CJSON_PUBLIC(void) cJSON_Minify(char *json) -{ - unsigned char *into = (unsigned char*)json; - - if (json == NULL) - { - return; - } - - while (*json) - { - if (*json == ' ') - { - json++; - } - else if (*json == '\t') - { - /* Whitespace characters. */ - json++; - } - else if (*json == '\r') - { - json++; - } - else if (*json=='\n') - { - json++; - } - else if ((*json == '/') && (json[1] == '/')) - { - /* double-slash comments, to end of line. */ - while (*json && (*json != '\n')) - { - json++; - } - } - else if ((*json == '/') && (json[1] == '*')) - { - /* multiline comments. */ - while (*json && !((*json == '*') && (json[1] == '/'))) - { - json++; - } - json += 2; - } - else if (*json == '\"') - { - /* string literals, which are \" sensitive. */ - *into++ = (unsigned char)*json++; - while (*json && (*json != '\"')) - { - if (*json == '\\') - { - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - else - { - /* All other characters. */ - *into++ = (unsigned char)*json++; - } - } - - /* and null-terminate. */ - *into = '\0'; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Invalid; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_False; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xff) == cJSON_True; -} - - -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & (cJSON_True | cJSON_False)) != 0; -} -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_NULL; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Number; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_String; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Array; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Object; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Raw; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) -{ - if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) - { - return false; - } - - /* check if type is valid */ - switch (a->type & 0xFF) - { - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - case cJSON_Number: - case cJSON_String: - case cJSON_Raw: - case cJSON_Array: - case cJSON_Object: - break; - - default: - return false; - } - - /* identical objects are equal */ - if (a == b) - { - return true; - } - - switch (a->type & 0xFF) - { - /* in these cases and equal type is enough */ - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - return true; - - case cJSON_Number: - if (a->valuedouble == b->valuedouble) - { - return true; - } - return false; - - case cJSON_String: - case cJSON_Raw: - if ((a->valuestring == NULL) || (b->valuestring == NULL)) - { - return false; - } - if (strcmp(a->valuestring, b->valuestring) == 0) - { - return true; - } - - return false; - - case cJSON_Array: - { - cJSON *a_element = a->child; - cJSON *b_element = b->child; - - for (; (a_element != NULL) && (b_element != NULL);) - { - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - - a_element = a_element->next; - b_element = b_element->next; - } - - /* one of the arrays is longer than the other */ - if (a_element != b_element) { - return false; - } - - return true; - } - - case cJSON_Object: - { - cJSON *a_element = NULL; - cJSON *b_element = NULL; - cJSON_ArrayForEach(a_element, a) - { - /* TODO This has O(n^2) runtime, which is horrible! */ - b_element = get_object_item(b, a_element->string, case_sensitive); - if (b_element == NULL) - { - return false; - } - - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - } - - /* doing this twice, once on a and b to prevent true comparison if a subset of b - * TODO: Do this the proper way, this is just a fix for now */ - cJSON_ArrayForEach(b_element, b) - { - a_element = get_object_item(a, b_element->string, case_sensitive); - if (a_element == NULL) - { - return false; - } - - if (!cJSON_Compare(b_element, a_element, case_sensitive)) - { - return false; - } - } - - return true; - } - - default: - return false; - } -} - -CJSON_PUBLIC(void *) cJSON_malloc(size_t size) -{ - return global_hooks.allocate(size); -} - -CJSON_PUBLIC(void) cJSON_free(void *object) -{ - global_hooks.deallocate(object); -} From 5b3353138985a460872ada5b1b52d0142c4e9666 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 13 Apr 2020 23:14:25 +0800 Subject: [PATCH 22/36] [TD-52] refact sdb code --- src/mnode/inc/mgmtSdb.h | 8 +- src/mnode/src/mgmtDb.c | 10 +- src/mnode/src/mgmtSdb.c | 285 +++++++++++++++++++------------------ src/mnode/src/mgmtTable.c | 44 +++--- src/mnode/src/mgmtUser.c | 10 +- src/mnode/src/mgmtVgroup.c | 14 +- 6 files changed, 189 insertions(+), 182 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 95f3e6d39b..83afa2a081 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -33,13 +33,13 @@ typedef enum { } ESdbTable; typedef enum { - SDB_KEY_TYPE_STRING, - SDB_KEY_TYPE_AUTO + SDB_KEY_STRING, + SDB_KEY_AUTO } ESdbKeyType; typedef enum { - SDB_OPER_TYPE_GLOBAL, - SDB_OPER_TYPE_LOCAL + SDB_OPER_GLOBAL, + SDB_OPER_LOCAL } ESdbOperType; typedef struct { diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 2de9656e3e..946ec29d8c 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -116,7 +116,7 @@ int32_t mgmtInitDbs() { .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtDbActionInsert, .deleteFp = mgmtDbActionDelete, .updateFp = mgmtDbActionUpdate, @@ -311,7 +311,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->cfg = *pCreate; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = sizeof(SDbObj) @@ -664,7 +664,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { pDb->status = true; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = tsDbUpdateSize @@ -749,7 +749,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = tsDbUpdateSize @@ -807,7 +807,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) { mPrint("db:%s, drop db from sdb", pDb->name); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb }; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 6a229e716c..5202f8511c 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -31,8 +31,8 @@ typedef struct { int32_t code; int64_t version; - void * pSync; - void * pWal; + void * sync; + void * wal; sem_t sem; pthread_mutex_t mutex; } SSdbSync; @@ -99,13 +99,13 @@ static char *sdbGetActionStr(int32_t action) { static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { static char str[16]; switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: + case SDB_KEY_STRING: return (char *)row; - case SDB_KEY_TYPE_AUTO: + case SDB_KEY_AUTO: sprintf(str, "%d", *(int32_t *)row); return str; default: - return "unknown"; + return "invalid"; } } @@ -135,14 +135,14 @@ int32_t sdbInit() { pthread_mutex_init(&tsSdbSync->mutex, NULL); SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; - tsSdbSync->pWal = walOpen(tsMnodeDir, &walCfg); - if (tsSdbSync->pWal == NULL) { + tsSdbSync->wal = walOpen(tsMnodeDir, &walCfg); + if (tsSdbSync->wal == NULL) { sdbError("failed to open sdb in %s", tsMnodeDir); return -1; } sdbTrace("open sdb file for read"); - walRestore(tsSdbSync->pWal, tsSdbSync, sdbProcessWrite); + walRestore(tsSdbSync->wal, tsSdbSync, sdbProcessWrite); int32_t totalRows = 0; int32_t numOfTables = 0; @@ -166,7 +166,7 @@ void sdbCleanUp() { if (tsSdbSync) { sem_destroy(&tsSdbSync->sem); pthread_mutex_destroy(&tsSdbSync->mutex); - walClose(tsSdbSync->pWal); + walClose(tsSdbSync->wal); tsSdbSync = NULL; } } @@ -174,25 +174,28 @@ void sdbCleanUp() { void sdbIncRef(void *handle, void *pRow) { if (pRow) { SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - //} + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), + *pRefCount); + } } } void sdbDecRef(void *handle, void *pRow) { if (pRow) { SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - //} - int8_t* updateEnd = pRow + pTable->refCountPos - 1; + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), + *pRefCount); + } + int8_t *updateEnd = pRow + pTable->refCountPos - 1; if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, + sdbGetkeyStr(pTable, pRow), *pRefCount); SSdbOperDesc oper = {.pObj = pRow}; (*pTable->destroyFp)(&oper); } @@ -228,7 +231,7 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { +static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { SRowMeta rowMeta; rowMeta.rowSize = pOper->rowSize; rowMeta.row = pOper->pObj; @@ -239,41 +242,144 @@ static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { pTable->numOfRows++; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; } -static int32_t sdbDeleteLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { +static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { pthread_mutex_lock(&pTable->mutex); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->deleteFp)(pOper); - int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; + int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; sdbDecRef(pTable, pOper->pObj); - + return TSDB_CODE_SUCCESS; } -static int32_t sdbUpdateLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { - sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); +static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { + sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; } +static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) { + int32_t code = 0; + + pthread_mutex_lock(&tsSdbSync->mutex); + tsSdbSync->version++; + pHead->version = tsSdbSync->version; + + code = sdbForwardDbReqToPeer(pHead); + if (code != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + return code; + } + + code = walWrite(tsSdbSync->wal, pHead); + pthread_mutex_unlock(&tsSdbSync->mutex); + + if (code < 0) { + sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + } else { + sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), + sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + walFsync(tsSdbSync->wal); + free(pHead); + + return code; +} + +static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) { + pthread_mutex_lock(&tsSdbSync->mutex); + if (pHead->version <= tsSdbSync->version) { + pthread_mutex_unlock(&tsSdbSync->mutex); + return TSDB_CODE_SUCCESS; + } else if (pHead->version != tsSdbSync->version + 1) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, + pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + tsSdbSync->version); + return TSDB_CODE_OTHERS; + } + + tsSdbSync->version = pHead->version; + sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + + int32_t code = -1; + if (action == SDB_ACTION_INSERT) { + SSdbOperDesc oper = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + + code = sdbInsertLocal(pTable, &oper); + } else if (action == SDB_ACTION_DELETE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper = { + .table = pTable, + .pObj = rowMeta->row, + }; + + code = sdbDeleteLocal(pTable, &oper); + } else if (action == SDB_ACTION_UPDATE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper1 = { + .table = pTable, + .pObj = rowMeta->row, + }; + sdbDeleteLocal(pTable, &oper1); + + SSdbOperDesc oper2 = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper2); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + code = sdbInsertLocal(pTable, &oper2); + } + + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; +} + static int sdbProcessWrite(void *param, void *data, int type) { SWalHead *pHead = data; - int32_t code = 0; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -281,108 +387,9 @@ static int sdbProcessWrite(void *param, void *data, int type) { assert(pTable != NULL); if (pHead->version == 0) { - // from mgmt, update version - pthread_mutex_lock(&tsSdbSync->mutex); - tsSdbSync->version++; - pHead->version = tsSdbSync->version; - - code = sdbForwardDbReqToPeer(pHead); - if (code != TSDB_CODE_SUCCESS) { - pthread_mutex_unlock(&tsSdbSync->mutex); - sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); - return code; - } - - code = walWrite(tsSdbSync->pWal, pHead); - pthread_mutex_unlock(&tsSdbSync->mutex); - - if (code < 0) { - sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); - } else { - sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), - sdbGetkeyStr(pTable, pHead->cont), pHead->version); - } - - walFsync(tsSdbSync->pWal); - free(pHead); - - return code; + return sdbProcessWriteFromApp(pTable, pHead, action); } else { - // for data from WAL or forward, version may be smaller - pthread_mutex_lock(&tsSdbSync->mutex); - - if (pHead->version <= tsSdbSync->version) { - pthread_mutex_unlock(&tsSdbSync->mutex); - return TSDB_CODE_SUCCESS; - } else if (pHead->version != tsSdbSync->version + 1) { - pthread_mutex_unlock(&tsSdbSync->mutex); - sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, - pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, - tsSdbSync->version); - return TSDB_CODE_OTHERS; - } else { - tsSdbSync->version = pHead->version; - sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - } - - - - code = -1; - if (action == SDB_ACTION_INSERT) { - SSdbOperDesc oper = { - .rowSize = pHead->len, - .rowData = pHead->cont, - .table = pTable, - }; - code = (*pTable->decodeFp)(&oper); - if (code < 0) { - sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; - } - - code = sdbInsertLocal(pTable, &oper); - } else if (action == SDB_ACTION_DELETE) { - SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); - - SSdbOperDesc oper = { - .table = pTable, - .pObj = rowMeta->row, - }; - - code = sdbDeleteLocal(pTable, &oper); - } else if (action == SDB_ACTION_UPDATE) { - SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); - - SSdbOperDesc oper1 = { - .table = pTable, - .pObj = rowMeta->row, - }; - sdbDeleteLocal(pTable, &oper1); - - SSdbOperDesc oper2 = { - .rowSize = pHead->len, - .rowData = pHead->cont, - .table = pTable, - }; - code = (*pTable->decodeFp)(&oper2); - if (code < 0) { - sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; - } - code = sdbInsertLocal(pTable, &oper2); - } - - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; + return sdbProcessWriteFromWal(pTable, pHead, action); } } @@ -396,7 +403,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + if (pTable->keyType == SDB_KEY_AUTO) { pthread_mutex_lock(&pTable->mutex); *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; @@ -407,7 +414,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); } - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; SWalHead *pHead = calloc(1, size); pHead->version = 0; @@ -439,13 +446,13 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t rowSize = 0; switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: + case SDB_KEY_STRING: rowSize = strlen((char *)pOper->pObj) + 1; break; - case SDB_KEY_TYPE_AUTO: + case SDB_KEY_AUTO: rowSize = sizeof(uint64_t); break; default: @@ -479,7 +486,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; SWalHead *pHead = calloc(1, size); pHead->version = 0; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 19b747b745..ef38702a96 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -240,7 +240,7 @@ static int32_t mgmtChildTableActionUpdateAll() { if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -254,7 +254,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -268,7 +268,7 @@ static int32_t mgmtChildTableActionUpdateAll() { pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -280,7 +280,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -294,7 +294,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -318,7 +318,7 @@ static int32_t mgmtInitChildTables() { .hashSessions = tsMaxTables, .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -433,7 +433,7 @@ static int32_t mgmtInitSuperTables() { .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -676,7 +676,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = sizeof(SSuperTableObj) + schemaSize @@ -700,7 +700,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable }; @@ -751,7 +751,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i pStable->sversion++; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -782,7 +782,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { pStable->schema = realloc(pStable->schema, schemaSize); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -817,7 +817,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -876,7 +876,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -913,7 +913,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -1061,7 +1061,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsSuperTableSdb, .pObj = pTable, }; @@ -1276,7 +1276,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj } SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_GLOBAL; + desc.type = SDB_OPER_GLOBAL; desc.pObj = pTable; desc.table = tsChildTableSdb; @@ -1430,7 +1430,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, .rowSize = tsChildTableUpdateSize @@ -1464,7 +1464,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, .rowSize = tsChildTableUpdateSize @@ -1606,7 +1606,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, }; @@ -1635,7 +1635,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { if (pTable->superTable == pStable) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, }; @@ -1724,7 +1724,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable }; @@ -1767,7 +1767,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { queueMsg->thandle, tstrerror(rpcMsg->code)); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable }; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index b216e8f36d..3a49e56331 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -104,7 +104,7 @@ int32_t mgmtInitUsers() { .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtUserActionInsert, .deleteFp = mgmtUserActionDelete, .updateFp = mgmtUserActionUpdate, @@ -144,7 +144,7 @@ void mgmtReleaseUser(SUserObj *pUser) { static int32_t mgmtUpdateUser(SUserObj *pUser) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, .rowSize = tsUserUpdateSize @@ -192,7 +192,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, .rowSize = sizeof(SUserObj) @@ -209,7 +209,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { static int32_t mgmtDropUser(SUserObj *pUser) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser }; @@ -483,7 +483,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsUserSdb, .pObj = pUser, }; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 09fdffb68e..19468dc547 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -166,7 +166,7 @@ int32_t mgmtInitVgroups() { .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_AUTO, + .keyType = SDB_KEY_AUTO, .insertFp = mgmtVgroupActionInsert, .deleteFp = mgmtVgroupActionDelete, .updateFp = mgmtVgroupActionUpdate, @@ -203,7 +203,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) { void mgmtUpdateVgroup(SVgObj *pVgroup) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, .rowSize = tsVgUpdateSize @@ -247,7 +247,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, .rowSize = sizeof(SVgObj) @@ -279,7 +279,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -586,7 +586,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { mgmtAddToShellQueue(newMsg); } else { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -649,7 +649,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -706,7 +706,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsVgroupSdb, .pObj = pVgroup, }; From 1fb5a8c2fd8ff2819f7450924125f5a1795ad54a Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 14 Apr 2020 00:01:31 +0800 Subject: [PATCH 23/36] fix some memory lost --- src/dnode/src/dnodeMClient.c | 1 + src/mnode/src/mgmtSdb.c | 3 ++- src/plugins/http/inc/tgHandle.h | 1 + src/plugins/http/src/httpSystem.c | 4 +++- src/plugins/http/src/tgHandle.c | 5 +++++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index f9794d040b..5dd015313a 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -249,6 +249,7 @@ static bool dnodeReadMnodeIpList() { PARSE_OVER: free(content); + cJSON_Delete(root); fclose(fp); return ret; } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 5202f8511c..bc0d1b81f2 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -167,6 +167,7 @@ void sdbCleanUp() { sem_destroy(&tsSdbSync->sem); pthread_mutex_destroy(&tsSdbSync->mutex); walClose(tsSdbSync->wal); + free(tsSdbSync); tsSdbSync = NULL; } } @@ -576,5 +577,5 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables); - tfree(pTable); + free(pTable); } diff --git a/src/plugins/http/inc/tgHandle.h b/src/plugins/http/inc/tgHandle.h index 5b8c49900f..5622694374 100644 --- a/src/plugins/http/inc/tgHandle.h +++ b/src/plugins/http/inc/tgHandle.h @@ -33,6 +33,7 @@ #define TG_PASS_URL_POS 3 void tgInitHandle(HttpServer *pServer); +void tgCleanupHandle(); bool tgProcessRquest(struct HttpContext *pContext); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index aa66af9825..46f31a12d6 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -122,12 +122,14 @@ void httpStopSystem() { if (httpServer != NULL) { httpServer->online = false; } + tgCleanupHandle(); } void httpCleanUpSystem() { httpPrint("http service cleanup"); httpStopSystem(); -#if 0 + +#if 1 if (httpServer == NULL) { return; } diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index c22a7bbdef..2be39743bd 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -116,6 +116,7 @@ void tgFreeSchemas() { } free(tgSchemas.schemas); tgSchemas.size = 0; + tgSchemas.schemas = NULL; } } @@ -290,6 +291,10 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } +void tgCleanupHandle() { + tgFreeSchemas(); +} + bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = &pContext->parser; if (pParser->path[TG_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[TG_USER_URL_POS].len <= 0) { From 8863766f7322372f2b72534797ae8d80ede44399 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 14 Apr 2020 11:02:01 +0800 Subject: [PATCH 24/36] make print output be python2 and python3 compatible. --- tests/pytest/util/log.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index e570af85f5..c7032df3c4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +from distutils.log import warn as printf class TDLog: @@ -22,27 +23,27 @@ class TDLog: self.path = "" def info(self, info): - print("%s %s" % (datetime.datetime.now(), info)) + printf("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) - def printNoPrefix(self, info): - print("\033[1;36m%s\033[0m" % (info)) + def printfNoPrefix(self, info): + printf("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog() From b286a296bf735ca0ecb99501c75e9540b4bb2f49 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 14 Apr 2020 16:38:15 +0800 Subject: [PATCH 25/36] [TD-52] first version mpeer --- src/dnode/CMakeLists.txt | 4 ++ src/dnode/inc/dnodeMgmt.h | 1 + src/dnode/src/dnodeMClient.c | 2 +- src/dnode/src/dnodeMgmt.c | 4 ++ src/inc/mnode.h | 1 - src/inc/mpeer.h | 5 ++ src/mnode/inc/mgmtSdb.h | 12 ++++ src/mnode/src/mgmtMain.c | 10 +-- src/mnode/src/mgmtMnode.c | 5 +- src/mnode/src/mgmtSdb.c | 119 ++++++++++++++++------------------- 10 files changed, 87 insertions(+), 76 deletions(-) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index a81f8c0c9d..ee05403a61 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -31,6 +31,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) TARGET_LINK_LIBRARIES(taosd balance sync) ENDIF () + IF (TD_MPEER) + TARGET_LINK_LIBRARIES(taosd mpeer sync) + ENDIF () + SET(PREPARE_ENV_CMD "prepare_env_cmd") SET(PREPARE_ENV_TARGET "prepare_env_target") ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index f944bd5add..0be6e40e75 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -24,6 +24,7 @@ int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); void dnodeMgmt(SRpcMsg *rpcMsg); void dnodeUpdateDnodeId(int32_t dnodeId); +int32_t dnodeGetDnodeId(); void* dnodeGetVnode(int32_t vgId); int32_t dnodeGetVnodeStatus(void *pVnode); diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 5dd015313a..53eea93137 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -294,4 +294,4 @@ uint32_t dnodeGetMnodeMasteIp() { void* dnodeGetMpeerInfos() { return &tsMnodeInfos; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 0ec769c0af..f4cbd3e1be 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -299,3 +299,7 @@ void dnodeUpdateDnodeId(int32_t dnodeId) { dnodeSaveDnodeId(); } } + +int32_t dnodeGetDnodeId() { + return tsDnodeId; +} \ No newline at end of file diff --git a/src/inc/mnode.h b/src/inc/mnode.h index f2c072453f..00b7519258 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -45,7 +45,6 @@ struct _mnode_obj; typedef struct _mnode_obj { int32_t mnodeId; - int32_t dnodeId; int64_t createdTime; int8_t reserved[14]; int8_t updateEnd[1]; diff --git a/src/inc/mpeer.h b/src/inc/mpeer.h index 157ea40119..e5051b39eb 100644 --- a/src/inc/mpeer.h +++ b/src/inc/mpeer.h @@ -45,6 +45,11 @@ void mpeerGetMpeerInfos(void *mpeers); char * mpeerGetMnodeStatusStr(int32_t status); char * mpeerGetMnodeRoleStr(int32_t role); +int32_t mpeerAddMnode(int32_t dnodeId); +int32_t mpeerRemoveMnode(int32_t dnodeId); + +int32_t sdbForwardDbReqToPeer(void *pHead); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 83afa2a081..27f9a51650 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -34,6 +34,7 @@ typedef enum { typedef enum { SDB_KEY_STRING, + SDB_KEY_INT, SDB_KEY_AUTO } ESdbKeyType; @@ -66,8 +67,19 @@ typedef struct { int32_t (*updateAllFp)(); } SSdbTableDesc; +typedef struct { + int32_t code; + int64_t version; + void * sync; + void * wal; + sem_t sem; + pthread_mutex_t mutex; +} SSdbObject; + int32_t sdbInit(); void sdbCleanUp(); +SSdbObject *sdbGetObj(); +int sdbProcessWrite(void *param, void *data, int type); void * sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index a074060f52..b6fb1ba425 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -109,6 +109,11 @@ int32_t mgmtStartSystem() { return -1; } + if (mpeerInit() < 0) { + mError("failed to init mpeers"); + return -1; + } + if (sdbInit() < 0) { mError("failed to init sdb"); return -1; @@ -122,11 +127,6 @@ int32_t mgmtStartSystem() { return -1; } - if (mpeerInit() < 0) { - mError("failed to init mpeers"); - return -1; - } - if (balanceInit() < 0) { mError("failed to init dnode balance") } diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 82da454793..faa66d1fd9 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -79,7 +79,7 @@ void mpeerGetMpeerInfos(void *param) { strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName); } -void mpeerCleanupDnodes() {} +void mpeerCleanupMnodes() {} int32_t mpeerGetMnodesNum() { return 1; } void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} bool mpeerInServerStatus() { return tsMnodeObj.status == TAOS_MN_STATUS_READY; } @@ -91,12 +91,11 @@ bool mpeerCheckRedirect() { return false; } int32_t mpeerInit() { mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); - return mpeerInitMnodes(); } void mpeerCleanup() { - mpeerCleanupDnodes(); + mpeerCleanupMnodes(); } char *mpeerGetMnodeStatusStr(int32_t status) { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index bc0d1b81f2..c47d9cd9d8 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -24,19 +24,11 @@ #include "tutil.h" #include "twal.h" #include "tsync.h" +#include "mpeer.h" #include "hashint.h" #include "hashstr.h" #include "mgmtSdb.h" -typedef struct { - int32_t code; - int64_t version; - void * sync; - void * wal; - sem_t sem; - pthread_mutex_t mutex; -} SSdbSync; - typedef struct _SSdbTable { char tableName[TSDB_DB_NAME_LEN + 1]; ESdbTable tableId; @@ -70,17 +62,16 @@ typedef enum { static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0}; static int32_t tsSdbNumOfTables = 0; -static SSdbSync * tsSdbSync; +static SSdbObject * tsSdbObj; -static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; -static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; -static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; -static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; -static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; -static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; -static int sdbProcessWrite(void *param, void *data, int type); +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData}; -uint64_t sdbGetVersion() { return tsSdbSync->version; } +uint64_t sdbGetVersion() { return tsSdbObj->version; } int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } @@ -101,6 +92,7 @@ static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { switch (pTable->keyType) { case SDB_KEY_STRING: return (char *)row; + case SDB_KEY_INT: case SDB_KEY_AUTO: sprintf(str, "%d", *(int32_t *)row); return str; @@ -113,40 +105,30 @@ static void *sdbGetTableFromId(int32_t tableId) { return tsSdbTableList[tableId]; } -// static void mpeerConfirmForward(void *ahandle, void *param, int32_t code) { -// sem_post(&tsSdbSync->sem); -// mPrint("mpeerConfirmForward"); -// } - -static int32_t sdbForwardDbReqToPeer(SWalHead *pHead) { - // int32_t code = syncForwardToPeer(NULL, pHead, NULL); - // if (code < 0) { - // return code; - // } - - // sem_wait(&tsSdbSync->sem); - // return tsSdbSync->code; +#ifndef _MPEER +int32_t sdbForwardDbReqToPeer(void *pHead) { return TSDB_CODE_SUCCESS; } +#endif int32_t sdbInit() { - tsSdbSync = calloc(1, sizeof(SSdbSync)); - sem_init(&tsSdbSync->sem, 0, 0); - pthread_mutex_init(&tsSdbSync->mutex, NULL); + tsSdbObj = calloc(1, sizeof(SSdbObject)); + sem_init(&tsSdbObj->sem, 0, 0); + pthread_mutex_init(&tsSdbObj->mutex, NULL); SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; - tsSdbSync->wal = walOpen(tsMnodeDir, &walCfg); - if (tsSdbSync->wal == NULL) { + tsSdbObj->wal = walOpen(tsMnodeDir, &walCfg); + if (tsSdbObj->wal == NULL) { sdbError("failed to open sdb in %s", tsMnodeDir); return -1; } sdbTrace("open sdb file for read"); - walRestore(tsSdbSync->wal, tsSdbSync, sdbProcessWrite); + walRestore(tsSdbObj->wal, tsSdbObj, sdbProcessWrite); int32_t totalRows = 0; int32_t numOfTables = 0; - for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) { + for (int32_t tableId = SDB_TABLE_MNODE; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; if (pTable->updateAllFp) { @@ -158,20 +140,24 @@ int32_t sdbInit() { sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows); } - sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbSync->version, totalRows, numOfTables); + sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables); return TSDB_CODE_SUCCESS; } void sdbCleanUp() { - if (tsSdbSync) { - sem_destroy(&tsSdbSync->sem); - pthread_mutex_destroy(&tsSdbSync->mutex); - walClose(tsSdbSync->wal); - free(tsSdbSync); - tsSdbSync = NULL; + if (tsSdbObj) { + sem_destroy(&tsSdbObj->sem); + pthread_mutex_destroy(&tsSdbObj->mutex); + walClose(tsSdbObj->wal); + free(tsSdbObj); + tsSdbObj = NULL; } } +SSdbObject *sdbGetObj() { + return tsSdbObj; +} + void sdbIncRef(void *handle, void *pRow) { if (pRow) { SSdbTable *pTable = handle; @@ -278,20 +264,20 @@ static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) { int32_t code = 0; - pthread_mutex_lock(&tsSdbSync->mutex); - tsSdbSync->version++; - pHead->version = tsSdbSync->version; + pthread_mutex_lock(&tsSdbObj->mutex); + tsSdbObj->version++; + pHead->version = tsSdbObj->version; code = sdbForwardDbReqToPeer(pHead); if (code != TSDB_CODE_SUCCESS) { - pthread_mutex_unlock(&tsSdbSync->mutex); + pthread_mutex_unlock(&tsSdbObj->mutex); sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); return code; } - code = walWrite(tsSdbSync->wal, pHead); - pthread_mutex_unlock(&tsSdbSync->mutex); + code = walWrite(tsSdbObj->wal, pHead); + pthread_mutex_unlock(&tsSdbObj->mutex); if (code < 0) { sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, @@ -301,26 +287,26 @@ static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_ sdbGetkeyStr(pTable, pHead->cont), pHead->version); } - walFsync(tsSdbSync->wal); + walFsync(tsSdbObj->wal); free(pHead); return code; } static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) { - pthread_mutex_lock(&tsSdbSync->mutex); - if (pHead->version <= tsSdbSync->version) { - pthread_mutex_unlock(&tsSdbSync->mutex); + pthread_mutex_lock(&tsSdbObj->mutex); + if (pHead->version <= tsSdbObj->version) { + pthread_mutex_unlock(&tsSdbObj->mutex); return TSDB_CODE_SUCCESS; - } else if (pHead->version != tsSdbSync->version + 1) { - pthread_mutex_unlock(&tsSdbSync->mutex); + } else if (pHead->version != tsSdbObj->version + 1) { + pthread_mutex_unlock(&tsSdbObj->mutex); sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, - tsSdbSync->version); + tsSdbObj->version); return TSDB_CODE_OTHERS; } - tsSdbSync->version = pHead->version; + tsSdbObj->version = pHead->version; sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); @@ -335,7 +321,7 @@ static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_ if (code < 0) { sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); + pthread_mutex_unlock(&tsSdbObj->mutex); return code; } @@ -369,17 +355,17 @@ static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_ if (code < 0) { sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); + pthread_mutex_unlock(&tsSdbObj->mutex); return code; } code = sdbInsertLocal(pTable, &oper2); } - pthread_mutex_unlock(&tsSdbSync->mutex); + pthread_mutex_unlock(&tsSdbObj->mutex); return code; } -static int sdbProcessWrite(void *param, void *data, int type) { +int sdbProcessWrite(void *param, void *data, int type) { SWalHead *pHead = data; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -426,7 +412,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { (*pTable->encodeFp)(pOper); pHead->len = pOper->rowSize; - int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType); if (code < 0) return code; } @@ -453,6 +439,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { case SDB_KEY_STRING: rowSize = strlen((char *)pOper->pObj) + 1; break; + case SDB_KEY_INT: case SDB_KEY_AUTO: rowSize = sizeof(uint64_t); break; @@ -467,7 +454,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; memcpy(pHead->cont, pOper->pObj, rowSize); - int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType); if (code < 0) return code; } @@ -497,7 +484,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { (*pTable->encodeFp)(pOper); pHead->len = pOper->rowSize; - int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType); if (code < 0) return code; } From a7e1c7cec6cde547734d4181c38ad1b8bd6794e7 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 14 Apr 2020 23:33:22 +0800 Subject: [PATCH 26/36] [TD-52] refactor sdb codes --- src/inc/mnode.h | 5 ++-- src/inc/mpeer.h | 19 ++++++------- src/inc/tbalance.h | 1 + src/mnode/inc/mgmtSdb.h | 7 ++--- src/mnode/src/mgmtDb.c | 4 +-- src/mnode/src/mgmtMnode.c | 54 ++++++++++++------------------------- src/mnode/src/mgmtSdb.c | 37 +++++++++---------------- src/mnode/src/mgmtShell.c | 21 +-------------- src/mnode/src/mgmtTable.c | 8 +++--- src/mnode/src/mgmtUser.c | 4 +-- src/mnode/src/mgmtVgroup.c | 4 +-- tests/script/tmp/dnode2.sim | 6 ----- tests/script/tmp/mnodes.sim | 7 +++++ 13 files changed, 62 insertions(+), 115 deletions(-) delete mode 100644 tests/script/tmp/dnode2.sim create mode 100644 tests/script/tmp/mnodes.sim diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 00b7519258..e8a0ba3bcc 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -49,11 +49,10 @@ typedef struct _mnode_obj { int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; - int8_t role; - int8_t status; - uint16_t port; uint32_t privateIp; uint32_t publicIp; + uint16_t port; + int8_t role; char mnodeName[TSDB_NODE_NAME_LEN + 1]; } SMnodeObj; diff --git a/src/inc/mpeer.h b/src/inc/mpeer.h index e5051b39eb..e7abf09321 100644 --- a/src/inc/mpeer.h +++ b/src/inc/mpeer.h @@ -28,27 +28,28 @@ enum _TAOS_MN_STATUS { TAOS_MN_STATUS_READY }; +// general implementation int32_t mpeerInit(); void mpeerCleanup(); + +// special implementation +int32_t mpeerInitMnodes(); +void mpeerCleanupMnodes(); +int32_t mpeerAddMnode(int32_t dnodeId); +int32_t mpeerRemoveMnode(int32_t dnodeId); + +void * mpeerGetMnode(int32_t mnodeId); int32_t mpeerGetMnodesNum(); void * mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode); void mpeerReleaseMnode(struct _mnode_obj *pMnode); -bool mpeerInServerStatus(); bool mpeerIsMaster(); -bool mpeerCheckRedirect(); void mpeerGetPrivateIpList(SRpcIpSet *ipSet); void mpeerGetPublicIpList(SRpcIpSet *ipSet); void mpeerGetMpeerInfos(void *mpeers); -char * mpeerGetMnodeStatusStr(int32_t status); -char * mpeerGetMnodeRoleStr(int32_t role); - -int32_t mpeerAddMnode(int32_t dnodeId); -int32_t mpeerRemoveMnode(int32_t dnodeId); - -int32_t sdbForwardDbReqToPeer(void *pHead); +int32_t mpeerForwardReqToPeer(void *pHead); #ifdef __cplusplus } diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h index 8cf8cb9fb9..c73d6a91a9 100644 --- a/src/inc/tbalance.h +++ b/src/inc/tbalance.h @@ -31,6 +31,7 @@ struct _dnode_obj; int32_t balanceInit(); void balanceCleanUp(); void balanceNotify(); +void balanceReset(); int32_t balanceAllocVnodes(struct _vg_obj *pVgroup); int32_t balanceDropDnode(struct _dnode_obj *pDnode); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 27f9a51650..2804d40a71 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -64,25 +64,22 @@ typedef struct { int32_t (*encodeFp)(SSdbOperDesc *pOper); int32_t (*decodeFp)(SSdbOperDesc *pDesc); int32_t (*destroyFp)(SSdbOperDesc *pDesc); - int32_t (*updateAllFp)(); + int32_t (*restoredFp)(); } SSdbTableDesc; typedef struct { - int32_t code; int64_t version; - void * sync; void * wal; - sem_t sem; pthread_mutex_t mutex; } SSdbObject; int32_t sdbInit(); void sdbCleanUp(); SSdbObject *sdbGetObj(); -int sdbProcessWrite(void *param, void *data, int type); void * sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); +int sdbProcessWrite(void *param, void *data, int type); int32_t sdbInsertRow(SSdbOperDesc *pOper); int32_t sdbDeleteRow(SSdbOperDesc *pOper); diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 946ec29d8c..089bf494e7 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -102,7 +102,7 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionUpdateAll() { +static int32_t mgmtDbActionRestored() { return 0; } @@ -123,7 +123,7 @@ int32_t mgmtInitDbs() { .encodeFp = mgmtDbActionEncode, .decodeFp = mgmtDbActionDecode, .destroyFp = mgmtDbActionDestroy, - .updateAllFp = mgmtDbActionUpdateAll + .restoredFp = mgmtDbActionRestored }; tsDbSdb = sdbOpenTable(&tableDesc); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index faa66d1fd9..ca18d6bdba 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -19,12 +19,9 @@ #include "trpc.h" #include "tsync.h" #include "mpeer.h" -#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" -extern int32_t mpeerInitMnodes(); -extern void mpeerCleanupMnodes(); static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -34,18 +31,24 @@ static SMnodeObj tsMnodeObj = {0}; int32_t mpeerInitMnodes() { tsMnodeObj.mnodeId = 1; - tsMnodeObj.dnodeId = 1; tsMnodeObj.privateIp = inet_addr(tsPrivateIp); tsMnodeObj.publicIp = inet_addr(tsPublicIp); tsMnodeObj.createdTime = taosGetTimestampMs(); tsMnodeObj.role = TAOS_SYNC_ROLE_MASTER; - tsMnodeObj.status = TAOS_MN_STATUS_READY; tsMnodeObj.port = tsMnodeDnodePort; sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId); return TSDB_CODE_SUCCESS; } +void mpeerCleanupMnodes() {} +int32_t mpeerAddMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; } +int32_t mpeerRemoveMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; } +void * mpeerGetMnode(int32_t mnodeId) { return &tsMnodeObj; } +int32_t mpeerGetMnodesNum() { return 1; } +void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} +bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; } + void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -58,20 +61,21 @@ void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) { void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; + ipSet->port = htons(tsMnodeObj.port); ipSet->ip[0] = htonl(tsMnodeObj.privateIp); } void mpeerGetPublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; + ipSet->port = htons(tsMnodeObj.port); ipSet->ip[0] = htonl(tsMnodeObj.publicIp); } void mpeerGetMpeerInfos(void *param) { SDMNodeInfos *mpeers = param; + mpeers->inUse = 0; mpeers->nodeNum = 1; mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId); mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp); @@ -79,12 +83,9 @@ void mpeerGetMpeerInfos(void *param) { strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName); } -void mpeerCleanupMnodes() {} -int32_t mpeerGetMnodesNum() { return 1; } -void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} -bool mpeerInServerStatus() { return tsMnodeObj.status == TAOS_MN_STATUS_READY; } -bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; } -bool mpeerCheckRedirect() { return false; } +int32_t mpeerForwardReqToPeer(void *pHead) { + return TSDB_CODE_SUCCESS; +} #endif @@ -98,20 +99,7 @@ void mpeerCleanup() { mpeerCleanupMnodes(); } -char *mpeerGetMnodeStatusStr(int32_t status) { - switch (status) { - case TAOS_MN_STATUS_OFFLINE: - return "offline"; - case TAOS_MN_STATUS_DROPPING: - return "dropping"; - case TAOS_MN_STATUS_READY: - return "ready"; - default: - return "undefined"; - } -} - -char *mpeerGetMnodeRoleStr(int32_t role) { +static char *mpeerGetMnodeRoleStr(int32_t role) { switch (role) { case TAOS_SYNC_ROLE_OFFLINE: return "offline"; @@ -159,12 +147,6 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 10; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pShow->bytes[cols] = 10; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); @@ -219,14 +201,12 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mpeerGetMnodeStatusStr(pMnode->status)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role)); cols++; numOfRows++; + + mpeerReleaseMnode(pMnode); } pShow->numOfReads += numOfRows; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index c47d9cd9d8..3038b0eadc 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -15,18 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" #include "taoserror.h" -#include "tchecksum.h" -#include "tglobalcfg.h" #include "tlog.h" #include "trpc.h" -#include "tutil.h" #include "twal.h" -#include "tsync.h" -#include "mpeer.h" #include "hashint.h" #include "hashstr.h" +#include "mpeer.h" #include "mgmtSdb.h" typedef struct _SSdbTable { @@ -39,13 +34,13 @@ typedef struct _SSdbTable { int32_t autoIndex; int64_t numOfRows; void * iHandle; - int32_t (*insertFp)(SSdbOperDesc *pDesc); - int32_t (*deleteFp)(SSdbOperDesc *pOper); - int32_t (*updateFp)(SSdbOperDesc *pOper); - int32_t (*decodeFp)(SSdbOperDesc *pOper); - int32_t (*encodeFp)(SSdbOperDesc *pOper); - int32_t (*destroyFp)(SSdbOperDesc *pOper); - int32_t (*updateAllFp)(); + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); + int32_t (*restoredFp)(); pthread_mutex_t mutex; } SSdbTable; @@ -105,15 +100,8 @@ static void *sdbGetTableFromId(int32_t tableId) { return tsSdbTableList[tableId]; } -#ifndef _MPEER -int32_t sdbForwardDbReqToPeer(void *pHead) { - return TSDB_CODE_SUCCESS; -} -#endif - int32_t sdbInit() { tsSdbObj = calloc(1, sizeof(SSdbObject)); - sem_init(&tsSdbObj->sem, 0, 0); pthread_mutex_init(&tsSdbObj->mutex, NULL); SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; @@ -131,8 +119,8 @@ int32_t sdbInit() { for (int32_t tableId = SDB_TABLE_MNODE; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; - if (pTable->updateAllFp) { - (*pTable->updateAllFp)(); + if (pTable->restoredFp) { + (*pTable->restoredFp)(); } totalRows += pTable->numOfRows; @@ -146,7 +134,6 @@ int32_t sdbInit() { void sdbCleanUp() { if (tsSdbObj) { - sem_destroy(&tsSdbObj->sem); pthread_mutex_destroy(&tsSdbObj->mutex); walClose(tsSdbObj->wal); free(tsSdbObj); @@ -268,7 +255,7 @@ static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_ tsSdbObj->version++; pHead->version = tsSdbObj->version; - code = sdbForwardDbReqToPeer(pHead); + code = mpeerForwardReqToPeer(pHead); if (code != TSDB_CODE_SUCCESS) { pthread_mutex_unlock(&tsSdbObj->mutex); sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, @@ -523,7 +510,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->encodeFp = pDesc->encodeFp; pTable->decodeFp = pDesc->decodeFp; pTable->destroyFp = pDesc->destroyFp; - pTable->updateAllFp = pDesc->updateAllFp; + pTable->restoredFp = pDesc->restoredFp; if (sdbInitIndexFp[pTable->keyType] != NULL) { pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index dbd7627d3f..5010429db3 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -42,7 +42,6 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *sec static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg); static void mgmtProcessMsgFromShell(SRpcMsg *pMsg); static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg); -static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg); static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg); static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg); static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg); @@ -142,19 +141,13 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } - if (mpeerCheckRedirect()) { + if (!mpeerIsMaster()) { // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect()); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER); rpcFreeCont(rpcMsg->pCont); return; } - if (!mpeerInServerStatus()) { - mgmtProcessMsgWhileNotReady(rpcMsg); - rpcFreeCont(rpcMsg->pCont); - return; - } - if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED); rpcFreeCont(rpcMsg->pCont); @@ -501,18 +494,6 @@ static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) { rpcSendResponse(&rpcRsp); } -static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg) { - mTrace("%s is ignored since SDB is not ready", taosMsg[rpcMsg->msgType]); - SRpcMsg rpcRsp = { - .msgType = 0, - .pCont = 0, - .contLen = 0, - .code = TSDB_CODE_NOT_READY, - .handle = rpcMsg->handle - }; - rpcSendResponse(&rpcRsp); -} - void mgmtSendSimpleResp(void *thandle, int32_t code) { SRpcMsg rpcRsp = { .msgType = 0, diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index feed12f97e..1c384fdfdf 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -220,7 +220,7 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionUpdateAll() { +static int32_t mgmtChildTableActionRestored() { void *pNode = NULL; void *pLastNode = NULL; SChildTableObj *pTable = NULL; @@ -320,7 +320,7 @@ static int32_t mgmtInitChildTables() { .encodeFp = mgmtChildTableActionEncode, .decodeFp = mgmtChildTableActionDecode, .destroyFp = mgmtChildTableActionDestroy, - .updateAllFp = mgmtChildTableActionUpdateAll + .restoredFp = mgmtChildTableActionRestored }; tsChildTableSdb = sdbOpenTable(&tableDesc); @@ -414,7 +414,7 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionUpdateAll() { +static int32_t mgmtSuperTableActionRestored() { return 0; } @@ -435,7 +435,7 @@ static int32_t mgmtInitSuperTables() { .encodeFp = mgmtSuperTableActionEncode, .decodeFp = mgmtSuperTableActionDecode, .destroyFp = mgmtSuperTableActionDestroy, - .updateAllFp = mgmtSuperTableActionUpdateAll + .restoredFp = mgmtSuperTableActionRestored }; tsSuperTableSdb = sdbOpenTable(&tableDesc); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 3a49e56331..ef01faf6ba 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -84,7 +84,7 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionUpdateAll() { +static int32_t mgmtUserActionRestored() { SAcctObj *pAcct = acctGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); @@ -111,7 +111,7 @@ int32_t mgmtInitUsers() { .encodeFp = mgmtUserActionEncode, .decodeFp = mgmtUserActionDecode, .destroyFp = mgmtUserActionDestroy, - .updateAllFp = mgmtUserActionUpdateAll + .restoredFp = mgmtUserActionRestored }; tsUserSdb = sdbOpenTable(&tableDesc); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 19468dc547..cc8dba52dd 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -152,7 +152,7 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionUpdateAll() { +static int32_t mgmtVgroupActionRestored() { return 0; } @@ -173,7 +173,7 @@ int32_t mgmtInitVgroups() { .encodeFp = mgmtVgroupActionEncode, .decodeFp = mgmtVgroupActionDecode, .destroyFp = mgmtVgroupActionDestroy, - .updateAllFp = mgmtVgroupActionUpdateAll, + .restoredFp = mgmtVgroupActionRestored, }; tsVgroupSdb = sdbOpenTable(&tableDesc); diff --git a/tests/script/tmp/dnode2.sim b/tests/script/tmp/dnode2.sim deleted file mode 100644 index 6d9a844fb6..0000000000 --- a/tests/script/tmp/dnode2.sim +++ /dev/null @@ -1,6 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 -system sh/exec_up.sh -n dnode1 -s start -system sh/exec_up.sh -n dnode2 -s start -sql connect \ No newline at end of file diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim new file mode 100644 index 0000000000..32e72f16ff --- /dev/null +++ b/tests/script/tmp/mnodes.sim @@ -0,0 +1,7 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 +system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 +system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 +system sh/cfg.sh -n dnode1 -c numOfMPeers -v 3 +system sh/cfg.sh -n dnode2 -c numOfMPeers -v 3 +system sh/cfg.sh -n dnode3 -c numOfMPeers -v 3 From cc52d293efc21124caa3e28c3b914fc64535cebc Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 15:09:47 +0800 Subject: [PATCH 27/36] fix error while sync sdb --- src/dnode/inc/dnodeMClient.h | 1 + src/dnode/inc/dnodeMgmt.h | 2 - src/dnode/src/dnodeMClient.c | 138 +++++++++++++++++++++++++++++++-- src/dnode/src/dnodeMgmt.c | 121 ----------------------------- src/inc/tcluster.h | 1 + src/mnode/inc/mgmtSdb.h | 6 +- src/mnode/src/mgmtDnode.c | 1 + src/mnode/src/mgmtSdb.c | 14 ++-- src/vnode/main/src/vnodeMain.c | 2 +- 9 files changed, 146 insertions(+), 140 deletions(-) diff --git a/src/dnode/inc/dnodeMClient.h b/src/dnode/inc/dnodeMClient.h index ba63894631..594fb84d3b 100644 --- a/src/dnode/inc/dnodeMClient.h +++ b/src/dnode/inc/dnodeMClient.h @@ -25,6 +25,7 @@ void dnodeCleanupMClient(); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); uint32_t dnodeGetMnodeMasteIp(); void * dnodeGetMpeerInfos(); +int32_t dnodeGetDnodeId(); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index 0be6e40e75..b8d01916fe 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -23,8 +23,6 @@ extern "C" { int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); void dnodeMgmt(SRpcMsg *rpcMsg); -void dnodeUpdateDnodeId(int32_t dnodeId); -int32_t dnodeGetDnodeId(); void* dnodeGetVnode(int32_t vgId); int32_t dnodeGetVnodeStatus(void *pVnode); diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 53eea93137..42eca2152b 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -21,30 +21,51 @@ #include "trpc.h" #include "tutil.h" #include "tsync.h" +#include "ttime.h" +#include "ttimer.h" #include "dnode.h" #include "dnodeMClient.h" #include "dnodeModule.h" #include "dnodeMgmt.h" +#include "vnode.h" #define MPEER_CONTENT_LEN 2000 static bool dnodeReadMnodeIpList(); static void dnodeSaveMnodeIpList(); +static void dnodeReadDnodeInfo(); +static void dnodeUpdateDnodeInfo(int32_t dnodeId); static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); +static void dnodeSendStatusMsg(void *handle, void *tmrId); static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); + static void *tsDnodeMClientRpc = NULL; static SRpcIpSet tsMnodeIpList = {0}; static SDMNodeInfos tsMnodeInfos = {0}; +static void *tsDnodeTmr = NULL; +static void *tsStatusTimer = NULL; +static uint32_t tsRebootTime; +static int32_t tsDnodeId = 0; +static char tsDnodeName[TSDB_NODE_NAME_LEN]; int32_t dnodeInitMClient() { + dnodeReadDnodeInfo(); + tsRebootTime = taosGetTimestampSec(); + + tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); + if (tsDnodeTmr == NULL) { + dError("failed to init dnode timer"); + return -1; + } + if (!dnodeReadMnodeIpList()) { memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet)); memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos)); tsMnodeIpList.port = tsMnodeDnodePort; tsMnodeIpList.numOfIps = 1; tsMnodeIpList.ip[0] = inet_addr(tsMasterIp); - if (tsSecondIp[0]) { + if (strcmp(tsSecondIp, tsMasterIp) != 0) { tsMnodeIpList.numOfIps = 2; tsMnodeIpList.ip[1] = inet_addr(tsSecondIp); } @@ -57,8 +78,6 @@ int32_t dnodeInitMClient() { } } - tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; - SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; @@ -79,11 +98,24 @@ int32_t dnodeInitMClient() { return -1; } + tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; + taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); + dPrint("mnode rpc client is opened"); return 0; } void dnodeCleanupMClient() { + if (tsStatusTimer != NULL) { + taosTmrStopA(&tsStatusTimer); + tsStatusTimer = NULL; + } + + if (tsDnodeTmr != NULL) { + taosTmrCleanUp(tsDnodeTmr); + tsDnodeTmr = NULL; + } + if (tsDnodeMClientRpc) { rpcClose(tsDnodeMClientRpc); tsDnodeMClientRpc = NULL; @@ -104,6 +136,7 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) { static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); return; } @@ -111,6 +144,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { SDMNodeInfos *mpeers = &pStatusRsp->mpeers; if (mpeers->nodeNum <= 0) { dError("status msg is invalid, num of ips is %d", mpeers->nodeNum); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); return; } @@ -122,14 +156,16 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp); } - if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0) { + if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0 || tsMnodeInfos.nodeNum == 0) { memcpy(&tsMnodeIpList, &mgmtIpSet, sizeof(SRpcIpSet)); - memcpy(&tsMnodeInfos, mpeers, sizeof(SDMNodeInfos)); + tsMnodeInfos.inUse = mpeers->inUse; + tsMnodeInfos.nodeNum = mpeers->nodeNum; dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); for (int32_t i = 0; i < mpeers->nodeNum; i++) { tsMnodeInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); tsMnodeInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); + strcpy(tsMnodeInfos.nodeInfos[i].nodeName, mpeers->nodeInfos[i].nodeName); dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, tsMnodeInfos.nodeInfos[i].nodeName); @@ -144,7 +180,8 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { pState->dnodeId = htonl(pState->dnodeId); dnodeProcessModuleStatus(pState->moduleStatus); - dnodeUpdateDnodeId(pState->dnodeId); + dnodeUpdateDnodeInfo(pState->dnodeId); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); } void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { @@ -295,3 +332,92 @@ uint32_t dnodeGetMnodeMasteIp() { void* dnodeGetMpeerInfos() { return &tsMnodeInfos; } + +static void dnodeSendStatusMsg(void *handle, void *tmrId) { + if (tsDnodeTmr == NULL) { + dError("dnode timer is already released"); + return; + } + + if (tsStatusTimer == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); + dError("failed to start status timer"); + return; + } + + int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SDMStatusMsg *pStatus = rpcMallocCont(contLen); + if (pStatus == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); + dError("failed to malloc status message"); + return; + } + + strcpy(pStatus->dnodeName, tsDnodeName); + pStatus->version = htonl(tsVersion); + pStatus->dnodeId = htonl(tsDnodeId); + pStatus->privateIp = htonl(inet_addr(tsPrivateIp)); + pStatus->publicIp = htonl(inet_addr(tsPublicIp)); + pStatus->lastReboot = htonl(tsRebootTime); + pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes); + pStatus->numOfCores = htons((uint16_t) tsNumOfCores); + pStatus->diskAvailable = tsAvailDataDirGB; + pStatus->alternativeRole = (uint8_t) tsAlternativeRole; + + vnodeBuildStatusMsg(pStatus); + contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + pStatus->openVnodes = htons(pStatus->openVnodes); + + SRpcMsg rpcMsg = { + .pCont = pStatus, + .contLen = contLen, + .msgType = TSDB_MSG_TYPE_DM_STATUS + }; + + dnodeSendMsgToMnode(&rpcMsg); +} + +static void dnodeReadDnodeInfo() { + char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; + sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); + + FILE *fp = fopen(dnodeIdFile, "r"); + if (!fp) return; + + char option[32] = {0}; + int32_t value = 0; + int32_t num = 0; + + num = fscanf(fp, "%s %d", option, &value); + if (num != 2) return; + if (strcmp(option, "dnodeId") != 0) return; + tsDnodeId = value;; + + fclose(fp); + dPrint("read dnodeId:%d successed", tsDnodeId); +} + +static void dnodeSaveDnodeInfo() { + char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; + sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); + + FILE *fp = fopen(dnodeIdFile, "w"); + if (!fp) return; + + fprintf(fp, "dnodeId %d\n", tsDnodeId); + + fclose(fp); + dPrint("save dnodeId successed"); +} + +void dnodeUpdateDnodeInfo(int32_t dnodeId) { + if (tsDnodeId == 0) { + dPrint("dnodeId is set to %d", dnodeId); + tsDnodeId = dnodeId; + dnodeSaveDnodeInfo(); + } +} + +int32_t dnodeGetDnodeId() { + return tsDnodeId; +} \ No newline at end of file diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index f4cbd3e1be..abfee2239b 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -21,8 +21,6 @@ #include "tlog.h" #include "trpc.h" #include "tsdb.h" -#include "ttime.h" -#include "ttimer.h" #include "twal.h" #include "dnodeMClient.h" #include "dnodeMgmt.h" @@ -38,52 +36,23 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); -static void dnodeSendStatusMsg(void *handle, void *tmrId); -static void dnodeReadDnodeId(); - -static void *tsDnodeTmr = NULL; -static void *tsStatusTimer = NULL; -static uint32_t tsRebootTime; -static int32_t tsDnodeId = 0; -static char tsDnodeName[TSDB_NODE_NAME_LEN]; int32_t dnodeInitMgmt() { - dnodeReadDnodeId(); - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; - tsRebootTime = taosGetTimestampSec(); - - tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); - if (tsDnodeTmr == NULL) { - dError("failed to init dnode timer"); - return -1; - } - int32_t code = dnodeOpenVnodes(); if (code != TSDB_CODE_SUCCESS) { return -1; } - taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); return TSDB_CODE_SUCCESS; } void dnodeCleanupMgmt() { - if (tsStatusTimer != NULL) { - taosTmrStopA(&tsStatusTimer); - tsStatusTimer = NULL; - } - - if (tsDnodeTmr != NULL) { - taosTmrCleanUp(tsDnodeTmr); - tsDnodeTmr = NULL; - } - dnodeCloseVnodes(); } @@ -213,93 +182,3 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { SMDCfgDnodeMsg *pCfg = (SMDCfgDnodeMsg *)pMsg->pCont; return tsCfgDynamicOptions(pCfg->config); } - -static void dnodeSendStatusMsg(void *handle, void *tmrId) { - if (tsDnodeTmr == NULL) { - dError("dnode timer is already released"); - return; - } - - if (tsStatusTimer == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to start status timer"); - return; - } - - int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SDMStatusMsg *pStatus = rpcMallocCont(contLen); - if (pStatus == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to malloc status message"); - return; - } - - strcpy(pStatus->dnodeName, tsDnodeName); - pStatus->version = htonl(tsVersion); - pStatus->dnodeId = htonl(tsDnodeId); - pStatus->privateIp = htonl(inet_addr(tsPrivateIp)); - pStatus->publicIp = htonl(inet_addr(tsPublicIp)); - pStatus->lastReboot = htonl(tsRebootTime); - pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes); - pStatus->numOfCores = htons((uint16_t) tsNumOfCores); - pStatus->diskAvailable = tsAvailDataDirGB; - pStatus->alternativeRole = (uint8_t) tsAlternativeRole; - - vnodeBuildStatusMsg(pStatus); - contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); - pStatus->openVnodes = htons(pStatus->openVnodes); - - SRpcMsg rpcMsg = { - .pCont = pStatus, - .contLen = contLen, - .msgType = TSDB_MSG_TYPE_DM_STATUS - }; - - dnodeSendMsgToMnode(&rpcMsg); - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); -} - -static void dnodeReadDnodeId() { - char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; - sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); - - FILE *fp = fopen(dnodeIdFile, "r"); - if (!fp) return; - - char option[32] = {0}; - int32_t value = 0; - int32_t num = 0; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return; - if (strcmp(option, "dnodeId") != 0) return; - tsDnodeId = value;; - - fclose(fp); - dPrint("read dnodeId:%d successed", tsDnodeId); -} - -static void dnodeSaveDnodeId() { - char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; - sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); - - FILE *fp = fopen(dnodeIdFile, "w"); - if (!fp) return; - - fprintf(fp, "dnodeId %d\n", tsDnodeId); - - fclose(fp); - dPrint("save dnodeId successed"); -} - -void dnodeUpdateDnodeId(int32_t dnodeId) { - if (tsDnodeId == 0) { - dPrint("dnodeId is set to %d", dnodeId); - tsDnodeId = dnodeId; - dnodeSaveDnodeId(); - } -} - -int32_t dnodeGetDnodeId() { - return tsDnodeId; -} \ No newline at end of file diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h index 769a819b90..a56285fe1c 100644 --- a/src/inc/tcluster.h +++ b/src/inc/tcluster.h @@ -37,6 +37,7 @@ int32_t clusterInit(); void clusterCleanUp(); char* clusterGetDnodeStatusStr(int32_t dnodeStatus); bool clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType); +void clusterMonitorDnodeModule(); int32_t clusterInitDnodes(); void clusterCleanupDnodes(); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 2804d40a71..8ecb5ef152 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -21,8 +21,8 @@ extern "C" { #endif typedef enum { - SDB_TABLE_MNODE = 0, - SDB_TABLE_DNODE = 1, + SDB_TABLE_DNODE = 0, + SDB_TABLE_MNODE = 1, SDB_TABLE_ACCOUNT = 2, SDB_TABLE_USER = 3, SDB_TABLE_DB = 4, @@ -90,7 +90,7 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow); void sdbIncRef(void *thandle, void *pRow); void sdbDecRef(void *thandle, void *pRow); int64_t sdbGetNumOfRows(void *handle); -int64_t sdbGetId(void *handle); +int32_t sdbGetId(void *handle); uint64_t sdbGetVersion(); #ifdef __cplusplus diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 51ac4e842d..196b8c1026 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -208,6 +208,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mTrace("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TAOS_DN_STATUS_READY; balanceNotify(); + clusterMonitorDnodeModule(); } clusterReleaseDnode(pDnode); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 3038b0eadc..2cec20c653 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "tlog.h" #include "trpc.h" +#include "tqueue.h" #include "twal.h" #include "hashint.h" #include "hashstr.h" @@ -67,7 +68,7 @@ static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIn static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData}; uint64_t sdbGetVersion() { return tsSdbObj->version; } -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; } +int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } static char *sdbGetActionStr(int32_t action) { @@ -116,7 +117,7 @@ int32_t sdbInit() { int32_t totalRows = 0; int32_t numOfTables = 0; - for (int32_t tableId = SDB_TABLE_MNODE; tableId < SDB_TABLE_MAX; ++tableId) { + for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; if (pTable->restoredFp) { @@ -275,8 +276,7 @@ static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_ } walFsync(tsSdbObj->wal); - free(pHead); - + taosFreeQitem(pHead); return code; } @@ -390,7 +390,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; - SWalHead *pHead = calloc(1, size); + SWalHead *pHead = taosAllocateQitem(size); pHead->version = 0; pHead->len = pOper->rowSize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; @@ -435,7 +435,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { } int32_t size = sizeof(SWalHead) + rowSize; - SWalHead *pHead = calloc(1, size); + SWalHead *pHead = taosAllocateQitem(size); pHead->version = 0; pHead->len = rowSize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; @@ -463,7 +463,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; - SWalHead *pHead = calloc(1, size); + SWalHead *pHead = taosAllocateQitem(size); pHead->version = 0; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 4d77e007ad..7ec9b0fef7 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -160,7 +160,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.writeToCache = vnodeWriteToQueue; syncInfo.confirmForward = dnodeSendRpcWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; - pVnode->sync = syncStart(&syncInfo);; + pVnode->sync = syncStart(&syncInfo); pVnode->events = NULL; pVnode->cq = NULL; From dcbf5972941dcdbe6b54f9ec03faf54c0e847da0 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 15 Apr 2020 17:21:33 +0800 Subject: [PATCH 28/36] remove the bug in rpcClose process --- src/rpc/src/rpcMain.c | 12 ++++++++---- src/rpc/src/rpcTcp.c | 20 +++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 5048d5db14..75b23950ed 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -286,15 +286,15 @@ void *rpcOpen(const SRpcInit *pInit) { void rpcClose(void *param) { SRpcInfo *pRpc = (SRpcInfo *)param; - (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); - for (int i = 0; i < pRpc->sessions; ++i) { if (pRpc->connList && pRpc->connList[i].user[0]) { rpcCloseConn((void *)(pRpc->connList + i)); } } + (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); + (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); + taosHashCleanup(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); @@ -521,11 +521,15 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, static void rpcCloseConn(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; SRpcInfo *pRpc = pConn->pRpc; - if (pConn->user[0] == 0) return; rpcLockConn(pConn); + if (pConn->user[0] == 0) { + rpcUnlockConn(pConn); + return; + } + pConn->user[0] = 0; if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 27b81deda5..8e8b285621 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -158,8 +158,9 @@ void taosCleanUpTcpServer(void *handle) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { - taosFreeFdObj(pThreadObj->pHead); - pThreadObj->pHead = pThreadObj->pHead; + SFdObj *pFdObj = pThreadObj->pHead; + pThreadObj->pHead = pFdObj->next; + taosFreeFdObj(pFdObj); } close(pThreadObj->pollFd); @@ -269,8 +270,9 @@ void taosCleanUpTcpClient(void *chandle) { if (pThreadObj == NULL) return; while (pThreadObj->pHead) { - taosFreeFdObj(pThreadObj->pHead); - pThreadObj->pHead = pThreadObj->pHead->next; + SFdObj *pFdObj = pThreadObj->pHead; + pThreadObj->pHead = pFdObj->next; + taosFreeFdObj(pFdObj); } close(pThreadObj->pollFd); @@ -456,14 +458,18 @@ static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; - pFdObj->signature = NULL; SThreadObj *pThreadObj = pFdObj->pThreadObj; + pthread_mutex_lock(&pThreadObj->mutex); + if (pFdObj->signature == NULL) { + pthread_mutex_unlock(&pThreadObj->mutex); + return; + } + + pFdObj->signature = NULL; close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - pthread_mutex_lock(&pThreadObj->mutex); - pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) From a5d0e01ab445b3e3e83f3e7a7f181c41086b6c76 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 20:18:15 +0800 Subject: [PATCH 29/36] fix error in sdb --- src/dnode/src/dnodeMClient.c | 21 ++++++++++++--------- src/dnode/src/dnodeMain.c | 2 +- src/inc/mpeer.h | 1 + src/mnode/src/mgmtDnode.c | 1 + src/mnode/src/mgmtMnode.c | 1 + src/mnode/src/mgmtSdb.c | 8 ++++++++ src/mnode/src/mgmtUser.c | 12 +++++++----- src/plugins/http/src/httpSystem.c | 2 +- 8 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 42eca2152b..d791ad0f01 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -28,6 +28,7 @@ #include "dnodeModule.h" #include "dnodeMgmt.h" #include "vnode.h" +#include "mpeer.h" #define MPEER_CONTENT_LEN 2000 @@ -148,6 +149,15 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { return; } + SDnodeState *pState = &pStatusRsp->dnodeState; + pState->numOfVnodes = htonl(pState->numOfVnodes); + pState->moduleStatus = htonl(pState->moduleStatus); + pState->createdTime = htonl(pState->createdTime); + pState->dnodeId = htonl(pState->dnodeId); + + dnodeProcessModuleStatus(pState->moduleStatus); + dnodeUpdateDnodeInfo(pState->dnodeId); + SRpcIpSet mgmtIpSet = {0}; mgmtIpSet.inUse = mpeers->inUse; mgmtIpSet.numOfIps = mpeers->nodeNum; @@ -167,20 +177,13 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); strcpy(tsMnodeInfos.nodeInfos[i].nodeName, mpeers->nodeInfos[i].nodeName); dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, - taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, + taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort, tsMnodeInfos.nodeInfos[i].nodeName); } dnodeSaveMnodeIpList(); + mpeerUpdateSync(); } - SDnodeState *pState = &pStatusRsp->dnodeState; - pState->numOfVnodes = htonl(pState->numOfVnodes); - pState->moduleStatus = htonl(pState->moduleStatus); - pState->createdTime = htonl(pState->createdTime); - pState->dnodeId = htonl(pState->dnodeId); - - dnodeProcessModuleStatus(pState->moduleStatus); - dnodeUpdateDnodeInfo(pState->dnodeId); taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 01e2c4dfcc..64d1ad4048 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -159,10 +159,10 @@ static int32_t dnodeInitSystem() { dPrint("starting to initialize TDengine ..."); if (dnodeInitStorage() != 0) return -1; - if (dnodeInitModules() != 0) return -1; if (dnodeInitRead() != 0) return -1; if (dnodeInitWrite() != 0) return -1; if (dnodeInitMClient() != 0) return -1; + if (dnodeInitModules() != 0) return -1; if (dnodeInitMnode() != 0) return -1; if (dnodeInitMgmt() != 0) return -1; if (dnodeInitShell() != 0) return -1; diff --git a/src/inc/mpeer.h b/src/inc/mpeer.h index e7abf09321..ba1b7d32cf 100644 --- a/src/inc/mpeer.h +++ b/src/inc/mpeer.h @@ -50,6 +50,7 @@ void mpeerGetPublicIpList(SRpcIpSet *ipSet); void mpeerGetMpeerInfos(void *mpeers); int32_t mpeerForwardReqToPeer(void *pHead); +void mpeerUpdateSync(); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 196b8c1026..d13d37586a 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -77,6 +77,7 @@ void * clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : N void * clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; } void clusterReleaseDnode(struct _dnode_obj *pDnode) {} void clusterUpdateDnode(struct _dnode_obj *pDnode) {} +void clusterMonitorDnodeModule() {} #endif diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index ca18d6bdba..e2edb201b9 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -48,6 +48,7 @@ void * mpeerGetMnode(int32_t mnodeId) { return &tsMnodeObj; } int32_t mpeerGetMnodesNum() { return 1; } void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; } +void mpeerUpdateSync() {} void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) { if (*pMnode == NULL) { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 2cec20c653..6bc3c20d18 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -130,6 +130,9 @@ int32_t sdbInit() { } sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables); + + mpeerUpdateSync(); + return TSDB_CODE_SUCCESS; } @@ -215,6 +218,11 @@ static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); sdbIncRef(pTable, pOper->pObj); pTable->numOfRows++; + + if (pTable->keyType == SDB_KEY_AUTO) { + pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj)); + } + pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index ef01faf6ba..7fa1a13bfd 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -85,11 +85,13 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { } static int32_t mgmtUserActionRestored() { - SAcctObj *pAcct = acctGetAcct("root"); - mgmtCreateUser(pAcct, "root", "taosdata"); - mgmtCreateUser(pAcct, "monitor", tsInternalPass); - mgmtCreateUser(pAcct, "_root", tsInternalPass); - acctReleaseAcct(pAcct); + if (strcmp(tsMasterIp, tsPrivateIp) == 0) { + SAcctObj *pAcct = acctGetAcct("root"); + mgmtCreateUser(pAcct, "root", "taosdata"); + mgmtCreateUser(pAcct, "monitor", tsInternalPass); + mgmtCreateUser(pAcct, "_root", tsInternalPass); + acctReleaseAcct(pAcct); + } return 0; } diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 46f31a12d6..deb0d877c7 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -129,7 +129,7 @@ void httpCleanUpSystem() { httpPrint("http service cleanup"); httpStopSystem(); -#if 1 +#if 0 if (httpServer == NULL) { return; } From 98286acce4f227f4bc3823ed2d02f89d809b4213 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 21:34:13 +0800 Subject: [PATCH 30/36] fix possible memory lost --- src/dnode/src/dnodeMClient.c | 2 +- src/dnode/src/dnodeMain.c | 1 - src/mnode/src/mgmtSdb.c | 3 ++- src/plugins/http/src/httpServer.c | 13 ++++++++----- src/plugins/http/src/httpSystem.c | 4 ++-- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index d791ad0f01..85454af095 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -59,7 +59,7 @@ int32_t dnodeInitMClient() { dError("failed to init dnode timer"); return -1; } - + if (!dnodeReadMnodeIpList()) { memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet)); memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos)); diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 64d1ad4048..5fba941788 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -177,7 +177,6 @@ static int32_t dnodeInitSystem() { static void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) { - tclearModuleStatus(TSDB_MOD_MGMT); dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED); dnodeCleanupShell(); dnodeCleanupMnode(); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 6bc3c20d18..70e2be5f4c 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -504,6 +504,7 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { void *sdbOpenTable(SSdbTableDesc *pDesc) { SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; strcpy(pTable->tableName, pDesc->tableName); @@ -557,7 +558,7 @@ void sdbCloseTable(void *handle) { } pthread_mutex_destroy(&pTable->mutex); - + sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables); free(pTable); } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 171f811b7d..5c321beffb 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -270,7 +270,7 @@ void httpCleanUpConnect(HttpServer *pServer) { for (i = 0; i < pServer->numOfThreads; ++i) { pThread = pServer->pThreads + i; - taosCloseSocket(pThread->pollFd); + //taosCloseSocket(pThread->pollFd); while (pThread->pHead) { httpCleanUpContext(pThread->pHead, 0); @@ -591,7 +591,6 @@ void httpAcceptHttpConnection(void *arg) { bool httpInitConnect(HttpServer *pServer) { int i; - pthread_attr_t thattr; HttpThread * pThread; pServer->pThreads = (HttpThread *)malloc(sizeof(HttpThread) * (size_t)pServer->numOfThreads); @@ -601,8 +600,6 @@ bool httpInitConnect(HttpServer *pServer) { } memset(pServer->pThreads, 0, sizeof(HttpThread) * (size_t)pServer->numOfThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pThread = pServer->pThreads; for (i = 0; i < pServer->numOfThreads; ++i) { sprintf(pThread->label, "%s%d", pServer->label, i); @@ -626,21 +623,27 @@ bool httpInitConnect(HttpServer *pServer) { return false; } + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, strerror(errno)); return false; } + pthread_attr_destroy(&thattr); httpTrace("http thread:%p:%s, initialized", pThread, pThread->label); pThread++; } + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pServer->thread), &thattr, (void *)httpAcceptHttpConnection, (void *)(pServer)) != 0) { httpError("http server:%s, failed to create Http accept thread, reason:%s", pServer->label, strerror(errno)); return false; } - pthread_attr_destroy(&thattr); httpTrace("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, pServer->serverIp, diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index deb0d877c7..2a118cc2b1 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -54,7 +54,7 @@ static HttpServer *httpServer = NULL; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); int httpInitSystem() { - taos_init(); + // taos_init(); httpServer = (HttpServer *)malloc(sizeof(HttpServer)); memset(httpServer, 0, sizeof(HttpServer)); @@ -129,7 +129,7 @@ void httpCleanUpSystem() { httpPrint("http service cleanup"); httpStopSystem(); -#if 0 +#if 1 if (httpServer == NULL) { return; } From 1c9d23b061c8d3c2c4b48fe983695d4039963257 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 22:22:12 +0800 Subject: [PATCH 31/36] Rearrange the source code directory --- deps/CMakeLists.txt | 2 ++ deps/cJson/CMakeLists.txt | 4 ++++ {src/util => deps/cJson}/inc/cJSON.h | 0 {src/util => deps/cJson}/src/cJSON.c | 0 deps/lz4/CMakeLists.txt | 4 ++++ {src/thirdparty => deps/lz4}/inc/lz4.h | 0 {src/thirdparty => deps/lz4}/src/lz4.c | 0 src/CMakeLists.txt | 4 ++-- src/dnode/CMakeLists.txt | 8 +++++--- src/plugins/http/CMakeLists.txt | 2 ++ src/query/CMakeLists.txt | 2 +- src/rpc/CMakeLists.txt | 4 +++- src/thirdparty/CMakeLists.txt | 4 ---- src/{vnode => }/tsdb/CMakeLists.txt | 0 src/{vnode => }/tsdb/inc/tsdb.h | 0 src/{vnode => }/tsdb/inc/tsdbMain.h | 0 src/{vnode => }/tsdb/src/tsdbCache.c | 0 src/{vnode => }/tsdb/src/tsdbCompactor.c | 0 src/{vnode => }/tsdb/src/tsdbFile.c | 0 src/{vnode => }/tsdb/src/tsdbMain.c | 0 src/{vnode => }/tsdb/src/tsdbMeta.c | 0 src/{vnode => }/tsdb/src/tsdbMetaFile.c | 0 src/{vnode => }/tsdb/src/tsdbRead.c | 0 src/{vnode => }/tsdb/tests/CMakeLists.txt | 0 src/{vnode => }/tsdb/tests/tsdbTests.cpp | 0 src/util/CMakeLists.txt | 8 +++++--- src/vnode/CMakeLists.txt | 18 +++++++++++++++--- src/vnode/cache/inc/cache.h | 17 ----------------- src/vnode/{main => }/inc/vnodeInt.h | 0 src/vnode/main/CMakeLists.txt | 18 ------------------ src/vnode/{main => }/src/vnodeMain.c | 0 src/vnode/{main => }/src/vnodeRead.c | 0 src/vnode/{main => }/src/vnodeWrite.c | 0 src/{vnode => }/wal/CMakeLists.txt | 0 src/{vnode => }/wal/src/walMain.c | 0 src/{vnode => }/wal/test/CMakeLists.txt | 0 src/{vnode => }/wal/test/waltest.c | 0 tests/tsim/CMakeLists.txt | 3 ++- 38 files changed, 45 insertions(+), 53 deletions(-) create mode 100644 deps/cJson/CMakeLists.txt rename {src/util => deps/cJson}/inc/cJSON.h (100%) rename {src/util => deps/cJson}/src/cJSON.c (100%) create mode 100644 deps/lz4/CMakeLists.txt rename {src/thirdparty => deps/lz4}/inc/lz4.h (100%) rename {src/thirdparty => deps/lz4}/src/lz4.c (100%) delete mode 100644 src/thirdparty/CMakeLists.txt rename src/{vnode => }/tsdb/CMakeLists.txt (100%) rename src/{vnode => }/tsdb/inc/tsdb.h (100%) rename src/{vnode => }/tsdb/inc/tsdbMain.h (100%) rename src/{vnode => }/tsdb/src/tsdbCache.c (100%) rename src/{vnode => }/tsdb/src/tsdbCompactor.c (100%) rename src/{vnode => }/tsdb/src/tsdbFile.c (100%) rename src/{vnode => }/tsdb/src/tsdbMain.c (100%) rename src/{vnode => }/tsdb/src/tsdbMeta.c (100%) rename src/{vnode => }/tsdb/src/tsdbMetaFile.c (100%) rename src/{vnode => }/tsdb/src/tsdbRead.c (100%) rename src/{vnode => }/tsdb/tests/CMakeLists.txt (100%) rename src/{vnode => }/tsdb/tests/tsdbTests.cpp (100%) delete mode 100644 src/vnode/cache/inc/cache.h rename src/vnode/{main => }/inc/vnodeInt.h (100%) delete mode 100644 src/vnode/main/CMakeLists.txt rename src/vnode/{main => }/src/vnodeMain.c (100%) rename src/vnode/{main => }/src/vnodeRead.c (100%) rename src/vnode/{main => }/src/vnodeWrite.c (100%) rename src/{vnode => }/wal/CMakeLists.txt (100%) rename src/{vnode => }/wal/src/walMain.c (100%) rename src/{vnode => }/wal/test/CMakeLists.txt (100%) rename src/{vnode => }/wal/test/waltest.c (100%) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index bc5bd8c037..5cdf84658b 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -5,3 +5,5 @@ ADD_SUBDIRECTORY(zlib-1.2.11) ADD_SUBDIRECTORY(pthread) ADD_SUBDIRECTORY(regex) ADD_SUBDIRECTORY(iconv) +ADD_SUBDIRECTORY(lz4) +ADD_SUBDIRECTORY(cJson) diff --git a/deps/cJson/CMakeLists.txt b/deps/cJson/CMakeLists.txt new file mode 100644 index 0000000000..43b31589b3 --- /dev/null +++ b/deps/cJson/CMakeLists.txt @@ -0,0 +1,4 @@ +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) + +add_library(cJson ${SOURCE_LIST}) +target_include_directories(cJson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) \ No newline at end of file diff --git a/src/util/inc/cJSON.h b/deps/cJson/inc/cJSON.h similarity index 100% rename from src/util/inc/cJSON.h rename to deps/cJson/inc/cJSON.h diff --git a/src/util/src/cJSON.c b/deps/cJson/src/cJSON.c similarity index 100% rename from src/util/src/cJSON.c rename to deps/cJson/src/cJSON.c diff --git a/deps/lz4/CMakeLists.txt b/deps/lz4/CMakeLists.txt new file mode 100644 index 0000000000..a142d8d468 --- /dev/null +++ b/deps/lz4/CMakeLists.txt @@ -0,0 +1,4 @@ +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) + +add_library(lz4 ${SOURCE_LIST}) +target_include_directories(lz4 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) \ No newline at end of file diff --git a/src/thirdparty/inc/lz4.h b/deps/lz4/inc/lz4.h similarity index 100% rename from src/thirdparty/inc/lz4.h rename to deps/lz4/inc/lz4.h diff --git a/src/thirdparty/src/lz4.c b/deps/lz4/src/lz4.c similarity index 100% rename from src/thirdparty/src/lz4.c rename to deps/lz4/src/lz4.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 326f00dbbd..1a8f7d8807 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,8 +3,6 @@ PROJECT(TDengine) # Base compile ADD_SUBDIRECTORY(os) -ADD_SUBDIRECTORY(thirdparty) - ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(util) ADD_SUBDIRECTORY(rpc) @@ -14,5 +12,7 @@ ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) +ADD_SUBDIRECTORY(tsdb) +ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index ee05403a61..8999770618 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -7,14 +7,16 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4) IF (TD_ACCOUNT) TARGET_LINK_LIBRARIES(taosd account) @@ -31,7 +33,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) TARGET_LINK_LIBRARIES(taosd balance sync) ENDIF () - IF (TD_MPEER) + IF (TD_MPEER) TARGET_LINK_LIBRARIES(taosd mpeer sync) ENDIF () diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 57ca4ca9f0..9b36f4029a 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -7,6 +7,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index 1bbd067804..01be5e643b 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index e8768c10dd..229b1077f8 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -4,6 +4,8 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) INCLUDE_DIRECTORIES(inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) @@ -22,7 +24,7 @@ ELSEIF (TD_WINDOWS_64) ENDIF () ADD_LIBRARY(trpc ${SRC}) -TARGET_LINK_LIBRARIES(trpc tutil) +TARGET_LINK_LIBRARIES(trpc tutil lz4) ADD_SUBDIRECTORY(test) diff --git a/src/thirdparty/CMakeLists.txt b/src/thirdparty/CMakeLists.txt deleted file mode 100644 index 9958e1e28c..0000000000 --- a/src/thirdparty/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) - -add_library(thirdparty ${SOURCE_LIST}) -target_include_directories(thirdparty PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) \ No newline at end of file diff --git a/src/vnode/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt similarity index 100% rename from src/vnode/tsdb/CMakeLists.txt rename to src/tsdb/CMakeLists.txt diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/tsdb/inc/tsdb.h similarity index 100% rename from src/vnode/tsdb/inc/tsdb.h rename to src/tsdb/inc/tsdb.h diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h similarity index 100% rename from src/vnode/tsdb/inc/tsdbMain.h rename to src/tsdb/inc/tsdbMain.h diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c similarity index 100% rename from src/vnode/tsdb/src/tsdbCache.c rename to src/tsdb/src/tsdbCache.c diff --git a/src/vnode/tsdb/src/tsdbCompactor.c b/src/tsdb/src/tsdbCompactor.c similarity index 100% rename from src/vnode/tsdb/src/tsdbCompactor.c rename to src/tsdb/src/tsdbCompactor.c diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c similarity index 100% rename from src/vnode/tsdb/src/tsdbFile.c rename to src/tsdb/src/tsdbFile.c diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c similarity index 100% rename from src/vnode/tsdb/src/tsdbMain.c rename to src/tsdb/src/tsdbMain.c diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c similarity index 100% rename from src/vnode/tsdb/src/tsdbMeta.c rename to src/tsdb/src/tsdbMeta.c diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c similarity index 100% rename from src/vnode/tsdb/src/tsdbMetaFile.c rename to src/tsdb/src/tsdbMetaFile.c diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c similarity index 100% rename from src/vnode/tsdb/src/tsdbRead.c rename to src/tsdb/src/tsdbRead.c diff --git a/src/vnode/tsdb/tests/CMakeLists.txt b/src/tsdb/tests/CMakeLists.txt similarity index 100% rename from src/vnode/tsdb/tests/CMakeLists.txt rename to src/tsdb/tests/CMakeLists.txt diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp similarity index 100% rename from src/vnode/tsdb/tests/tsdbTests.cpp rename to src/tsdb/tests/tsdbTests.cpp diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 6c8f94ec39..a814bf52ab 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -6,9 +6,11 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) - TARGET_LINK_LIBRARIES(tutil thirdparty pthread os m rt) + TARGET_LINK_LIBRARIES(tutil pthread os m rt) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) ADD_DEFINITIONS(-DUSE_LIBICONV) @@ -65,7 +67,7 @@ ELSEIF (TD_WINDOWS_64) LIST(APPEND SRC ./src/tutil.c) LIST(APPEND SRC ./src/version.c) ADD_LIBRARY(tutil ${SRC}) - TARGET_LINK_LIBRARIES(tutil thirdparty iconv regex pthread os winmm IPHLPAPI ws2_32) + TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32) ELSEIF(TD_DARWIN_64) ADD_DEFINITIONS(-DUSE_LIBICONV) LIST(APPEND SRC ./src/hash.c) @@ -102,7 +104,7 @@ ELSEIF(TD_DARWIN_64) LIST(APPEND SRC ./src/version.c) LIST(APPEND SRC ./src/hash.c) ADD_LIBRARY(tutil ${SRC}) - TARGET_LINK_LIBRARIES(tutil thirdparty iconv pthread os) + TARGET_LINK_LIBRARIES(tutil iconv pthread os) ENDIF() #IF (TD_CLUSTER) diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index 3df980ece1..51065b8645 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,6 +1,18 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -ADD_SUBDIRECTORY(wal) -ADD_SUBDIRECTORY(tsdb) -ADD_SUBDIRECTORY(main) +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) + INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) + INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) + INCLUDE_DIRECTORIES(inc) + AUX_SOURCE_DIRECTORY(src SRC) + + ADD_LIBRARY(vnode ${SRC}) + TARGET_LINK_LIBRARIES(vnode tsdb) +ENDIF () \ No newline at end of file diff --git a/src/vnode/cache/inc/cache.h b/src/vnode/cache/inc/cache.h deleted file mode 100644 index 52f357ee8f..0000000000 --- a/src/vnode/cache/inc/cache.h +++ /dev/null @@ -1,17 +0,0 @@ -#if !defined(_TD_CACHE_H_) -#define _TD_CACHE_H_ - -#define TD_MIN_CACHE_BLOCK_SIZE 1024*1024 /* 1M */ -#define TD_MAX_CACHE_BLOCK_SIZE 64*1024*1024 /* 64M */ - -typedef void cache_pool_t; - -typedef struct SCacheBlock -{ - int32_t blockId; - char data[]; -} SCacheBlock; - - - -#endif // _TD_CACHE_H_ diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h similarity index 100% rename from src/vnode/main/inc/vnodeInt.h rename to src/vnode/inc/vnodeInt.h diff --git a/src/vnode/main/CMakeLists.txt b/src/vnode/main/CMakeLists.txt deleted file mode 100644 index 19b1b4c22a..0000000000 --- a/src/vnode/main/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb) -ENDIF () \ No newline at end of file diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/src/vnodeMain.c similarity index 100% rename from src/vnode/main/src/vnodeMain.c rename to src/vnode/src/vnodeMain.c diff --git a/src/vnode/main/src/vnodeRead.c b/src/vnode/src/vnodeRead.c similarity index 100% rename from src/vnode/main/src/vnodeRead.c rename to src/vnode/src/vnodeRead.c diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c similarity index 100% rename from src/vnode/main/src/vnodeWrite.c rename to src/vnode/src/vnodeWrite.c diff --git a/src/vnode/wal/CMakeLists.txt b/src/wal/CMakeLists.txt similarity index 100% rename from src/vnode/wal/CMakeLists.txt rename to src/wal/CMakeLists.txt diff --git a/src/vnode/wal/src/walMain.c b/src/wal/src/walMain.c similarity index 100% rename from src/vnode/wal/src/walMain.c rename to src/wal/src/walMain.c diff --git a/src/vnode/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt similarity index 100% rename from src/vnode/wal/test/CMakeLists.txt rename to src/wal/test/CMakeLists.txt diff --git a/src/vnode/wal/test/waltest.c b/src/wal/test/waltest.c similarity index 100% rename from src/vnode/wal/test/waltest.c rename to src/wal/test/waltest.c diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt index 2fec751cec..2eaf3fe7cf 100644 --- a/tests/tsim/CMakeLists.txt +++ b/tests/tsim/CMakeLists.txt @@ -3,6 +3,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(inc) @@ -12,4 +13,4 @@ ENDIF () AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(tsim ${SRC}) -TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread ) +TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson) From 85420cf37d61c1f93ec738a3364bf8427d141b81 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 23:08:06 +0800 Subject: [PATCH 32/36] add test program --- tests/test/c/CMakeLists.txt | 7 +- tests/test/c/insertPerRow.c | 314 ++++++++++++++++++ .../{benchmarkPerTable.c => insertPerTable.c} | 0 3 files changed, 319 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/insertPerRow.c rename tests/test/c/{benchmarkPerTable.c => insertPerTable.c} (100%) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 532f304b99..9f34d6b774 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -6,6 +6,9 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - add_executable(benchmarkPerTable benchmarkPerTable.c) - target_link_libraries(benchmarkPerTable taos_static pthread) + add_executable(insertPerTable insertPerTable.c) + target_link_libraries(insertPerTable taos_static pthread) + + add_executable(insertPerRow insertPerRow.c) + target_link_libraries(insertPerRow taos_static pthread) ENDIF() diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c new file mode 100644 index 0000000000..eea69e6462 --- /dev/null +++ b/tests/test/c/insertPerRow.c @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tlog.h" +#include "ttimer.h" +#include "tutil.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +typedef struct { + int64_t rowsPerTable; + int64_t pointsPerTable; + int64_t tableBeginIndex; + int64_t tableEndIndex; + int threadIndex; + char dbName[32]; + char stableName[64]; + pthread_t thread; +} SInfo; + +void *syncTest(void *param); +void generateRandomPoints(); +void shellParseArgument(int argc, char *argv[]); +void createDbAndTable(); +void insertData(); + +int32_t randomData[MAX_RANDOM_POINTS]; +int64_t rowsPerTable = 10000; +int64_t pointsPerTable = 1; +int64_t numOfThreads = 1; +int64_t numOfTablesPerThread = 1; +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t cache = 16384; +int32_t tables = 1000; + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + generateRandomPoints(); + taos_init(); + createDbAndTable(); + insertData(); +} + +void createDbAndTable() { + dPrint("start to create table"); + + TAOS * con; + struct timeval systemTime; + int64_t st, et; + char qstr[64000]; + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); + if (taos_query(con, qstr)) { + dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); + exit(0); + } + + sprintf(qstr, "use %s", dbName); + if (taos_query(con, qstr)) { + dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = numOfTablesPerThread * numOfThreads; + + if (strcmp(stableName, "no") != 0) { + int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ") tags(t int)"); + + if (taos_query(con, qstr)) { + dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + for (int64_t t = 0; t < totalTables; ++t) { + sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); + if (taos_query(con, qstr)) { + dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } else { + for (int64_t t = 0; t < totalTables; ++t) { + int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ")"); + + if (taos_query(con, qstr)) { + dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables); +} + +void insertData() { + struct timeval systemTime; + int64_t st, et; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + dPrint("%d threads are spawned to insert data", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + + // Start threads to write + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].rowsPerTable = rowsPerTable; + pInfo[i].pointsPerTable = pointsPerTable; + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + double seconds = (et - st) / 1000.0 / 1000.0; + + int64_t totalTables = numOfTablesPerThread * numOfThreads; + int64_t totalRows = totalTables * rowsPerTable; + int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable; + double speedOfRows = totalRows / seconds; + double speedOfPoints = totalPoints / seconds; + + dPrint( + "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf " + "PointsPerSecond:%.1lf%s", + GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC); + + dPrint("threads exit"); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void *syncTest(void *param) { + TAOS * con; + SInfo * pInfo = (SInfo *)param; + struct timeval systemTime; + int64_t st, et; + char qstr[65000]; + int maxBytes = 60000; + + dPrint("thread:%d, start to run", pInfo->threadIndex); + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + taos_query(con, qstr); + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + int64_t start = 1430000000000; + int64_t interval = 1000; // 1000 ms + + char *sql = qstr; + char inserStr[] = "insert into"; + int len = sprintf(sql, "%s", inserStr); + + for (int64_t row = 0; row < pInfo->rowsPerTable; row++) { + for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) { + len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table); + len += sprintf(sql + len, "(%ld", start + row * interval); + for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) { + len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]); + // len += sprintf(sql + len, ",%ld", row); + } + len += sprintf(sql + len, ")"); + if (len > maxBytes) { + // if (taos_query(con, qstr)) { + // dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, + // table, row, taos_errstr(con)); + // } + dPrint("%s ", qstr); + // "insert into" + len = sprintf(sql, "%s", inserStr); + } + } + } + + if (len != strlen(inserStr)) { + taos_query(con, qstr); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + int64_t totalRows = totalTables * pInfo->rowsPerTable; + int64_t totalPoints = totalRows * pInfo->pointsPerTable; + dPrint("thread:%d, insert finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex, + (et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints); + + return NULL; +} + +void generateRandomPoints() { + for (int r = 0; r < MAX_RANDOM_POINTS; ++r) { + randomData[r] = rand() % 1000; + } +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of TDengine, the insert method is table-by-table\n"); + + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table"); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread); + printf("%s%s\n", indent, "-tables"); + printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables); + printf("%s%s\n", indent, "-cache"); + printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + rowsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + pointsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTablesPerThread = atoi(argv[++i]); + } else if (strcmp(argv[i], "-tables") == 0) { + tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-cache") == 0) { + cache = atoi(argv[++i]); + } else { + } + } + + dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC); + dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC); + dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); + dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC); + dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC); + dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC); + dPrint("%sdbName:%s%s", GREEN, dbName, NC); + dPrint("%stableName:%s%s", GREEN, stableName, NC); + dPrint("%sstart to run%s", GREEN, NC); +} diff --git a/tests/test/c/benchmarkPerTable.c b/tests/test/c/insertPerTable.c similarity index 100% rename from tests/test/c/benchmarkPerTable.c rename to tests/test/c/insertPerTable.c From 25444ef7bc20aa43a62af64b1af23ad810badd25 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 23:12:27 +0800 Subject: [PATCH 33/36] add test program --- tests/test/c/insertPerRow.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index eea69e6462..3c9564bcd2 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -45,7 +45,7 @@ int32_t randomData[MAX_RANDOM_POINTS]; int64_t rowsPerTable = 10000; int64_t pointsPerTable = 1; int64_t numOfThreads = 1; -int64_t numOfTablesPerThread = 1; +int64_t numOfTablesPerThread = 200; char dbName[32] = "db"; char stableName[64] = "st"; int32_t cache = 16384; @@ -219,11 +219,11 @@ void *syncTest(void *param) { } len += sprintf(sql + len, ")"); if (len > maxBytes) { - // if (taos_query(con, qstr)) { - // dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, - // table, row, taos_errstr(con)); - // } - dPrint("%s ", qstr); + if (taos_query(con, qstr)) { + dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, + table, row, taos_errstr(con)); + } + // "insert into" len = sprintf(sql, "%s", inserStr); } From 6b94643111e14f08dac759d785fef193bea8ff73 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 23:18:14 +0800 Subject: [PATCH 34/36] add test program --- tests/test/c/insertPerRow.c | 4 +++- tests/test/c/insertPerTable.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index 3c9564bcd2..74ab238c3f 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -253,7 +253,7 @@ void generateRandomPoints() { void printHelp() { char indent[10] = " "; - printf("Used to test the performance of TDengine, the insert method is table-by-table\n"); + printf("Used to test the performance of TDengine\n After writing one row of data to all tables, write the next row\n"); printf("%s%s\n", indent, "-d"); printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); @@ -284,6 +284,8 @@ void shellParseArgument(int argc, char *argv[]) { exit(0); } else if (strcmp(argv[i], "-d") == 0) { strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-s") == 0) { strcpy(stableName, argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { diff --git a/tests/test/c/insertPerTable.c b/tests/test/c/insertPerTable.c index ee4e0fecbd..ca63407463 100644 --- a/tests/test/c/insertPerTable.c +++ b/tests/test/c/insertPerTable.c @@ -258,7 +258,7 @@ void generateRandomPoints() { void printHelp() { char indent[10] = " "; - printf("Used to test the performance of TDengine, the insert method is table-by-table\n"); + printf("Used to test the performance of TDengine\n After writing all the data in one table, start the next table\n"); printf("%s%s\n", indent, "-d"); printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); @@ -289,6 +289,8 @@ void shellParseArgument(int argc, char *argv[]) { exit(0); } else if (strcmp(argv[i], "-d") == 0) { strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-s") == 0) { strcpy(stableName, argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { From 0765c20c3b55733913e6ff9eec05926a8ba9adec Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 23:31:50 +0800 Subject: [PATCH 35/36] add import test --- tests/test/c/CMakeLists.txt | 3 + tests/test/c/importOneRow.c | 119 ++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/test/c/importOneRow.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 9f34d6b774..8189dd4d09 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -11,4 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) add_executable(insertPerRow insertPerRow.c) target_link_libraries(insertPerRow taos_static pthread) + + add_executable(importOneRow importOneRow.c) + target_link_libraries(importOneRow taos_static pthread) ENDIF() diff --git a/tests/test/c/importOneRow.c b/tests/test/c/importOneRow.c new file mode 100644 index 0000000000..ee548a4df1 --- /dev/null +++ b/tests/test/c/importOneRow.c @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tlog.h" +#include "ttimer.h" +#include "tutil.h" + +void taos_error(TAOS *taos); +void* taos_execute(void *param); + +typedef struct { + pthread_t pid; + int index; +} ThreadObj; + +int threadNum = 1; +int rowNum = 1000; +int replica = 1; + +int main(int argc, char *argv[]) { + if (argc == 1) { + printf("usage: %s rowNum threadNum replica configDir\n", argv[0]); + printf("default rowNum %d\n", rowNum); + printf("default threadNum %d\n", threadNum); + printf("default replica %d\n", replica); + exit(0); + } + + // a simple way to parse input parameters + if (argc >= 2) rowNum = atoi(argv[1]); + if (argc >= 3) threadNum = atoi(argv[2]); + if (argc >= 4) replica = atoi(argv[3]); + if (argc >= 5) strcpy(configDir, argv[4]); + + printf("rowNum:%d threadNum:%d replica:%d\n", threadNum, rowNum, replica); + + taos_init(); + + ThreadObj *threads = calloc(threadNum, sizeof(ThreadObj)); + for (int i = 0; i < threadNum; ++i) { + ThreadObj * pthread = threads + i; + pthread_attr_t thattr; + pthread->index = i; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pthread_create(&pthread->pid, &thattr, taos_execute, pthread); + } + + for (int i = 0; i < threadNum; i++) { + pthread_join(threads[i].pid, NULL); + } + + printf("all finished\n"); + + return 0; +} + +void taos_error(TAOS *con) { + fprintf(stderr, "TDengine error: %s\n", taos_errstr(con)); + taos_close(con); + exit(1); +} + +void* taos_execute(void *param) { + ThreadObj *pThread = (ThreadObj *)param; + + void *taos = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (taos == NULL) taos_error(taos); + + char sql[1024] = {0}; + sprintf(sql, "create database if not exists db replica %d", replica); + taos_query(taos, sql); + + sprintf(sql, "create table if not exists db.t%d (ts timestamp, i int, j float, k double)", pThread->index); + taos_query(taos, sql); + + int64_t timestamp = 1530374400000L; + + sprintf(sql, "insert into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, 0, 0, 0); + int code = taos_query(taos, sql); + if (code != 0) printf("error code:%d, sql:%s\n", code, sql); + int affectrows = taos_affected_rows(taos); + if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql); + + timestamp -= 1000; + + int total_affect_rows = affectrows; + + for (int i = 1; i < rowNum; ++i) { + sprintf(sql, "import into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, i, i, i); + code = taos_query(taos, sql); + if (code != 0) printf("error code:%d, sql:%s\n", code, sql); + int affectrows = taos_affected_rows(taos); + if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql); + + total_affect_rows += affectrows; + + timestamp -= 1000; + } + + printf("thread:%d run finished total_affect_rows:%d\n", pThread->index, total_affect_rows); + + return NULL; +} From 13b44b30d5942ca904a97b57bd624a6a393fd54c Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 15 Apr 2020 23:39:48 +0800 Subject: [PATCH 36/36] add import test --- tests/test/c/importOneRow.c | 50 +++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/tests/test/c/importOneRow.c b/tests/test/c/importOneRow.c index ee548a4df1..9c0b39f7c8 100644 --- a/tests/test/c/importOneRow.c +++ b/tests/test/c/importOneRow.c @@ -20,6 +20,10 @@ #include "ttimer.h" #include "tutil.h" +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + void taos_error(TAOS *taos); void* taos_execute(void *param); @@ -32,22 +36,42 @@ int threadNum = 1; int rowNum = 1000; int replica = 1; -int main(int argc, char *argv[]) { - if (argc == 1) { - printf("usage: %s rowNum threadNum replica configDir\n", argv[0]); - printf("default rowNum %d\n", rowNum); - printf("default threadNum %d\n", threadNum); - printf("default replica %d\n", replica); - exit(0); +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of TDengine\n After writing one row of data to all tables, write the next row\n"); + + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "Number of records to write table, default is ", rowNum); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "Number of threads to be used, default is ", threadNum); + printf("%s%s\n", indent, "-replica"); + printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-r") == 0) { + rowNum = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + threadNum = atoi(argv[++i]); + } else if (strcmp(argv[i], "-replica") == 0) { + replica = atoi(argv[++i]); + } else { + } } - // a simple way to parse input parameters - if (argc >= 2) rowNum = atoi(argv[1]); - if (argc >= 3) threadNum = atoi(argv[2]); - if (argc >= 4) replica = atoi(argv[3]); - if (argc >= 5) strcpy(configDir, argv[4]); + dPrint("%s rowNum:%d %s", GREEN, rowNum, NC); + dPrint("%s threadNum:%d %s", GREEN, threadNum, NC); + dPrint("%s replica:%d %s", GREEN, replica, NC); +} - printf("rowNum:%d threadNum:%d replica:%d\n", threadNum, rowNum, replica); +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); taos_init();