xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9501 from taosdata/feature/dnode3 

add support vnodes config
This commit is contained in:
Shengliang Guan 2021-12-30 11:57:44 +08:00 committed by GitHub
parent 026f6afd36 68f8d6451e
commit f598d31b00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 193 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
include/common/tglobal.h
View File

@ -30,6 +30,7 @@ extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern int32_t tsStatusInterval;
extern int8_t tsEnableTelemetryReporting;
extern int32_t tsNumOfSupportVnodes;
// common
extern int tsRpcTimer;
@ -48,14 +49,18 @@ extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults;
extern char tsTempDir[];
extern int64_t tsMaxVnodeQueuedBytes;
extern int tsCompatibleModel; // 2.0 compatible model
extern int8_t tsEnableSlaveQuery;
extern int8_t tsEnableAdjustMaster;
extern int8_t tsPrintAuth;
extern int64_t tsTickPerDay[3];
// query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked
extern int8_t tsKeepOriginalColumnName;
extern int8_t tsDeadLockKillQuery;
// client
extern int32_t tsMaxSQLStringLen;
@ -72,16 +77,6 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the
extern int32_t tsProjectExecInterval;
extern int64_t tsMaxRetentWindow;
// balance
extern int8_t tsEnableSlaveQuery;
// interna
extern int8_t tsPrintAuth;
extern char tsVnodeDir[];
extern char tsMnodeDir[];
extern int64_t tsTickPerDay[3];
// system info
extern float tsTotalLogDirGB;
extern float tsTotalTmpDirGB;
@ -102,17 +97,13 @@ extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
#ifdef TD_TSZ
// lossy
extern char lossyColumns[];
extern double fPrecision;
extern double dPrecision;
extern uint32_t maxRange;
extern uint32_t curRange;
extern char Compressor[];
#endif
// long query
extern int8_t tsDeadLockKillQuery;
extern char tsLossyColumns[];
extern double tsFPrecision;
extern double tsDPrecision;
extern uint32_t tsMaxRange;
extern uint32_t tsCurRange;
extern char tsCompressor[];
typedef struct {
char dir[TSDB_FILENAME_LEN];

4
include/common/tmsg.h
View File

@ -694,8 +694,8 @@ typedef struct {
int64_t clusterId;
int64_t rebootTime;
int64_t updateTime;
int16_t numOfCores;
int16_t numOfSupportVnodes;
int32_t numOfCores;
int32_t numOfSupportVnodes;
char dnodeEp[TSDB_EP_LEN];
SClusterCfg clusterCfg;
SVnodeLoads vnodeLoads;

4
include/dnode/mgmt/dnode.h
View File

@ -27,8 +27,8 @@ typedef struct SDnode SDnode;
typedef struct {
int32_t sver;
int16_t numOfCores;
int16_t numOfSupportVnodes;
int32_t numOfCores;
int32_t numOfSupportVnodes;
int16_t numOfCommitThreads;
int8_t enableTelem;
int32_t statusInterval;

1
include/os/osEnv.h
View File

@ -21,7 +21,6 @@ extern "C" {
#endif
extern char tsOsName[];
extern char tsDnodeDir[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];

98
source/common/src/tglobal.c
View File

@ -15,17 +15,18 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "ulog.h"
#include "tlog.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tcompare.h"
#include "tutil.h"
#include "ttimezone.h"
#include "tlocale.h"
#include "tconfig.h"
#include "tep.h"
#include "tglobal.h"
#include "tlocale.h"
#include "tlog.h"
#include "ttimezone.h"
#include "tutil.h"
#include "ulog.h"
// cluster
char tsFirst[TSDB_EP_LEN] = {0};
@ -36,6 +37,7 @@ uint16_t tsServerPort = 6030;
int32_t tsStatusInterval = 1; // second
int8_t tsEnableTelemetryReporting = 0;
char tsEmail[TSDB_FQDN_LEN] = {0};
int32_t tsNumOfSupportVnodes = 16;
// common
int32_t tsRpcTimer = 300;
@ -50,8 +52,9 @@ float tsRatioOfQueryCores = 1.0f;
int8_t tsDaylight = 0;
int8_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB
int8_t tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1;
int8_t tsPrintAuth = 0;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
* metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@ -122,27 +125,13 @@ int32_t tsRetrieveBlockingModel = 0;
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
int8_t tsKeepOriginalColumnName = 0;
// long query death-lock
int8_t tsDeadLockKillQuery = 0;
// tsdb config
// For backward compatibility
bool tsdbForceKeepFile = false;
// balance
int8_t tsEnableFlowCtrl = 1;
int8_t tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1;
// monitor
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
// internal
int8_t tsCompactMnodeWal = 0;
int8_t tsPrintAuth = 0;
char tsVnodeDir[PATH_MAX] = {0};
char tsDnodeDir[PATH_MAX] = {0};
char tsMnodeDir[PATH_MAX] = {0};
int32_t tsDiskCfgNum = 0;
#ifndef _STORAGE
@ -170,21 +159,18 @@ float tsMinimalDataDirGB = 2.0f;
int32_t tsTotalMemoryMB = 0;
uint32_t tsVersion = 0;
#ifdef TD_TSZ
//
// lossy compress 6
//
char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress.
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
// can close lossy compress.
// below option can take effect when tsLossyColumns not empty
double fPrecision = 1E-8; // float column precision
double dPrecision = 1E-16; // double column precision
uint32_t maxRange = 500; // max range
uint32_t curRange = 100; // range
char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
#endif
double tsFPrecision = 1E-8; // float column precision
double tsDPrecision = 1E-16; // double column precision
uint32_t tsMaxRange = 500; // max range
uint32_t tsCurRange = 100; // range
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
// long query death-lock
int8_t tsDeadLockKillQuery = 0;
int32_t (*monStartSystemFp)() = NULL;
void (*monStopSystemFp)() = NULL;
@ -200,7 +186,6 @@ void taosSetAllDebugFlag() {
dDebugFlag = debugFlag;
vDebugFlag = debugFlag;
jniDebugFlag = debugFlag;
odbcDebugFlag = debugFlag;
qDebugFlag = debugFlag;
rpcDebugFlag = debugFlag;
uDebugFlag = debugFlag;
@ -218,7 +203,7 @@ int32_t taosCfgDynamicOptions(char *msg) {
int32_t vint = 0;
paGetToken(msg, &option, &olen);
if (olen == 0) return -1;;
if (olen == 0) return -1;
paGetToken(option + olen + 1, &value, &vlen);
if (vlen == 0)
@ -366,6 +351,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "supportVnodes";
cfg.ptr = &tsNumOfSupportVnodes;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// directory
cfg.option = "configDir";
cfg.ptr = configDir;
@ -749,17 +744,6 @@ static void doInitGlobalConfig(void) {
cfg.maxValue = 10000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_GB;
taosAddConfigOption(cfg);
// module configs
cfg.option = "flowctrl";
cfg.ptr = &tsEnableFlowCtrl;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "slaveQuery";
@ -893,16 +877,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "odbcDebugFlag";
cfg.ptr = &odbcDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "uDebugFlag";
cfg.ptr = &uDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1066,7 +1040,6 @@ static void doInitGlobalConfig(void) {
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "dPrecision";
@ -1102,12 +1075,9 @@ static void doInitGlobalConfig(void) {
#else
// assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5);
#endif
}
void taosInitGlobalCfg() {
pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
}
void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); }
int32_t taosCheckAndPrintCfg() {
char fqdn[TSDB_FQDN_LEN];

2
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -139,7 +139,7 @@ void dmnWaitSignal() {
void dmnInitOption(SDnodeOpt *pOption) {
pOption->sver = 30000000; //3.0.0.0
pOption->numOfCores = tsNumOfCores;
pOption->numOfSupportVnodes = 16;
pOption->numOfSupportVnodes = tsNumOfSupportVnodes;
pOption->numOfCommitThreads = 1;
pOption->statusInterval = tsStatusInterval;
pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore;

4
source/dnode/mgmt/impl/src/dndDnode.c
View File

@ -370,8 +370,8 @@ void dndSendStatusMsg(SDnode *pDnode) {
pStatus->clusterId = htobe64(pMgmt->clusterId);
pStatus->rebootTime = htobe64(pMgmt->rebootTime);
pStatus->updateTime = htobe64(pMgmt->updateTime);
pStatus->numOfCores = htons(pDnode->opt.numOfCores);
pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfSupportVnodes);
pStatus->numOfCores = htonl(pDnode->opt.numOfCores);
pStatus->numOfSupportVnodes = htonl(pDnode->opt.numOfSupportVnodes);
tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN);
pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval);

2
source/dnode/mgmt/impl/test/dnode/dnode.cpp
View File

@ -57,7 +57,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) {
CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id");
CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint");
CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes");
CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes");
CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "support_vnodes");
CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status");
CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time");
CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason");

1
source/dnode/mgmt/impl/test/sut/src/base.cpp
View File

@ -24,7 +24,6 @@ void Testbase::InitLog(const char* path) {
tmrDebugFlag = 0;
uDebugFlag = 143;
rpcDebugFlag = 0;
odbcDebugFlag = 0;
qDebugFlag = 0;
wDebugFlag = 0;
sDebugFlag = 0;

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -125,8 +125,8 @@ typedef struct {
int64_t lastAccessTime;
int32_t accessTimes;
int16_t numOfVnodes;
int16_t numOfSupportVnodes;
int16_t numOfCores;
int32_t numOfSupportVnodes;
int32_t numOfCores;
EDndStatus status;
EDndReason offlineReason;
uint16_t port;

6
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -278,8 +278,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) {
pStatus->clusterId = htobe64(pStatus->clusterId);
pStatus->rebootTime = htobe64(pStatus->rebootTime);
pStatus->updateTime = htobe64(pStatus->updateTime);
pStatus->numOfCores = htons(pStatus->numOfCores);
pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes);
pStatus->numOfCores = htonl(pStatus->numOfCores);
pStatus->numOfSupportVnodes = htonl(pStatus->numOfSupportVnodes);
pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval);
pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime);
}
@ -638,7 +638,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "max_vnodes");
strcpy(pSchema[cols].name, "support_vnodes");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;

4
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -265,7 +265,7 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) {
taosArrayPush(pArray, pDnode);
}
mDebug("dnode:%d, numOfVnodes:%d numOfSupportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes,
mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes,
pDnode->numOfSupportVnodes, isMnode, isReady);
sdbRelease(pSdb, pDnode);
}
@ -333,6 +333,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
uint32_t hashMax = UINT32_MAX;
uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups;
if (maxVgId < 2) maxVgId = 2;
for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) {
SVgObj *pVgroup = &pVgroups[v];
pVgroup->vgId = maxVgId++;

1
source/util/src/tlog.c
View File

@ -85,7 +85,6 @@ int32_t dDebugFlag = 135;
int32_t vDebugFlag = 135;
int32_t cDebugFlag = 131;
int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 131;
int32_t uDebugFlag = 131;

1
tests/script/sh/deploy.sh
View File

@ -120,6 +120,7 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG
echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG
echo "fqdn ${HOSTNAME}" >> $TAOS_CFG
echo "serverPort ${NODE}" >> $TAOS_CFG
echo "supportVnodes 16" >> $TAOS_CFG
echo "dataDir $DATA_DIR" >> $TAOS_CFG
echo "logDir $LOG_DIR" >> $TAOS_CFG
echo "debugFlag 0" >> $TAOS_CFG

38
tests/script/tmp/dnodes.sim
View File

@ -1,6 +1,3 @@
system sh/stop_dnodes.sh
############## config parameter #####################
$node1 = 192.168.0.201
$node2 = 192.168.0.202
@ -10,16 +7,36 @@ $node4 = 192.168.0.204
$self = $node1
$num = 25
############### deploy firstEp #####################
#deploy = 0, start = 1, stop = 2
$option = 0
print =============== option:$option
############### stop dnodes #####################
if $option == 0 then
system sh/stop_dnodes.sh
endi
############### process firstEp #####################
$firstEp = $node1 . :7100
$firstPort = 7100
if $self == $node1 then
if $option == 1 then
system sh/exec.sh -n dnode1 -s start
endi
if $option == 2 then
system sh/exec.sh -n dnode1 -s stop -x SIGINT
endi
if $option == 0 then
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp
system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp
system sh/cfg.sh -n dnode1 -c fqdn -v $node1
system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
sql connect
@ -56,8 +73,9 @@ if $self == $node1 then
sql create dnode $node4 port $port
endw
endi
endi
############### deploy nodes #####################
############### process nodes #####################
$i = 0
while $i < $num
@ -67,6 +85,15 @@ while $i < $num
$dnodename = dnode . $index
$i = $i + 1
if $option == 1 then
system sh/exec.sh -n $dnodename -s start
endi
if $option == 2 then
system sh/exec.sh -n $dnodename -s stop -x SIGINT
endi
if $option == 0 then
system sh/deploy.sh -n $dnodename -i 1
system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp
system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp
@ -74,4 +101,5 @@ while $i < $num
system sh/cfg.sh -n $dnodename -c serverPort -v $port
system sh/exec.sh -n $dnodename -s start
endi
endw