xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #2655 from taosdata/feature/vnode 

Feature/vnode
This commit is contained in:
Shengliang Guan 2020-07-13 18:30:11 +08:00 committed by GitHub
parent 708eb1671a 6e5cb647db
commit a4d710e69b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 99 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
packaging/cfg/taos.cfg
View File

@ -51,18 +51,15 @@
# number of threads per CPU core
# numOfThreadsPerCore 1.0
# number of vnodes per core in DNode
# numOfVnodesPerCore 8
# number of vgroups per db
# maxVgroupsPerDb 0
# max number of tables per vnode
# maxTablesPerVnode 1000000
# the ratio of threads responsible for querying in the total thread
# ratioOfQueryThreads 0.5
# number of total vnodes in DNode
# numOfTotalVnodes 0
# max number of tables per vnode
# maxtablesPerVnode 1000
# interval of check load balance when the management node is in normal operation
# balanceInterval 300

2
src/common/inc/tglobal.h
View File

@ -68,7 +68,9 @@ extern int64_t tsMaxRetentWindow;
// db parameters in client
extern int32_t tsCacheBlockSize;
extern int32_t tsBlocksPerVnode;
extern int32_t tsMinTablePerVnode;
extern int32_t tsMaxTablePerVnode;
extern int32_t tsTableIncStepPerVnode;
extern int32_t tsMaxVgroupsPerDb;
extern int16_t tsDaysPerFile;
extern int32_t tsDaysToKeep;

35
src/common/src/tglobal.c
View File

@ -38,7 +38,7 @@ uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035]
uint16_t tsDnodeDnodePort = 6035; // udp/tcp
uint16_t tsSyncPort = 6040;
int32_t tsStatusInterval = 1; // second
int16_t tsNumOfVnodesPerCore = 8;
int16_t tsNumOfVnodesPerCore = 32;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
int32_t tsNumOfMnodes = 3;
int32_t tsEnableVnodeBak = 1;
@ -112,7 +112,10 @@ int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = 100;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
// balance
int32_t tsEnableBalance = 1;
int32_t tsAlternativeRole = 0;
@ -389,16 +392,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "numOfVnodesPerCore";
cfg.ptr = &tsNumOfVnodesPerCore;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 64;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "numOfTotalVnodes";
cfg.ptr = &tsNumOfTotalVnodes;
cfg.valType = TAOS_CFG_VTYPE_INT16;
@ -622,6 +615,26 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "minTablesPerVnode";
cfg.ptr = &tsMinTablePerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TABLES;
cfg.maxValue = TSDB_MAX_TABLES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "tableIncStepPerVnode";
cfg.ptr = &tsTableIncStepPerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TABLES;
cfg.maxValue = TSDB_MAX_TABLES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "cache";
cfg.ptr = &tsCacheBlockSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;

10
src/inc/taosdef.h
View File

@ -274,8 +274,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value
#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MAX_VNODES 256
#define TSDB_MIN_VNODES 50
#define TSDB_MAX_VNODES 2048
#define TSDB_MIN_VNODES 256
#define TSDB_INVALID_VNODE_NUM 0
#define TSDB_DNODE_ROLE_ANY 0
@ -296,9 +296,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DEFAULT_TOTAL_BLOCKS 4
#define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 5000000
#define TSDB_DEFAULT_TABLES 200000
#define TSDB_TABLES_STEP 10000
#define TSDB_MAX_TABLES 10000000
#define TSDB_DEFAULT_TABLES 1000000
#define TSDB_TABLES_STEP 1000
#define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650

10
src/mnode/src/mnodeVgroup.c
View File

@ -323,7 +323,7 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
if (pDb == NULL) return TSDB_CODE_MND_APP_ERROR;
int32_t minIdPoolSize = TSDB_MAX_TABLES;
int32_t maxIdPoolSize = TSDB_MIN_TABLES;
int32_t maxIdPoolSize = tsMinTablePerVnode;
for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
SVgObj *pVgroup = pDb->vgList[v];
if (pVgroup == NULL) continue;
@ -347,10 +347,10 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
// realloc all vgroups in db
int32_t newIdPoolSize;
if (minIdPoolSize * 4 < TSDB_TABLES_STEP) {
if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
newIdPoolSize = minIdPoolSize * 4;
} else {
newIdPoolSize = ((minIdPoolSize / TSDB_TABLES_STEP) + 1) * TSDB_TABLES_STEP;
newIdPoolSize = ((minIdPoolSize / tsTableIncStepPerVnode) + 1) * tsTableIncStepPerVnode;
}
if (newIdPoolSize > tsMaxTablePerVnode) {
@ -471,6 +471,8 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
}
pMsg->expected = pVgroup->numOfVnodes;
pMsg->successed = 0;
pMsg->received = 0;
mnodeSendCreateVgroupMsg(pVgroup, pMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
@ -836,6 +838,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected,
mnodeMsg->rpcMsg.handle, rpcMsg->ahandle);
assert(mnodeMsg->received <= mnodeMsg->expected);
if (mnodeMsg->received != mnodeMsg->expected) return;
if (mnodeMsg->received == mnodeMsg->successed) {

2
src/vnode/src/vnodeMain.c
View File

@ -59,7 +59,7 @@ int32_t vnodeInitResources() {
vnodeInitWriteFp();
vnodeInitReadFp();
tsDnodeVnodesHash = taosHashInit(TSDB_MAX_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
if (tsDnodeVnodesHash == NULL) {
vError("failed to init vnode list");
return TSDB_CODE_VND_OUT_OF_MEMORY;

1
tests/CMakeLists.txt
View File

@ -12,3 +12,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON)
ADD_SUBDIRECTORY(examples/c)
ADD_SUBDIRECTORY(tsim)
ADD_SUBDIRECTORY(test/c)
ADD_SUBDIRECTORY(comparisonTest/tdengine)

12
tests/comparisonTest/tdengine/CMakeLists.txt Normal file
View File

@ -0,0 +1,12 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
add_executable(tdengineTest tdengineTest.c)
target_link_libraries(tdengineTest taos_static tutil common pthread)
ENDIF()

49
tests/comparisonTest/tdengine/tdengineTest.c
View File

@ -7,6 +7,7 @@
#include <time.h>
#include <pthread.h>
#include <sys/time.h>
#include <inttypes.h>
typedef struct {
char sql[256];
@ -123,19 +124,21 @@ void writeDataImp(void *param) {
if (taos == NULL)
taos_error(taos);
int code = taos_query(taos, "use db");
TAOS_RES* result = taos_query(taos, "use db");
int32_t code = taos_errno(result);
if (code != 0) {
taos_error(taos);
}
taos_free_result(result);
char sql[65000];
char *sql = calloc(1, 8*1024*1024);
int sqlLen = 0;
int lastMachineid = 0;
int counter = 0;
int totalRecords = 0;
for (int j = pThread->sID; j <= pThread->eID; j++) {
char fileName[256];
char fileName[300];
sprintf(fileName, "%s/testdata%d.csv", arguments.dataDir, j);
FILE *fp = fopen(fileName, "r");
@ -162,7 +165,7 @@ void writeDataImp(void *param) {
int64_t timestamp;
int temperature;
float humidity;
sscanf(line, "%d%s%d%lld%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
sscanf(line, "%d%s%d%" PRId64 "%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
if (counter == 0) {
sqlLen = sprintf(sql, "insert into");
@ -174,14 +177,16 @@ void writeDataImp(void *param) {
machineid, machineid, machinename, machinegroup);
}
sqlLen += sprintf(sql + sqlLen, "(%lld,%d,%f)", timestamp, temperature, humidity);
sqlLen += sprintf(sql + sqlLen, "(%" PRId64 ",%d,%f)", timestamp, temperature, humidity);
counter++;
if (counter >= arguments.rowsPerRequest) {
int code = taos_query(taos, sql);
TAOS_RES *result = taos_query(taos, sql);
int32_t code = taos_errno(result);
if (code != 0) {
printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
}
taos_free_result(result);
totalRecords += counter;
counter = 0;
@ -194,15 +199,18 @@ void writeDataImp(void *param) {
}
if (counter > 0) {
int code = taos_query(taos, sql);
TAOS_RES *result = taos_query(taos, sql);
int32_t code = taos_errno(result);
if (code != 0) {
printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
}
taos_free_result(result);
totalRecords += counter;
}
__sync_fetch_and_add(&statis.totalRows, totalRecords);
free(sql);
}
void writeData() {
@ -215,19 +223,23 @@ void writeData() {
taos_init();
void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
if (taos == NULL) taos_error(taos);
int code = taos_query(taos, "create database if not exists db");
TAOS_RES *result = taos_query(taos, "create database if not exists db");
int32_t code = taos_errno(result);
if (code != 0) {
taos_error(taos);
}
taos_free_result(result);
code = taos_query(taos, "create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
"tags(devid int, devname binary(16), devgroup int)");
result = taos_query(taos,
"create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
"tags(devid int, devname binary(16), devgroup int)");
code = taos_errno(result);
if (code != 0) {
taos_error(taos);
}
taos_free_result(result);
int64_t st = getTimeStampMs();
@ -292,17 +304,12 @@ void readData() {
int64_t st = getTimeStampMs();
int code = taos_query(taos, line);
TAOS_RES *result = taos_query(taos, line);
int32_t code = taos_errno(result);
if (code != 0) {
taos_error(taos);
}
void *result = taos_use_result(taos);
if (result == NULL) {
printf("failed to get result, reason:%s\n", taos_errstr(taos));
exit(1);
}
TAOS_ROW row;
int rows = 0;
//int num_fields = taos_field_count(taos);

4
tests/pytest/util/dnodes.py
View File

@ -28,7 +28,9 @@ class TDSimClient:
"locale": "en_US.UTF-8",
"charset": "UTF-8",
"asyncLog": "0",
"maxTablesPerVnode": "4",
"minTablesPerVnode": "4",
"maxTablesPerVnode": "1000",
"tableIncStepPerVnode": "10000",
"maxVgroupsPerDb": "1000",
"sdbDebugFlag": "143",
"rpcDebugFlag": "135",

2
tests/script/sh/deploy.sh
View File

@ -139,7 +139,9 @@ echo "clog 2" >> $TAOS_CFG
echo "statusInterval 1" >> $TAOS_CFG
echo "numOfTotalVnodes 4" >> $TAOS_CFG
echo "maxVgroupsPerDb 4" >> $TAOS_CFG
echo "minTablesPerVnode 4" >> $TAOS_CFG
echo "maxTablesPerVnode 1000" >> $TAOS_CFG
echo "tableIncStepPerVnode 10000" >> $TAOS_CFG
echo "asyncLog 0" >> $TAOS_CFG
echo "numOfMnodes 1" >> $TAOS_CFG
echo "locale en_US.UTF-8" >> $TAOS_CFG

13
tests/script/tmp/prepare.sim
View File

@ -25,10 +25,15 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20
system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2
system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 2
system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 2
system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 2
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1