xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

TD-1767

This commit is contained in:
Shengliang Guan 2020-10-22 06:28:44 +00:00
parent 28e6927df8
commit 44963201ab
14 changed files with 104 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -16,6 +16,7 @@ SET(TD_GRANT FALSE)
SET(TD_SYNC TRUE)
SET(TD_MQTT TRUE)
SET(TD_TSDB_PLUGINS FALSE)
SET(TD_STORAGE FALSE)
SET(TD_COVER FALSE)
SET(TD_MEM_CHECK FALSE)

4
cmake/define.inc
View File

@ -25,6 +25,10 @@ IF (TD_TSDB_PLUGINS)
ADD_DEFINITIONS(-D_TSDB_PLUGINS)
ENDIF ()
IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE)
ENDIF ()
IF (TD_GODLL)
ADD_DEFINITIONS(-D_TD_GO_DLL_)
ENDIF ()

18
src/common/inc/tglobal.h
View File

@ -20,8 +20,6 @@
extern "C" {
#endif
struct SDnodeTier;
// cluster
extern char tsFirst[];
extern char tsSecond[];
@ -157,9 +155,6 @@ extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
// dnode
extern struct SDnodeTier *pDnodeTier;
// log
extern int32_t tsAsyncLog;
extern int32_t tsNumOfLogLines;
@ -182,6 +177,14 @@ extern int32_t wDebugFlag;
extern int32_t cqDebugFlag;
extern int32_t debugFlag;
typedef struct {
char dir[TSDB_FILENAME_LEN];
int level;
int primary;
} SDiskCfg;
extern int32_t tsDiskCfgNum;
extern SDiskCfg tsDiskCfg[];
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
void taosInitGlobalCfg();
@ -190,7 +193,10 @@ void taosSetAllDebugFlag();
bool taosCfgDynamicOptions(char *msg);
int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
void taosAddDataDir(int index, char *v1, int level, int primary);
void taosReadDataDirCfg(char *v1, char *v2, char *v3);
void taosPrintDataDirCfg();
#ifdef __cplusplus
}
#endif

38
src/common/src/tglobal.c
View File

@ -160,6 +160,13 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDataDir[TSDB_FILENAME_LEN] = {0};
char tsScriptDir[TSDB_FILENAME_LEN] = {0};
int32_t tsDiskCfgNum = 0;
#ifndef _STORAGE
SDiskCfg tsDiskCfg[1];
#else
SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
#endif
/*
* minimum scale for whole system, millisecond by default
@ -305,6 +312,34 @@ bool taosCfgDynamicOptions(char *msg) {
return false;
}
void taosAddDataDir(int index, char *v1, int level, int primary) {
tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
tsDiskCfg[index].level = level;
tsDiskCfg[index].primary = primary;
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
}
#ifndef _STORAGE
void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
taosAddDataDir(0, tsDataDir, 0, 1);
tstrncpy(tsDiskCfg[0].dir, tsDataDir, TSDB_FILENAME_LEN);
}
#endif
void taosPrintDataDirCfg() {
for (int i = 0; i < tsDiskCfgNum; ++i) {
SDiskCfg *cfg = &tsDiskCfg[i];
uInfo(" dataDir:%s level:%d primary:%d", cfg->dir, cfg->level, cfg->primary);
}
}
static void taosCheckDataDirCfg() {
if (tsDiskCfgNum <= 0) {
taosAddDataDir(0, tsDataDir, 0, 1);
tsDiskCfgNum = 1;
}
}
static void doInitGlobalConfig(void) {
osInit();
srand(taosSafeRand());
@ -386,7 +421,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "dataDir";
cfg.ptr = tsDataDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
@ -1328,6 +1363,7 @@ bool taosCheckGlobalCfg() {
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
}
taosCheckDataDirCfg();
taosGetSystemInfo();
tsSetLocale();

20
src/dnode/src/dnodeMain.c
View File

@ -169,17 +169,17 @@ static void dnodeCheckDataDirOpenned(char *dir) {
}
static int32_t dnodeInitStorage() {
pDnodeTier = dnodeNewTier();
if (pDnodeTier == NULL) {
tsDnodeTier = dnodeNewTier();
if (tsDnodeTier == NULL) {
dError("failed to create new dnode tier since %s", tstrerror(terrno));
return -1;
}
if (dnodeAddDisks(pDnodeTier, NULL, 0) < 0) {
if (dnodeAddDisks(tsDnodeTier, tsDiskCfg, tsDiskCfgNum) < 0) {
dError("failed to add disks to dnode tier since %s", tstrerror(terrno));
return -1;
}
strncpy(tsDataDir, DNODE_PRIMARY_DISK(pDnodeTier)->dir, TSDB_FILENAME_LEN);
strncpy(tsDataDir, DNODE_PRIMARY_DISK(tsDnodeTier)->dir, TSDB_FILENAME_LEN);
tdGetVnodeRootDir(tsDataDir, tsVnodeDir);
//TODO(dengyihao): no need to init here
@ -195,12 +195,12 @@ static int32_t dnodeInitStorage() {
return -1;
}
for (int i = 0; i < pDnodeTier->nTiers; i++) {
for (int i = 0; i < tsDnodeTier->nTiers; i++) {
char dirName[TSDB_FILENAME_LEN];
STier *pTier = pDnodeTier->tiers + i;
STier *pTier = tsDnodeTier->tiers + i;
for (int j = 0; j < pTier->nDisks; j++) {
SDisk *pDisk = dnodeGetDisk(pDnodeTier, i, j);
SDisk *pDisk = dnodeGetDisk(tsDnodeTier, i, j);
tdGetVnodeRootDir(dirName, pDisk->dir);
if (dnodeCreateDir(dirName) < 0) {
@ -223,9 +223,9 @@ static int32_t dnodeInitStorage() {
}
static void dnodeCleanupStorage() {
if (pDnodeTier) {
dnodeCloseTier(pDnodeTier);
pDnodeTier = NULL;
if (tsDnodeTier) {
dnodeCloseTier(tsDnodeTier);
tsDnodeTier = NULL;
}
}

8
src/dnode/src/dnodeTier.c
View File

@ -40,7 +40,7 @@ SDnodeTier *dnodeNewTier() {
return NULL;
}
pDnodeTier->map = taosHashInit(DNODE_MAX_TIERS * DNODE_MAX_DISKS_PER_TIER * 2,
pDnodeTier->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pDnodeTier->map == NULL) {
terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
@ -217,7 +217,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
SDiskID diskid = {0};
SDisk * pDisk = NULL;
if (level < 0 || level >= DNODE_MAX_TIERS) {
if (level < 0 || level >= TSDB_MAX_TIERS) {
terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
return -1;
@ -231,7 +231,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
pTier = pDnodeTier->tiers + level;
diskid.level = level;
if (pTier->nDisks >= DNODE_MAX_DISKS_PER_TIER) {
if (pTier->nDisks >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
return -1;
@ -268,7 +268,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
diskid.did = pTier->nDisks;
} else {
diskid.did = pTier->nDisks + 1;
if (diskid.did >= DNODE_MAX_DISKS_PER_TIER) {
if (diskid.did >= TSDB_MAX_DISKS_PER_TIER) {
terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
return -1;

18
src/inc/dnode.h
View File

@ -20,6 +20,8 @@
extern "C" {
#endif
#include "taosdef.h"
#include "tglobal.h"
#include "hash.h"
#include "taoserror.h"
#include "trpc.h"
@ -71,16 +73,6 @@ void dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
void dnodeSendStatusMsgToMnode();
// DNODE TIER
#define DNODE_MAX_TIERS 3
#define DNODE_MAX_DISKS_PER_TIER 16
typedef struct {
char dir[TSDB_FILENAME_LEN];
int level;
int primary;
} SDiskCfg;
typedef struct {
int level;
int did;
@ -100,16 +92,17 @@ typedef struct {
typedef struct {
int level;
int nDisks;
SDisk *disks[DNODE_MAX_DISKS_PER_TIER];
SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
} STier;
typedef struct SDnodeTier {
pthread_rwlock_t rwlock;
int nTiers;
STier tiers[DNODE_MAX_TIERS];
STier tiers[TSDB_MAX_TIERS];
SHashObj * map;
} SDnodeTier;
extern struct SDnodeTier *tsDnodeTier;
#define DNODE_PRIMARY_DISK(pDnodeTier) (pDnodeTier)->tiers[0].disks[0]
static FORCE_INLINE int dnodeRLockTiers(SDnodeTier *pDnodeTier) {
@ -155,7 +148,6 @@ int dnodeCheckTiers(SDnodeTier *pDnodeTier);
SDisk * dnodeAssignDisk(SDnodeTier *pDnodeTier, int level);
SDisk * dnodeGetDiskByName(SDnodeTier *pDnodeTier, char *dirName);
#ifdef __cplusplus
}
#endif

4
src/inc/taosdef.h
View File

@ -432,6 +432,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TAOS_QTYPE_CQ 3
#define TAOS_QTYPE_QUERY 4
#define TSDB_MAX_TIERS 3
#define TSDB_MAX_DISKS_PER_TIER 16
#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
typedef enum {
TSDB_SUPER_TABLE = 0, // super table
TSDB_CHILD_TABLE = 1, // table created from super table

1
src/os/src/detail/osSysinfo.c
View File

@ -533,7 +533,6 @@ void taosPrintOsInfo() {
uInfo(" os release: %s", buf.release);
uInfo(" os version: %s", buf.version);
uInfo(" os machine: %s", buf.machine);
uInfo("==================================");
}
void taosKillSystem() {

6
src/tsdb/src/tsdbFile.c
View File

@ -26,7 +26,7 @@
#include "dnode.h"
#include "tpath.h"
struct SDnodeTier *pDnodeTier = NULL;
struct SDnodeTier *tsDnodeTier = NULL;
const char * tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"};
static void tsdbDestroyFile(SFile *pFile);
@ -79,8 +79,8 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL);
char dataDir[TSDB_FILENAME_LEN] = "\0";
for (int level = 0; level < pDnodeTier->nTiers; level++) {
STier *pTier = pDnodeTier->tiers + level;
for (int level = 0; level < tsDnodeTier->nTiers; level++) {
STier *pTier = tsDnodeTier->tiers + level;
for (int did = 0; did < pTier->nDisks; did++) {
SDisk *pDisk = pTier->disks[did];

4
src/util/CMakeLists.txt
View File

@ -29,3 +29,7 @@ ELSEIF (TD_WINDOWS)
ELSEIF(TD_DARWIN)
TARGET_LINK_LIBRARIES(tutil iconv)
ENDIF()
IF (TD_STORAGE)
TARGET_LINK_LIBRARIES(tutil storage)
ENDIF ()

1
src/util/inc/tconfig.h
View File

@ -47,6 +47,7 @@ enum {
TAOS_CFG_VTYPE_STRING,
TAOS_CFG_VTYPE_IPSTR,
TAOS_CFG_VTYPE_DIRECTORY,
TAOS_CFG_VTYPE_DATA_DIRCTORY,
};
enum {

27
src/util/src/tconfig.c
View File

@ -199,7 +199,7 @@ SGlobalCfg *taosGetConfigOption(const char *option) {
return NULL;
}
static void taosReadConfigOption(const char *option, char *value) {
static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) {
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue;
@ -224,6 +224,9 @@ static void taosReadConfigOption(const char *option, char *value) {
case TAOS_CFG_VTYPE_DIRECTORY:
taosReadDirectoryConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_DATA_DIRCTORY:
taosReadDirectoryConfig(cfg, value);
taosReadDataDirCfg(value, value2, value3);
default:
uError("config option:%s, input value:%s, can't be recognized", option, value);
break;
@ -307,8 +310,8 @@ void taosReadGlobalLogCfg() {
}
bool taosReadGlobalCfg() {
char * line, *option, *value, *value1;
int olen, vlen, vlen1;
char * line, *option, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
char fileName[PATH_MAX] = {0};
sprintf(fileName, "%s/taos.cfg", configDir);
@ -331,8 +334,8 @@ bool taosReadGlobalCfg() {
while (!feof(fp)) {
memset(line, 0, len);
option = value = NULL;
olen = vlen = 0;
option = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
taosGetline(&line, &len, fp);
line[len - 1] = 0;
@ -345,11 +348,13 @@ bool taosReadGlobalCfg() {
if (vlen == 0) continue;
value[vlen] = 0;
// For dataDir, the format is:
// dataDir /mnt/disk1 0
paGetToken(value + vlen + 1, &value1, &vlen1);
taosReadConfigOption(option, value);
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) value2[vlen2] = 0;
paGetToken(value + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
taosReadConfigOption(option, value, value2, value3);
}
fclose(fp);
@ -397,4 +402,6 @@ void taosPrintGlobalCfg() {
}
taosPrintOsInfo();
taosPrintDataDirCfg();
uInfo("==================================");
}

4
src/vnode/src/vnodeMain.c
View File

@ -418,8 +418,8 @@ void vnodeRelease(void *pVnodeRaw) {
char rootDir[TSDB_FILENAME_LEN] = {0};
char newDir[TSDB_FILENAME_LEN] = {0};
for (int i = 0; i < pDnodeTier->nTiers; i++) {
STier *pTier = pDnodeTier->tiers + i;
for (int i = 0; i < tsDnodeTier->nTiers; i++) {
STier *pTier = tsDnodeTier->tiers + i;
for (int j = 0; j < pTier->nDisks; j++) {
SDisk *pDisk = pTier->disks[j];