xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/2.0tsdb' of https://github.com/taosdata/TDengine into feature/2.0tsdb

This commit is contained in:
Haojun Liao 2020-06-03 15:37:51 +08:00
parent e016a975f0 f81120e352
commit 56743e89f0
10 changed files with 81 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/client/src/tscUtil.c
View File

@ -651,6 +651,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
SDataRow trow = (SDataRow)pDataBlock; SDataRow trow = (SDataRow)pDataBlock;
dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen);
dataRowSetVersion(trow, pTableMeta->sversion);
int toffset = 0; int toffset = 0;
for (int32_t j = 0; j < tinfo.numOfColumns; j++) { for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
@ -759,10 +760,6 @@ void tscCloseTscObj(STscObj* pObj) {
taosTmrStopA(&(pObj->pTimer)); taosTmrStopA(&(pObj->pTimer));
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
if (pSql) {
sem_destroy(&pSql->rspSem);
}
pthread_mutex_destroy(&pObj->mutex); pthread_mutex_destroy(&pObj->mutex);
if (pObj->pDnodeConn != NULL) { if (pObj->pDnodeConn != NULL) {

20
src/common/inc/tdataformat.h
View File

@ -119,22 +119,24 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
// ----------------- Data row structure // ----------------- Data row structure
/* A data row, the format is like below: /* A data row, the format is like below:
* |<------------------------------------- len ---------------------------------->| * |<--------------------+--------------------------- len ---------------------------------->|
* |<--Head ->|<--------- flen -------------->| | * |<-- Head -->|<--------- flen -------------->| |
* +----------+---------------------------------+---------------------------------+ * +---------------------+---------------------------------+---------------------------------+
* | int32_t | | | * | int16_t | int16_t | | |
* +----------+---------------------------------+---------------------------------+ * +----------+----------+---------------------------------+---------------------------------+
* | len | First part | Second part | * | len | sversion | First part | Second part |
* +----------+---------------------------------+---------------------------------+ * +----------+----------+---------------------------------+---------------------------------+
*/ */
typedef void *SDataRow; typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t) #define TD_DATA_ROW_HEAD_SIZE sizeof(int16_t)*2
#define dataRowLen(r) (*(int32_t *)(r)) #define dataRowLen(r) (*(int16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)

5
src/common/src/tdataformat.c
View File

@ -159,7 +159,10 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
/** /**
* Initialize a data row * Initialize a data row
*/ */
void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); } void tdInitDataRow(SDataRow row, STSchema *pSchema) {
dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema));
dataRowSetVersion(row, schemaVersion(pSchema));
}
SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
int32_t size = dataRowMaxBytesFromSchema(pSchema); int32_t size = dataRowMaxBytesFromSchema(pSchema);

1
src/inc/taosmsg.h
View File

@ -602,6 +602,7 @@ typedef struct {
} SMDVnodeDesc; } SMDVnodeDesc;
typedef struct { typedef struct {
char db[TSDB_DB_NAME_LEN + 1];
SMDVnodeCfg cfg; SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg; } SMDCreateVnodeMsg;

4
src/mnode/src/mnodeVgroup.c
View File

@ -539,6 +539,8 @@ SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) {
SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
if (pVnode == NULL) return NULL; if (pVnode == NULL) return NULL;
strcpy(pVnode->db, pVgroup->dbName);
SMDVnodeCfg *pCfg = &pVnode->cfg; SMDVnodeCfg *pCfg = &pVnode->cfg;
pCfg->vgId = htonl(pVgroup->vgId); pCfg->vgId = htonl(pVgroup->vgId);
pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cfgVersion = htonl(pDb->cfgVersion);
@ -594,7 +596,7 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) {
} }
void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle); mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p db:%s", pVgroup->vgId, pVgroup->vgId, ahandle, pVgroup->dbName);
SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.handle = ahandle, .handle = ahandle,

2
src/rpc/src/rpcTcp.c
View File

@ -211,7 +211,7 @@ static void* taosAcceptTcpConnection(void *arg) {
tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label); tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
break; break;
} }
tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); tError("%s TCP accept failure(%s)", pServerObj->label, strerror(errno));
continue; continue;
} }

4
src/tsdb/inc/tsdbMain.h
View File

@ -69,11 +69,13 @@ typedef struct {
} SMemTable; } SMemTable;
// ---------- TSDB TABLE DEFINITION // ---------- TSDB TABLE DEFINITION
#define TSDB_MAX_TABLE_SCHEMAS 16
typedef struct STable { typedef struct STable {
int8_t type; int8_t type;
STableId tableId; STableId tableId;
uint64_t superUid; // Super table UID uint64_t superUid; // Super table UID
STSchema * schema; int16_t numOfSchemas;
STSchema ** schema;
STSchema * tagSchema; STSchema * tagSchema;
SKVRow tagVal; SKVRow tagVal;
SMemTable * mem; SMemTable * mem;

70
src/tsdb/src/tsdbMeta.c
View File

@ -37,18 +37,24 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) {
ptr = (char *)ptr + sizeof(int); ptr = (char *)ptr + sizeof(int);
memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name)); memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name));
ptr = (char *)ptr + varDataLen(pTable->name); ptr = (char *)ptr + varDataLen(pTable->name);
T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid); T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid);
T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid); T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid);
T_APPEND_MEMBER(ptr, pTable, STable, superUid); T_APPEND_MEMBER(ptr, pTable, STable, superUid);
if (pTable->type == TSDB_SUPER_TABLE) { if (pTable->type == TSDB_SUPER_TABLE) {
ptr = tdEncodeSchema(ptr, pTable->schema); T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas);
for (int i = 0; i < pTable->numOfSchemas; i++) {
ptr = tdEncodeSchema(ptr, pTable->schema[i]);
}
ptr = tdEncodeSchema(ptr, pTable->tagSchema); ptr = tdEncodeSchema(ptr, pTable->tagSchema);
} else if (pTable->type == TSDB_CHILD_TABLE) { } else if (pTable->type == TSDB_CHILD_TABLE) {
ptr = tdEncodeKVRow(ptr, pTable->tagVal); ptr = tdEncodeKVRow(ptr, pTable->tagVal);
} else { } else {
ptr = tdEncodeSchema(ptr, pTable->schema); T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas);
for (int i = 0; i < pTable->numOfSchemas; i++) {
ptr = tdEncodeSchema(ptr, pTable->schema[i]);
}
} }
if (pTable->type == TSDB_STREAM_TABLE) { if (pTable->type == TSDB_STREAM_TABLE) {
@ -71,6 +77,11 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) {
STable *tsdbDecodeTable(void *cont, int contLen) { STable *tsdbDecodeTable(void *cont, int contLen) {
STable *pTable = (STable *)calloc(1, sizeof(STable)); STable *pTable = (STable *)calloc(1, sizeof(STable));
if (pTable == NULL) return NULL; if (pTable == NULL) return NULL;
pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
if (pTable->schema == NULL) {
free(pTable);
return NULL;
}
void *ptr = cont; void *ptr = cont;
T_READ_MEMBER(ptr, int8_t, pTable->type); T_READ_MEMBER(ptr, int8_t, pTable->type);
@ -78,28 +89,34 @@ STable *tsdbDecodeTable(void *cont, int contLen) {
ptr = (char *)ptr + sizeof(int); ptr = (char *)ptr + sizeof(int);
pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1); pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1);
if (pTable->name == NULL) return NULL; if (pTable->name == NULL) return NULL;
varDataSetLen(pTable->name, len); varDataSetLen(pTable->name, len);
memcpy(pTable->name->data, ptr, len); memcpy(pTable->name->data, ptr, len);
ptr = (char *)ptr + len; ptr = (char *)ptr + len;
T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid); T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid);
T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid); T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid);
T_READ_MEMBER(ptr, uint64_t, pTable->superUid); T_READ_MEMBER(ptr, uint64_t, pTable->superUid);
if (pTable->type == TSDB_SUPER_TABLE) { if (pTable->type == TSDB_SUPER_TABLE) {
pTable->schema = tdDecodeSchema(&ptr); T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas);
for (int i = 0; i < pTable->numOfSchemas; i++) {
pTable->schema[i] = tdDecodeSchema(&ptr);
}
pTable->tagSchema = tdDecodeSchema(&ptr); pTable->tagSchema = tdDecodeSchema(&ptr);
} else if (pTable->type == TSDB_CHILD_TABLE) { } else if (pTable->type == TSDB_CHILD_TABLE) {
ptr = tdDecodeKVRow(ptr, &pTable->tagVal); ptr = tdDecodeKVRow(ptr, &pTable->tagVal);
} else { } else {
pTable->schema = tdDecodeSchema(&ptr); T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas);
for (int i = 0; i < pTable->numOfSchemas; i++) {
pTable->schema[i] = tdDecodeSchema(&ptr);
}
} }
if (pTable->type == TSDB_STREAM_TABLE) { if (pTable->type == TSDB_STREAM_TABLE) {
ptr = taosDecodeString(ptr, &(pTable->sql)); ptr = taosDecodeString(ptr, &(pTable->sql));
} }
pTable->lastKey = TSKEY_INITIAL_VAL; pTable->lastKey = TSKEY_INITIAL_VAL;
return pTable; return pTable;
} }
@ -221,13 +238,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
return 0; return 0;
} }
// Get the newest table schema
STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) {
if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) { if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) {
return pTable->schema; return pTable->schema[pTable->numOfSchemas - 1];
} else if (pTable->type == TSDB_CHILD_TABLE) { } else if (pTable->type == TSDB_CHILD_TABLE) {
STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid);
if (pSuper == NULL) return NULL; if (pSuper == NULL) return NULL;
return pSuper->schema; return pSuper->schema[pSuper->numOfSchemas-1];
} else { } else {
return NULL; return NULL;
} }
@ -287,13 +305,16 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
} }
pTable->type = pCfg->type; pTable->type = pCfg->type;
pTable->numOfSchemas = 0;
if (isSuper) { if (isSuper) {
pTable->type = TSDB_SUPER_TABLE; pTable->type = TSDB_SUPER_TABLE;
pTable->tableId.uid = pCfg->superUid; pTable->tableId.uid = pCfg->superUid;
pTable->tableId.tid = -1; pTable->tableId.tid = -1;
pTable->superUid = TSDB_INVALID_SUPER_TABLE_ID; pTable->superUid = TSDB_INVALID_SUPER_TABLE_ID;
pTable->schema = tdDupSchema(pCfg->schema); pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
pTable->numOfSchemas = 1;
pTable->schema[0] = tdDupSchema(pCfg->schema);
pTable->tagSchema = tdDupSchema(pCfg->tagSchema); pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN); tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
@ -328,14 +349,18 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
if (pCfg->type == TSDB_CHILD_TABLE) { if (pCfg->type == TSDB_CHILD_TABLE) {
pTable->superUid = pCfg->superUid; pTable->superUid = pCfg->superUid;
pTable->tagVal = tdKVRowDup(pCfg->tagValues); pTable->tagVal = tdKVRowDup(pCfg->tagValues);
} else if (pCfg->type == TSDB_NORMAL_TABLE) {
pTable->superUid = -1;
pTable->schema = tdDupSchema(pCfg->schema);
} else { } else {
ASSERT(pCfg->type == TSDB_STREAM_TABLE); pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
pTable->superUid = -1; pTable->numOfSchemas = 1;
pTable->schema = tdDupSchema(pCfg->schema); pTable->schema[0] = tdDupSchema(pCfg->schema);
pTable->sql = strdup(pCfg->sql);
if (pCfg->type == TSDB_NORMAL_TABLE) {
pTable->superUid = -1;
} else {
ASSERT(pCfg->type == TSDB_STREAM_TABLE);
pTable->superUid = -1;
pTable->sql = strdup(pCfg->sql);
}
} }
} }
@ -568,7 +593,7 @@ static int tsdbFreeTable(STable *pTable) {
if (pTable->type == TSDB_CHILD_TABLE) { if (pTable->type == TSDB_CHILD_TABLE) {
kvRowFree(pTable->tagVal); kvRowFree(pTable->tagVal);
} else { } else {
tdFreeSchema(pTable->schema); for (int i = 0; i < pTable->numOfSchemas; i++) tdFreeSchema(pTable->schema[i]);
} }
if (pTable->type == TSDB_STREAM_TABLE) { if (pTable->type == TSDB_STREAM_TABLE) {
@ -630,9 +655,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
} }
// Update the pMeta->maxCols and pMeta->maxRowBytes // Update the pMeta->maxCols and pMeta->maxRowBytes
if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE) { if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE) {
if (schemaNCols(pTable->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pTable->schema); if (schemaNCols(pTable->schema[pTable->numOfSchemas - 1]) > pMeta->maxCols)
int bytes = dataRowMaxBytesFromSchema(pTable->schema); pMeta->maxCols = schemaNCols(pTable->schema[pTable->numOfSchemas - 1]);
int bytes = dataRowMaxBytesFromSchema(pTable->schema[pTable->numOfSchemas - 1]);
if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes; if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
} }

1
src/vnode/inc/vnodeInt.h
View File

@ -51,6 +51,7 @@ typedef struct {
SSyncCfg syncCfg; SSyncCfg syncCfg;
SWalCfg walCfg; SWalCfg walCfg;
char *rootDir; char *rootDir;
char db[TSDB_DB_NAME_LEN + 1];
} SVnodeObj; } SVnodeObj;
int vnodeWriteToQueue(void *param, void *pHead, int type); int vnodeWriteToQueue(void *param, void *pHead, int type);

9
src/vnode/src/vnodeMain.c
View File

@ -498,7 +498,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
} }
len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db);
len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion);
len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize);
len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks);
@ -570,6 +570,13 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
goto PARSE_OVER; goto PARSE_OVER;
} }
cJSON *db = cJSON_GetObjectItem(root, "db");
if (!db || db->type != cJSON_String || db->valuestring == NULL) {
vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId);
goto PARSE_OVER;
}
strcpy(pVnode->db, db->valuestring);
cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
if (!cfgVersion || cfgVersion->type != cJSON_Number) { if (!cfgVersion || cfgVersion->type != cJSON_Number) {
vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId); vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId);