From 043d759efdfd285e052b0244e2ccdf7f31ccc481 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 2 Jun 2020 15:51:40 +0800 Subject: [PATCH 1/5] fix printf format issue in rpcTcp.c [TD-505] --- src/rpc/src/rpcTcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 1e9b39414e..defd277db1 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -211,7 +211,7 @@ static void* taosAcceptTcpConnection(void *arg) { tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label); break; } - tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); + tError("%s TCP accept failure(%s)", pServerObj->label, strerror(errno)); continue; } From 775c5313d4eb023b239635dcbb8a6d6b861efd7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Jun 2020 10:05:07 +0000 Subject: [PATCH 2/5] [TD-509] add db name to vnodeObj --- src/inc/taosmsg.h | 1 + src/mnode/src/mnodeVgroup.c | 4 +++- src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeMain.c | 9 ++++++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index da6d0847ec..a11f34dd14 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -590,6 +590,7 @@ typedef struct { } SMDVnodeDesc; typedef struct { + char db[TSDB_DB_NAME_LEN + 1]; SMDVnodeCfg cfg; SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; } SMDCreateVnodeMsg; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9c24f7eaf3..920d8503b2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -539,6 +539,8 @@ SMDCreateVnodeMsg *mnodeBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); if (pVnode == NULL) return NULL; + strcpy(pVnode->db, pVgroup->dbName); + SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); pCfg->cfgVersion = htonl(pDb->cfgVersion); @@ -594,7 +596,7 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) { } void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { - mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle); + mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p db:%s", pVgroup->vgId, pVgroup->vgId, ahandle, pVgroup->dbName); SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .handle = ahandle, diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 0168304b51..7c95e81cf5 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -51,6 +51,7 @@ typedef struct { SSyncCfg syncCfg; SWalCfg walCfg; char *rootDir; + char db[TSDB_DB_NAME_LEN + 1]; } SVnodeObj; int vnodeWriteToQueue(void *param, void *pHead, int type); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 87ee5c2c1c..aa2ddc7776 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -496,7 +496,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { } len += snprintf(content + len, maxLen - len, "{\n"); - + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db); len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); @@ -568,6 +568,13 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } + cJSON *db = cJSON_GetObjectItem(root, "db"); + if (!db || db->type != cJSON_String || db->valuestring == NULL) { + vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId); + goto PARSE_OVER; + } + strcpy(pVnode->db, db->valuestring); + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); if (!cfgVersion || cfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId); From c2e2529c41c2ec8ba40ffa95e1a5dd221d5e4b45 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 3 Jun 2020 10:18:14 +0800 Subject: [PATCH 3/5] fix issue deref after free in tscUtil.c [TD-511] --- src/client/src/tscUtil.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4b9d2b920f..b8fd195aa6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -759,10 +759,6 @@ void tscCloseTscObj(STscObj* pObj) { taosTmrStopA(&(pObj->pTimer)); tscFreeSqlObj(pSql); - if (pSql) { - sem_destroy(&pSql->rspSem); - } - pthread_mutex_destroy(&pObj->mutex); if (pObj->pDnodeConn != NULL) { From 195f2b39dee2a38aba3548e620f35a570cf21858 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Jun 2020 06:24:24 +0000 Subject: [PATCH 4/5] TD-90 --- src/client/src/tscUtil.c | 1 + src/common/inc/tdataformat.h | 20 +++++++++++--------- src/common/src/tdataformat.c | 5 ++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b8fd195aa6..79c580e1a2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -651,6 +651,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { for (int32_t i = 0; i < numOfRows; ++i) { SDataRow trow = (SDataRow)pDataBlock; dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); + dataRowSetVersion(trow, pTableMeta->sversion); int toffset = 0; for (int32_t j = 0; j < tinfo.numOfColumns; j++) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index f71f52edc5..3ee29f47dd 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -119,22 +119,24 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Data row structure /* A data row, the format is like below: - * |<------------------------------------- len ---------------------------------->| - * |<--Head ->|<--------- flen -------------->| | - * +----------+---------------------------------+---------------------------------+ - * | int32_t | | | - * +----------+---------------------------------+---------------------------------+ - * | len | First part | Second part | - * +----------+---------------------------------+---------------------------------+ + * |<--------------------+--------------------------- len ---------------------------------->| + * |<-- Head -->|<--------- flen -------------->| | + * +---------------------+---------------------------------+---------------------------------+ + * | int16_t | int16_t | | | + * +----------+----------+---------------------------------+---------------------------------+ + * | len | sversion | First part | Second part | + * +----------+----------+---------------------------------+---------------------------------+ */ typedef void *SDataRow; -#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t) +#define TD_DATA_ROW_HEAD_SIZE sizeof(int16_t)*2 -#define dataRowLen(r) (*(int32_t *)(r)) +#define dataRowLen(r) (*(int16_t *)(r)) +#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) +#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 9816f0472f..2427c5d3d0 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -159,7 +159,10 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { /** * Initialize a data row */ -void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); } +void tdInitDataRow(SDataRow row, STSchema *pSchema) { + dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); + dataRowSetVersion(row, schemaVersion(pSchema)); +} SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { int32_t size = dataRowMaxBytesFromSchema(pSchema); From f81120e35252a7608f1226615957f9936b30fe7b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Jun 2020 07:29:55 +0000 Subject: [PATCH 5/5] TD-90 --- src/tsdb/inc/tsdbMain.h | 4 ++- src/tsdb/src/tsdbMeta.c | 70 ++++++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 453fbdc86b..db1dcbd35d 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -69,11 +69,13 @@ typedef struct { } SMemTable; // ---------- TSDB TABLE DEFINITION +#define TSDB_MAX_TABLE_SCHEMAS 16 typedef struct STable { int8_t type; STableId tableId; uint64_t superUid; // Super table UID - STSchema * schema; + int16_t numOfSchemas; + STSchema ** schema; STSchema * tagSchema; SKVRow tagVal; SMemTable * mem; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 16e034c228..6aaece59eb 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -37,18 +37,24 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { ptr = (char *)ptr + sizeof(int); memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name)); ptr = (char *)ptr + varDataLen(pTable->name); - + T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid); T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid); T_APPEND_MEMBER(ptr, pTable, STable, superUid); if (pTable->type == TSDB_SUPER_TABLE) { - ptr = tdEncodeSchema(ptr, pTable->schema); + T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas); + for (int i = 0; i < pTable->numOfSchemas; i++) { + ptr = tdEncodeSchema(ptr, pTable->schema[i]); + } ptr = tdEncodeSchema(ptr, pTable->tagSchema); } else if (pTable->type == TSDB_CHILD_TABLE) { ptr = tdEncodeKVRow(ptr, pTable->tagVal); } else { - ptr = tdEncodeSchema(ptr, pTable->schema); + T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas); + for (int i = 0; i < pTable->numOfSchemas; i++) { + ptr = tdEncodeSchema(ptr, pTable->schema[i]); + } } if (pTable->type == TSDB_STREAM_TABLE) { @@ -71,6 +77,11 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { STable *tsdbDecodeTable(void *cont, int contLen) { STable *pTable = (STable *)calloc(1, sizeof(STable)); if (pTable == NULL) return NULL; + pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); + if (pTable->schema == NULL) { + free(pTable); + return NULL; + } void *ptr = cont; T_READ_MEMBER(ptr, int8_t, pTable->type); @@ -78,28 +89,34 @@ STable *tsdbDecodeTable(void *cont, int contLen) { ptr = (char *)ptr + sizeof(int); pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1); if (pTable->name == NULL) return NULL; - + varDataSetLen(pTable->name, len); memcpy(pTable->name->data, ptr, len); - + ptr = (char *)ptr + len; T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid); T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid); T_READ_MEMBER(ptr, uint64_t, pTable->superUid); if (pTable->type == TSDB_SUPER_TABLE) { - pTable->schema = tdDecodeSchema(&ptr); + T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas); + for (int i = 0; i < pTable->numOfSchemas; i++) { + pTable->schema[i] = tdDecodeSchema(&ptr); + } pTable->tagSchema = tdDecodeSchema(&ptr); } else if (pTable->type == TSDB_CHILD_TABLE) { ptr = tdDecodeKVRow(ptr, &pTable->tagVal); } else { - pTable->schema = tdDecodeSchema(&ptr); + T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas); + for (int i = 0; i < pTable->numOfSchemas; i++) { + pTable->schema[i] = tdDecodeSchema(&ptr); + } } if (pTable->type == TSDB_STREAM_TABLE) { ptr = taosDecodeString(ptr, &(pTable->sql)); } - + pTable->lastKey = TSKEY_INITIAL_VAL; return pTable; } @@ -221,13 +238,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) { return 0; } +// Get the newest table schema STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) { - return pTable->schema; + return pTable->schema[pTable->numOfSchemas - 1]; } else if (pTable->type == TSDB_CHILD_TABLE) { STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); if (pSuper == NULL) return NULL; - return pSuper->schema; + return pSuper->schema[pSuper->numOfSchemas-1]; } else { return NULL; } @@ -299,13 +317,16 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { } pTable->type = pCfg->type; + pTable->numOfSchemas = 0; if (isSuper) { pTable->type = TSDB_SUPER_TABLE; pTable->tableId.uid = pCfg->superUid; pTable->tableId.tid = -1; pTable->superUid = TSDB_INVALID_SUPER_TABLE_ID; - pTable->schema = tdDupSchema(pCfg->schema); + pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); + pTable->numOfSchemas = 1; + pTable->schema[0] = tdDupSchema(pCfg->schema); pTable->tagSchema = tdDupSchema(pCfg->tagSchema); tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN); @@ -340,14 +361,18 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { if (pCfg->type == TSDB_CHILD_TABLE) { pTable->superUid = pCfg->superUid; pTable->tagVal = tdKVRowDup(pCfg->tagValues); - } else if (pCfg->type == TSDB_NORMAL_TABLE) { - pTable->superUid = -1; - pTable->schema = tdDupSchema(pCfg->schema); } else { - ASSERT(pCfg->type == TSDB_STREAM_TABLE); - pTable->superUid = -1; - pTable->schema = tdDupSchema(pCfg->schema); - pTable->sql = strdup(pCfg->sql); + pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); + pTable->numOfSchemas = 1; + pTable->schema[0] = tdDupSchema(pCfg->schema); + + if (pCfg->type == TSDB_NORMAL_TABLE) { + pTable->superUid = -1; + } else { + ASSERT(pCfg->type == TSDB_STREAM_TABLE); + pTable->superUid = -1; + pTable->sql = strdup(pCfg->sql); + } } } @@ -580,7 +605,7 @@ static int tsdbFreeTable(STable *pTable) { if (pTable->type == TSDB_CHILD_TABLE) { kvRowFree(pTable->tagVal); } else { - tdFreeSchema(pTable->schema); + for (int i = 0; i < pTable->numOfSchemas; i++) tdFreeSchema(pTable->schema[i]); } if (pTable->type == TSDB_STREAM_TABLE) { @@ -642,9 +667,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { } // Update the pMeta->maxCols and pMeta->maxRowBytes - if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE) { - if (schemaNCols(pTable->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pTable->schema); - int bytes = dataRowMaxBytesFromSchema(pTable->schema); + if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE) { + if (schemaNCols(pTable->schema[pTable->numOfSchemas - 1]) > pMeta->maxCols) + pMeta->maxCols = schemaNCols(pTable->schema[pTable->numOfSchemas - 1]); + int bytes = dataRowMaxBytesFromSchema(pTable->schema[pTable->numOfSchemas - 1]); if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes; }