From 3aa37460c7e699cde2ba03233d21dde57b36f5de Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 10 Mar 2024 08:50:05 +0800 Subject: [PATCH 1/5] enh: logs and show grants --- include/common/tgrant.h | 7 ++----- source/common/src/systable.c | 3 +++ source/common/src/tgrant.c | 6 ------ source/util/src/tlog.c | 11 +++++++++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index ca9e030117..ac0d3155c7 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -61,9 +61,6 @@ typedef enum { int32_t grantCheck(EGrantType grant); int32_t grantCheckExpire(EGrantType grant); char* tGetMachineId(); -#ifdef TD_UNIQ_GRANT -int32_t grantCheckLE(EGrantType grant); -#endif // #ifndef GRANTS_CFG #ifdef TD_ENTERPRISE @@ -76,7 +73,7 @@ int32_t grantCheckLE(EGrantType grant); {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #else #define GRANTS_SCHEMA \ @@ -88,7 +85,7 @@ int32_t grantCheckLE(EGrantType grant); {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #endif // #define GRANT_CFG_ADD diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 25cc5d7c79..8ef0d1061c 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -363,6 +363,9 @@ static const SSysDbTableSchema useGrantsLogsSchema[] = { static const SSysDbTableSchema useMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, +#ifndef TD_UNIQ_GRANT + {.name = "machine_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, +#endif {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c index 2b5d91788e..fa1a7d90b7 100644 --- a/source/common/src/tgrant.c +++ b/source/common/src/tgrant.c @@ -21,11 +21,5 @@ int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } int32_t grantCheckExpire(EGrantType grant) { return TSDB_CODE_SUCCESS; } -#ifdef TD_UNIQ_GRANT -int32_t grantCheckLE(EGrantType grant) { return TSDB_CODE_SUCCESS; } -#endif #else -#ifdef TD_UNIQ_GRANT -int32_t grantCheckExpire(EGrantType grant) { return TSDB_CODE_SUCCESS; } -#endif #endif \ No newline at end of file diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index eae02125d2..16ca1b64c3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -41,6 +41,12 @@ #define LOG_BUF_SIZE(x) ((x)->buffSize) #define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#ifdef TD_ENTERPRISE +#define LOG_EDITION_FLG ("E") +#else +#define LOG_EDITION_FLG ("C") +#endif + typedef struct { char *buffer; int32_t buffStart; @@ -490,8 +496,9 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { time_t curTime = timeSecs.tv_sec; ptm = taosLocalTime(&curTime, &Tm, NULL); - return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), flags); + return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", ptm->tm_mon + 1, ptm->tm_mday, + ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), + LOG_EDITION_FLG, flags); } static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { From 0b15e3415d13ff45bb1c34404405265f064bfe05 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 18 Mar 2024 18:06:18 +0800 Subject: [PATCH 2/5] enh: machine id persistence --- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 - source/dnode/mnode/impl/src/mndDnode.c | 51 +++++++++++++++++++------- source/libs/parser/src/parInsertSql.c | 2 +- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 8ef0d1061c..cc84f7f45b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -364,7 +364,7 @@ static const SSysDbTableSchema useGrantsLogsSchema[] = { static const SSysDbTableSchema useMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, #ifndef TD_UNIQ_GRANT - {.name = "machine_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, #endif {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a092057ce..923c094fc6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -214,8 +214,6 @@ typedef struct { uint16_t port; char fqdn[TSDB_FQDN_LEN]; char ep[TSDB_EP_LEN]; - char active[TSDB_ACTIVE_KEY_LEN]; - char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeObj; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index befb6d3521..dc1aa2e1ef 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -33,7 +33,7 @@ #include "taos_monitor.h" #define TSDB_DNODE_VER_NUMBER 2 -#define TSDB_DNODE_RESERVE_SIZE 64 +#define TSDB_DNODE_RESERVE_SIZE 40 static const char *offlineReason[] = { "", @@ -183,11 +183,10 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { SDB_SET_INT64(pRaw, dataPos, pDnode->updateTime, _OVER) SDB_SET_INT16(pRaw, dataPos, pDnode->port, _OVER) SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER) SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER) - SDB_SET_INT16(pRaw, dataPos, TSDB_ACTIVE_KEY_LEN, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pDnode->active, TSDB_ACTIVE_KEY_LEN, _OVER) - SDB_SET_INT16(pRaw, dataPos, TSDB_CONN_ACTIVE_KEY_LEN, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN, _OVER) + SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible + SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible SDB_SET_DATALEN(pRaw, dataPos, _OVER); terrno = 0; @@ -227,13 +226,14 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pDnode->updateTime, _OVER) SDB_GET_INT16(pRaw, dataPos, &pDnode->port, _OVER) SDB_GET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER) SDB_GET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER) if (sver > 1) { int16_t keyLen = 0; SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pDnode->active, keyLen, _OVER) + if (keyLen > 0) SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pDnode->connActive, keyLen, _OVER) + if (keyLen > 0) SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) } terrno = 0; @@ -271,12 +271,7 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; #ifdef TD_ENTERPRISE - if (strncmp(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN) != 0) { - strncpy(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN); - } - if (strncmp(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN) != 0) { - strncpy(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN); - } + tstrncpy(pOld->machineId, pNew->machineId, TSDB_MACHINE_ID_LEN + 1); #endif return 0; } @@ -705,6 +700,35 @@ _OVER: */ } +static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { + int32_t code = 0; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj"); + if (pTrans == NULL) { + code = terrno; + goto _exit; + } + + pDnode->updateTime = taosGetTimestampMs(); + + SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + code = terrno; + goto _exit; + } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + code = terrno; + goto _exit; + } + +_exit: + mndTransDrop(pTrans); + return code; +} + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; @@ -876,6 +900,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode->memTotal = statusReq.memTotal; if (pDnode->machineId[0] == 0 && statusReq.machineId[0] != 0) { tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1); + mndUpdateDnodeObj(pMnode, pDnode); } SStatusRsp statusRsp = {0}; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 5eb0b67884..9a7d0f4839 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -969,7 +969,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt } _exit: - for (int32_t i = 0; i < TARRAY_SIZE(pTagVals); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)TARRAY_GET_ELEM(pTagVals, i); if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFreeClear(p->pData); From 7007af6fe5b113350fa791524f2273aa5f162c59 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 18 Mar 2024 18:25:14 +0800 Subject: [PATCH 3/5] enh: naming of variables --- source/common/src/systable.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cc84f7f45b..1994dc4d80 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -348,20 +348,20 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, }; -static const SSysDbTableSchema useGrantsFullSchema[] = { +static const SSysDbTableSchema userGrantsFullSchema[] = { {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; -static const SSysDbTableSchema useGrantsLogsSchema[] = { +static const SSysDbTableSchema userGrantsLogsSchema[] = { {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; -static const SSysDbTableSchema useMachinesSchema[] = { +static const SSysDbTableSchema userMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, #ifndef TD_UNIQ_GRANT {.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, @@ -398,9 +398,9 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false}, {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false}, - {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true}, - {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true}, - {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true}, + {TSDB_INS_TABLE_GRANTS_FULL, userGrantsFullSchema, tListLen(userGrantsFullSchema), true}, + {TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true}, + {TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { From 7df12ffe500744ac41e2d023442411ec8b8eecaa Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 19 Mar 2024 08:40:48 +0800 Subject: [PATCH 4/5] fix: update machine id --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index dc1aa2e1ef..6030c40789 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -898,7 +898,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode->numOfDiskCfg = statusReq.numOfDiskCfg; pDnode->memAvail = statusReq.memAvail; pDnode->memTotal = statusReq.memTotal; - if (pDnode->machineId[0] == 0 && statusReq.machineId[0] != 0) { + if (memcmp(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN) != 0) { tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1); mndUpdateDnodeObj(pMnode, pDnode); } From 3da55a48ca1973ed264a25ed338b1b17f95ff341 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 19 Mar 2024 11:03:30 +0800 Subject: [PATCH 5/5] enh: code optimization --- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6030c40789..84fe10c586 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -231,9 +231,9 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { if (sver > 1) { int16_t keyLen = 0; SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER) - if (keyLen > 0) SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) + SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) SDB_GET_INT16(pRaw, dataPos, &keyLen, _OVER) - if (keyLen > 0) SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) + SDB_GET_BINARY(pRaw, dataPos, NULL, keyLen, _OVER) } terrno = 0;