From b2ad162a8b770032594020bd18f600f57b761610 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Feb 2024 17:13:58 +0800 Subject: [PATCH 1/7] enh: remove obsolete codes --- include/common/tmsg.h | 1 - source/dnode/mnode/impl/src/mndDnode.c | 3 --- 2 files changed, 4 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b438ba026..87d8ed76a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1613,7 +1613,6 @@ typedef struct { SEp ep; char active[TSDB_ACTIVE_KEY_LEN]; char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; - char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeInfo; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cb8a24e675..6974c6b0e8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -410,9 +410,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.ep.port = pDnode->port; dInfo.offlineReason = pDnode->offlineReason; tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN); - tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN); - tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1); sdbRelease(pSdb, pDnode); if (mndIsMnode(pMnode, pDnode->id)) { dInfo.isMnode = 1; From c7932675ce271bb98506cea5a1be7600c0583304 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 21 Feb 2024 09:07:38 +0800 Subject: [PATCH 2/7] enh: get machine id --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 7 +------ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 11 +++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a3a29f6f77..fa59f56496 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "systable.h" -#include "tgrant.h" extern SConfig *tsCfg; @@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.memTotal = tsTotalMemoryKB * 1024; req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024; tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN); - char *machine = tGetMachineId(); - if (machine) { - tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1); - taosMemoryFreeClear(machine); - } + tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1); req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..049b1bdf84 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#include "tgrant.h" #endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { @@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) { pData->rebootTime = taosGetTimestampMs(); pData->dropped = 0; pData->stopped = 0; + char *machineId = tGetMachineId(); + if (machineId) { + tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1); + taosMemoryFreeClear(machineId); + } else { +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) + terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; + return -1; +#endif + } pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4769ef8538..504c405506 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -109,6 +109,7 @@ typedef struct { SMsgCb msgCb; bool validMnodeEps; int64_t ipWhiteVer; + char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeData; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6974c6b0e8..c65c228224 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -141,7 +141,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN); taosMemoryFreeClear(machineId); } else { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; goto _OVER; #endif From 6ac6e8f2bd382c2e3b41dd604e43f197135a770c Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 17:46:36 +0800 Subject: [PATCH 3/7] enh: unit test and optimization for base58 --- include/util/tbase58.h | 3 ++ source/util/src/tbase58.c | 34 +++++++++--- source/util/test/CMakeLists.txt | 8 +++ source/util/test/tbaseCodecTest.cpp | 80 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 source/util/test/tbaseCodecTest.cpp diff --git a/include/util/tbase58.h b/include/util/tbase58.h index e1b03f8a8f..ab62e1926e 100644 --- a/include/util/tbase58.h +++ b/include/util/tbase58.h @@ -22,6 +22,9 @@ extern "C" { #endif +#define TBASE_MAX_ILEN 4096 +#define TBASE_MAX_OLEN 5653 + uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen); char *base58_encode(const uint8_t *value, int32_t vlen); diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index fa3ecadd14..e1cee72b47 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -18,24 +18,29 @@ #include #include -#define BASE_BUF_SIZE 256 +#define TBASE_BUF_SIZE 256 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pb = value; const uint8_t *pe = pb + vlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (vlen > TBASE_MAX_ILEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + while (pb != pe && *pb == 0) { ++pb; ++nz; } size = (pe - pb) * 69 / 50 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { int32_t num = *pb; int32_t i = 0; for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) { - num += ((int32_t)buf[j]) << 8; + num += ((int32_t)pbuf[j]) << 8; pbuf[j] = num % 58; num /= 58; } @@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pi = pbuf + (size - len); while (pi != pbuf + size && *pi == 0) ++pi; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!result) { terrno = TSDB_CODE_OUT_OF_MEMORY; if (bfree) taosMemoryFree(pbuf); @@ -83,11 +88,23 @@ static const signed char index_58[256] = { uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const char *pe = value + inlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (inlen > TBASE_MAX_OLEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + for (int32_t i = 0; i < inlen; ++i) { + if (value[i] == 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } + while (*value && isspace(*value)) ++value; while (*value == '1') { ++nz; @@ -95,7 +112,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { } size = (int32_t)(pe - value) * 733 / 1000 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -106,6 +123,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { while (*value && !isspace(*value)) { int32_t num = index_58[(uint8_t)*value]; if (num == -1) { + terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); return NULL; } @@ -127,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, inlen + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index f4f3880388..5e32bd82b7 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -100,3 +100,11 @@ add_test( NAME talgoTest COMMAND talgoTest ) + +# tbaseCodecTest +add_executable(tbaseCodecTest "tbaseCodecTest.cpp") +target_link_libraries(tbaseCodecTest os util common gtest_main) +add_test( + NAME tbaseCodecTest + COMMAND tbaseCodecTest + diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp new file mode 100644 index 0000000000..70639e7ec9 --- /dev/null +++ b/source/util/test/tbaseCodecTest.cpp @@ -0,0 +1,80 @@ +#include +#include + +#include +#include "os.h" +#include "taos.h" +#include "taoserror.h" +#include "tbase58.h" +#include "tbase64.h" +#include "tglobal.h" + +using namespace std; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + ASSERT_NE(nullptr, pEnc); + + int32_t encLen = strlen(pEnc); + std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int32_t decLen = 0; + char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); + std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + ASSERT_NE(nullptr, pDec); + ASSERT_EQ(rawLen, decLen); + ASSERT_LE(rawLen, encLen); + ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen)); + taosMemoryFreeClear(pDec); + taosMemoryFreeClear(pEnc); +} + +TEST(TD_BASE_CODEC_TEST, tbase58_test) { + const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN; + const int32_t TEST_LEN_STEP = 10; + int32_t rawLen = 0; + uint8_t *pRaw = NULL; + + pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX); + ASSERT_NE(nullptr, pRaw); + + // 1. normal case + // string blend with char and '\0' + rawLen = TEST_LEN_MAX; + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + checkBase58Codec(pRaw, rawLen, i); + pRaw[i] = i & 127; + } + + // string without '\0' + for (int32_t i = 0; i < TEST_LEN_MAX; ++i) { + pRaw[i] = i & 127; + } + checkBase58Codec(pRaw, TEST_LEN_MAX, 0); + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + rawLen = i; + checkBase58Codec(pRaw, rawLen, i); + } + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); + + // 2. overflow case + char tmp[1]; + char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1); + ASSERT_EQ(nullptr, pEnc); + char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL); + ASSERT_EQ(nullptr, pDec); + + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); +} \ No newline at end of file From 5b0fc1544d08e74487deca8f5300bb4e99ba23f7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 18:00:12 +0800 Subject: [PATCH 4/7] enh: unit test and optimization for base58 --- source/util/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 5e32bd82b7..3514c578e9 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -107,4 +107,4 @@ target_link_libraries(tbaseCodecTest os util common gtest_main) add_test( NAME tbaseCodecTest COMMAND tbaseCodecTest - +) From 75caea4fbf6b47a17b87528960f69ce5d79e40d9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 19:04:21 +0800 Subject: [PATCH 5/7] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index e1cee72b47..2566d54431 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -145,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, inlen + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; From 31cf877f50c6b2221daa48b3e252323b81588938 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:16:27 +0800 Subject: [PATCH 6/7] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index 2566d54431..5eb72879c3 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -87,6 +87,7 @@ static const signed char index_58[256] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { + const char *pb = value; const char *pe = value + inlen; uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; @@ -98,20 +99,22 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { return NULL; } - for (int32_t i = 0; i < inlen; ++i) { - if (value[i] == 0) { + while (pb != pe) { + if (*pb == 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } + ++pb; } - while (*value && isspace(*value)) ++value; - while (*value == '1') { + pb = value; + while (pb != pe && *pb && isspace(*pb)) ++pb; + while (pb != pe && *pb == '1') { ++nz; - ++value; + ++pb; } - size = (int32_t)(pe - value) * 733 / 1000 + 1; + size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -120,8 +123,8 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { bfree = true; } - while (*value && !isspace(*value)) { - int32_t num = index_58[(uint8_t)*value]; + while (pb != pe && *pb && !isspace(*pb)) { + int32_t num = index_58[(uint8_t)*pb]; if (num == -1) { terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); @@ -134,11 +137,11 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { num >>= 8; } len = i; - ++value; + ++pb; } - while (isspace(*value)) ++value; - if (*value != 0) { + while (pb != pe && isspace(*pb)) ++pb; + if (*pb != 0) { if (bfree) taosMemoryFree(pbuf); return NULL; } From ef7b38d4f69257df739a6e602deca68482e4bd81 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:30:47 +0800 Subject: [PATCH 7/7] enh: code optimization for tbase58 --- source/util/test/tbaseCodecTest.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp index 70639e7ec9..4c56979885 100644 --- a/source/util/test/tbaseCodecTest.cpp +++ b/source/util/test/tbaseCodecTest.cpp @@ -3,10 +3,10 @@ #include #include "os.h" +#include "osTime.h" #include "taos.h" #include "taoserror.h" #include "tbase58.h" -#include "tbase64.h" #include "tglobal.h" using namespace std; @@ -23,14 +23,17 @@ int main(int argc, char **argv) { } static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { - char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + int64_t start = taosGetTimestampUs(); + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); ASSERT_NE(nullptr, pEnc); int32_t encLen = strlen(pEnc); - std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int64_t endOfEnc = taosGetTimestampUs(); + std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl; int32_t decLen = 0; char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); - std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us" + << std::endl; ASSERT_NE(nullptr, pDec); ASSERT_EQ(rawLen, decLen); ASSERT_LE(rawLen, encLen); @@ -51,7 +54,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { // 1. normal case // string blend with char and '\0' rawLen = TEST_LEN_MAX; - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { checkBase58Codec(pRaw, rawLen, i); pRaw[i] = i & 127; } @@ -61,7 +64,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { pRaw[i] = i & 127; } checkBase58Codec(pRaw, TEST_LEN_MAX, 0); - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { rawLen = i; checkBase58Codec(pRaw, rawLen, i); }