From efff5e2bf9181cd483d3e76b29b3608640ca8c52 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 15 Aug 2024 16:06:12 +0800 Subject: [PATCH] feat: s3 use global s3BucketName --- source/common/src/cos.c | 24 ++++++++++++------------ source/common/src/tglobal.c | 20 ++++++++++++-------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index aa587bf07b..8392b0564a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -9,7 +9,7 @@ extern int8_t tsS3EpNum; extern char tsS3Endpoint[][TSDB_FQDN_LEN]; extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; -extern char tsS3BucketName[][TSDB_FQDN_LEN]; +extern char tsS3BucketName[TSDB_FQDN_LEN]; extern char tsS3AppId[][TSDB_FQDN_LEN]; extern char tsS3Hostname[][TSDB_FQDN_LEN]; extern int8_t tsS3Https; @@ -130,13 +130,13 @@ int32_t s3CheckCfg() { (void)fprintf(stderr, "put object %s: success.\n\n", objectname[0]); // list buckets - (void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName[i]); - code = s3ListBucketByEp(tsS3BucketName[i], i); + (void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName); + code = s3ListBucketByEp(tsS3BucketName, i); if (code != 0) { - (void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName[i]); + (void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); TAOS_CHECK_GOTO(code, &lino, _exit); } - (void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName[i]); + (void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); // test range get uint8_t *pBlock = NULL; @@ -975,7 +975,7 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8 contentLength; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1058,7 +1058,7 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec contentLength; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1154,7 +1154,7 @@ static void s3FreeObjectKey(void *pItem) { static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) { S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1222,7 +1222,7 @@ static int32_t s3DeleteObjectsByEp(const char *object_name[], int nobject, int8_ int32_t code = 0; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1298,7 +1298,7 @@ static int32_t s3GetObjectBlockByEp(const char *object_name, int64_t offset, int const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1371,7 +1371,7 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa const char *ifMatch = 0, *ifNotMatch = 0; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], @@ -1448,7 +1448,7 @@ static long s3SizeByEp(const char *object_name, int8_t epIndex) { int status = 0; S3BucketContext bucketContext = {tsS3Hostname[epIndex], - tsS3BucketName[epIndex], + tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId[epIndex], diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4e32288123..3373b56ad8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -299,7 +299,7 @@ char tsS3Endpoint[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; char tsS3AccessKey[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; char tsS3AccessKeyId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; char tsS3AccessKeySecret[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; -char tsS3BucketName[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; +char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; int8_t tsS3Enabled = false; int8_t tsS3EnabledCfg = false; @@ -404,10 +404,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } TAOS_CHECK_RETURN(taosSplitS3Cfg(pCfg, "s3Endpoint", tsS3Endpoint, &num)); - if (num != tsS3EpNum) TAOS_RETURN(TSDB_CODE_INVALID_CFG); + if (num != tsS3EpNum) { + uError("invalid s3 ep num:%d, expected:%d, ", num, tsS3EpNum); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } - TAOS_CHECK_RETURN(taosSplitS3Cfg(pCfg, "s3BucketName", tsS3BucketName, &num)); - if (num != tsS3EpNum) TAOS_RETURN(TSDB_CODE_INVALID_CFG); + SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3BucketName"); + tstrncpy(tsS3BucketName, pItem->str, TSDB_FQDN_LEN); for (int i = 0; i < tsS3EpNum; ++i) { char *proto = strstr(tsS3Endpoint[i], "https://"); @@ -419,9 +423,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { char *cos = strstr(tsS3Endpoint[i], "cos."); if (cos) { - char *appid = strrchr(tsS3BucketName[i], '-'); + char *appid = strrchr(tsS3BucketName, '-'); if (!appid) { - uError("failed to locate appid in bucket:%s", tsS3BucketName[i]); + uError("failed to locate appid in bucket:%s", tsS3BucketName); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } else { tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN); @@ -432,7 +436,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL); tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL); - if (tsS3BucketName[0][0] != '<') { + if (tsS3BucketName[0] != '<') { #if defined(USE_COS) || defined(USE_S3) #ifdef TD_ENTERPRISE /*if (tsDiskCfgNum > 1) */ tsS3Enabled = true; @@ -818,7 +822,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));