xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: s3 use global s3BucketName

This commit is contained in:
Shungang Li 2024-08-15 16:06:12 +08:00
parent 41d8c20117
commit efff5e2bf9
2 changed files with 24 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
source/common/src/cos.c
View File

@ -9,7 +9,7 @@ extern int8_t tsS3EpNum;
extern char tsS3Endpoint[][TSDB_FQDN_LEN]; extern char tsS3Endpoint[][TSDB_FQDN_LEN];
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[][TSDB_FQDN_LEN]; extern char tsS3BucketName[TSDB_FQDN_LEN];
extern char tsS3AppId[][TSDB_FQDN_LEN]; extern char tsS3AppId[][TSDB_FQDN_LEN];
extern char tsS3Hostname[][TSDB_FQDN_LEN]; extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern int8_t tsS3Https; extern int8_t tsS3Https;
@ -130,13 +130,13 @@ int32_t s3CheckCfg() {
(void)fprintf(stderr, "put object %s: success.\n\n", objectname[0]); (void)fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
// list buckets // list buckets
(void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName[i]); (void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName);
code = s3ListBucketByEp(tsS3BucketName[i], i); code = s3ListBucketByEp(tsS3BucketName, i);
if (code != 0) { if (code != 0) {
(void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName[i]); (void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
TAOS_CHECK_GOTO(code, &lino, _exit); TAOS_CHECK_GOTO(code, &lino, _exit);
} }
(void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName[i]); (void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
// test range get // test range get
uint8_t *pBlock = NULL; uint8_t *pBlock = NULL;
@ -975,7 +975,7 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8
contentLength; contentLength;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1058,7 +1058,7 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec
contentLength; contentLength;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1154,7 +1154,7 @@ static void s3FreeObjectKey(void *pItem) {
static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) { static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) {
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1222,7 +1222,7 @@ static int32_t s3DeleteObjectsByEp(const char *object_name[], int nobject, int8_
int32_t code = 0; int32_t code = 0;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1298,7 +1298,7 @@ static int32_t s3GetObjectBlockByEp(const char *object_name, int64_t offset, int
const char *ifMatch = 0, *ifNotMatch = 0; const char *ifMatch = 0, *ifNotMatch = 0;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1371,7 +1371,7 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa
const char *ifMatch = 0, *ifNotMatch = 0; const char *ifMatch = 0, *ifNotMatch = 0;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
@ -1448,7 +1448,7 @@ static long s3SizeByEp(const char *object_name, int8_t epIndex) {
int status = 0; int status = 0;
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName[epIndex], tsS3BucketName,
protocolG, protocolG,
uriStyleG, uriStyleG,
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],

20
source/common/src/tglobal.c
View File

@ -299,7 +299,7 @@ char tsS3Endpoint[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<endpoint>"};
char tsS3AccessKey[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskey>"}; char tsS3AccessKey[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskey>"};
char tsS3AccessKeyId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskeyid>"}; char tsS3AccessKeyId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskeyid>"};
char tsS3AccessKeySecret[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskeysecrect>"}; char tsS3AccessKeySecret[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<accesskeysecrect>"};
char tsS3BucketName[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<bucketname>"}; char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"}; char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"};
int8_t tsS3Enabled = false; int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false; int8_t tsS3EnabledCfg = false;
@ -404,10 +404,14 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
} }
TAOS_CHECK_RETURN(taosSplitS3Cfg(pCfg, "s3Endpoint", tsS3Endpoint, &num)); TAOS_CHECK_RETURN(taosSplitS3Cfg(pCfg, "s3Endpoint", tsS3Endpoint, &num));
if (num != tsS3EpNum) TAOS_RETURN(TSDB_CODE_INVALID_CFG); if (num != tsS3EpNum) {
uError("invalid s3 ep num:%d, expected:%d, ", num, tsS3EpNum);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
TAOS_CHECK_RETURN(taosSplitS3Cfg(pCfg, "s3BucketName", tsS3BucketName, &num)); SConfigItem *pItem = NULL;
if (num != tsS3EpNum) TAOS_RETURN(TSDB_CODE_INVALID_CFG); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3BucketName");
tstrncpy(tsS3BucketName, pItem->str, TSDB_FQDN_LEN);
for (int i = 0; i < tsS3EpNum; ++i) { for (int i = 0; i < tsS3EpNum; ++i) {
char *proto = strstr(tsS3Endpoint[i], "https://"); char *proto = strstr(tsS3Endpoint[i], "https://");
@ -419,9 +423,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
char *cos = strstr(tsS3Endpoint[i], "cos."); char *cos = strstr(tsS3Endpoint[i], "cos.");
if (cos) { if (cos) {
char *appid = strrchr(tsS3BucketName[i], '-'); char *appid = strrchr(tsS3BucketName, '-');
if (!appid) { if (!appid) {
uError("failed to locate appid in bucket:%s", tsS3BucketName[i]); uError("failed to locate appid in bucket:%s", tsS3BucketName);
TAOS_RETURN(TSDB_CODE_INVALID_CFG); TAOS_RETURN(TSDB_CODE_INVALID_CFG);
} else { } else {
tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN); tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN);
@ -432,7 +436,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL); tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL);
tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL); tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL);
if (tsS3BucketName[0][0] != '<') { if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3) #if defined(USE_COS) || defined(USE_S3)
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true; /*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
@ -818,7 +822,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));