xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: s3 support multi proto

This commit is contained in:
Shungang Li 2024-08-19 18:39:08 +08:00
parent 55f7b7d771
commit e298b5acb3
2 changed files with 27 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
source/common/src/cos.c
View File

@ -12,7 +12,7 @@ extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[TSDB_FQDN_LEN]; extern char tsS3BucketName[TSDB_FQDN_LEN];
extern char tsS3AppId[][TSDB_FQDN_LEN]; extern char tsS3AppId[][TSDB_FQDN_LEN];
extern char tsS3Hostname[][TSDB_FQDN_LEN]; extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern int8_t tsS3Https; extern int8_t tsS3Https[];
static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex); static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex);
static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size,
@ -33,13 +33,13 @@ static int verifyPeerG = 0;
static const char *awsRegionG = NULL; static const char *awsRegionG = NULL;
static int forceG = 0; static int forceG = 0;
static int showResponsePropertiesG = 0; static int showResponsePropertiesG = 0;
static S3Protocol protocolG = S3ProtocolHTTPS; static S3Protocol protocolG[TSDB_MAX_EP_NUM] = {S3ProtocolHTTPS};
// static S3Protocol protocolG = S3ProtocolHTTP; // static S3Protocol protocolG = S3ProtocolHTTP;
static S3UriStyle uriStyleG = S3UriStylePath; static S3UriStyle uriStyleG[TSDB_MAX_EP_NUM] = {S3UriStylePath};
static int retriesG = 5; static int retriesG = 5;
static int timeoutMsG = 0; static int timeoutMsG = 0;
extern int8_t tsS3Oss; extern int8_t tsS3Oss[];
int32_t s3Begin() { int32_t s3Begin() {
S3Status status; S3Status status;
@ -55,9 +55,11 @@ int32_t s3Begin() {
TAOS_RETURN(TSDB_CODE_FAILED); TAOS_RETURN(TSDB_CODE_FAILED);
} }
protocolG = !tsS3Https; for (int i = 0; i < tsS3EpNum; i++) {
if (tsS3Oss) { protocolG[i] = !tsS3Https[i];
uriStyleG = S3UriStyleVirtualHost; if (tsS3Oss[i]) {
uriStyleG[i] = S3UriStyleVirtualHost;
}
} }
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_SUCCESS);
@ -976,8 +978,8 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1059,8 +1061,8 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1155,8 +1157,8 @@ static void s3FreeObjectKey(void *pItem) {
static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) { static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) {
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1223,8 +1225,8 @@ static int32_t s3DeleteObjectsByEp(const char *object_name[], int nobject, int8_
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1299,8 +1301,8 @@ static int32_t s3GetObjectBlockByEp(const char *object_name, int64_t offset, int
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1372,8 +1374,8 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,
@ -1449,8 +1451,8 @@ static long s3SizeByEp(const char *object_name, int8_t epIndex) {
S3BucketContext bucketContext = {tsS3Hostname[epIndex], S3BucketContext bucketContext = {tsS3Hostname[epIndex],
tsS3BucketName, tsS3BucketName,
protocolG, protocolG[epIndex],
uriStyleG, uriStyleG[epIndex],
tsS3AccessKeyId[epIndex], tsS3AccessKeyId[epIndex],
tsS3AccessKeySecret[epIndex], tsS3AccessKeySecret[epIndex],
0, 0,

9
source/common/src/tglobal.c
View File

@ -303,10 +303,10 @@ char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"}; char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"};
int8_t tsS3Enabled = false; int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false; int8_t tsS3EnabledCfg = false;
int8_t tsS3Oss = false; int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false};
int8_t tsS3StreamEnabled = false; int8_t tsS3StreamEnabled = false;
int8_t tsS3Https = true; int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true};
char tsS3Hostname[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<hostname>"}; char tsS3Hostname[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<hostname>"};
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
@ -431,11 +431,10 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN); tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN);
} }
} }
tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL);
tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL);
} }
tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL);
tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL);
if (tsS3BucketName[0] != '<') { if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3) #if defined(USE_COS) || defined(USE_S3)
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE