From 9f7666ace58290c1a26761d2d0541f11a3e90b46 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 15 Nov 2023 12:13:49 +0800 Subject: [PATCH 1/3] fix(cos/multipart): close infile fd in exception --- source/common/src/cos.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 71f0df5ebe..0dbe9c1286 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -497,12 +497,6 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); } while (S3_status_is_retryable(data.status) && should_retry()); - if (data.infileFD) { - taosCloseFile(&data.infileFD); - } else if (data.gb) { - growbuffer_destroy(data.gb); - } - if (data.status != S3StatusOK) { s3PrintError(__func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); @@ -622,6 +616,12 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { for (i = 0; i < manager.next_etags_pos; i++) { taosMemoryFree(manager.etags[i]); } + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + growbuffer_destroy(manager.gb); taosMemoryFree(manager.etags); } From 2a49b4351d828f3ff79fdc98e0eceedf90dca3d6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 15 Nov 2023 13:40:39 +0800 Subject: [PATCH 2/3] cos/multipart: limit max part to 1k --- source/common/src/cos.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 0dbe9c1286..db95fb02bb 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -513,9 +513,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { manager.gb = 0; // div round up - int seq; - uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; - int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 1000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); From 3b45a81eee092fd33563ce1aee25ae4c9dd4473b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 15 Nov 2023 14:41:00 +0800 Subject: [PATCH 3/3] cos/list: destroy empty array --- source/common/src/cos.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index db95fb02bb..f36ab42ada 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -722,9 +722,10 @@ static SArray *getListByPrefix(const char *prefix) { return data.objectArray; } } else { - taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); s3PrintError(__func__, data.status, data.err_msg); } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); return NULL; }