From 60ad82d006188f32f8702620108b1ef39e13f8cf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 09:38:48 +0800 Subject: [PATCH 1/5] fix(cos/log): dump file name & line no --- source/common/CMakeLists.txt | 4 ++-- source/common/src/cos.c | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index c7831b2b6d..d3df1345df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -60,8 +60,8 @@ if(${BUILD_S3}) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) find_library(XML2_LIBRARY xml2) - find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) target_link_libraries( common diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 4cfc67fa94..fd462d5a11 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -66,11 +66,12 @@ static int should_retry() { return 0; } -static void s3PrintError(const char *func, S3Status status, char error_details[]) { +static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, + char error_details[]) { if (status < S3StatusErrorAccessDenied) { - uError("%s: %s", __func__, S3_get_status_name(status)); + uError("%s/%s:%d-%s: %s", filename, lineno, __func__, funcname, S3_get_status_name(status)); } else { - uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s/%s:%d-%s: %s, %s", filename, lineno, __func__, funcname, S3_get_status_name(status), error_details); } } @@ -437,7 +438,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan // printListMultipartHeader(data.allDetails); } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); return -1; } @@ -498,7 +499,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(data.status) && should_retry()); if (data.status != S3StatusOK) { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, @@ -556,7 +557,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.upload_id == 0 || manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -581,7 +582,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { partContentLength, 0, timeoutMsG, &partData); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { - s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -609,7 +610,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { timeoutMsG, &manager); } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -722,7 +723,7 @@ static SArray *getListByPrefix(const char *prefix) { return data.objectArray; } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); } taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); @@ -741,7 +742,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - s3PrintError(__func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } } } From 9d3ffa2c2cc9104599d18653f54f00a2de031609 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 12:24:51 +0800 Subject: [PATCH 2/5] cos/print-error: fix error format --- source/common/src/cos.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index fd462d5a11..684f437d05 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -69,9 +69,9 @@ static int should_retry() { static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, char error_details[]) { if (status < S3StatusErrorAccessDenied) { - uError("%s/%s:%d-%s: %s", filename, lineno, __func__, funcname, S3_get_status_name(status)); + uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status)); } else { - uError("%s/%s:%d-%s: %s, %s", filename, lineno, __func__, funcname, S3_get_status_name(status), error_details); + uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details); } } @@ -885,7 +885,7 @@ long s3Size(const char *object_name) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } size = cbd.content_length; From 8038fbaa047229f5472997021ae91645f27f90e1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 14:58:03 +0800 Subject: [PATCH 3/5] cos/multi: clear manager to init --- source/common/src/cos.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 684f437d05..aecba57e0a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -233,7 +233,7 @@ typedef struct put_object_callback_data { #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M -typedef struct UploadManager { +typedef struct { char err_msg[512]; S3Status status; uint64_t content_length; @@ -308,6 +308,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); + manager->status = S3StatusOK; return S3StatusOK; } @@ -509,15 +510,15 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } else { uint64_t totalContentLength = contentLength; uint64_t todoContentLength = contentLength; - UploadManager manager; - manager.upload_id = 0; - manager.gb = 0; + UploadManager manager = {0}; + // manager.upload_id = 0; + // manager.gb = 0; // div round up int seq; uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; int totalSeq = (contentLength + chunk_size - 1) / chunk_size; - const int max_part_num = 1000; + const int max_part_num = 10000; if (totalSeq > max_part_num) { chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; totalSeq = (contentLength + chunk_size - 1) / chunk_size; From 56badd0db07891b4aeb9b423068dced0252c8dbf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 15:12:40 +0800 Subject: [PATCH 4/5] cos/chunk-size: reduce to 64m from 768m --- source/common/src/cos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index aecba57e0a..b4e654c67a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -231,7 +231,7 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M +#define MULTIPART_CHUNK_SIZE (64 << 20) // multipart is 768M typedef struct { char err_msg[512]; @@ -516,7 +516,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // div round up int seq; - uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; int totalSeq = (contentLength + chunk_size - 1) / chunk_size; const int max_part_num = 10000; if (totalSeq > max_part_num) { From abcd36b76de5f47da88a69947a8ea30d1365bb87 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 16:26:22 +0800 Subject: [PATCH 5/5] jenkins: remove 50 --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49d5f8884b..8ddee6dbbd 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest()