diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49d5f8884b..8ddee6dbbd 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index c7831b2b6d..d3df1345df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -60,8 +60,8 @@ if(${BUILD_S3}) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) find_library(XML2_LIBRARY xml2) - find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) target_link_libraries( common diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 4cfc67fa94..b4e654c67a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -66,11 +66,12 @@ static int should_retry() { return 0; } -static void s3PrintError(const char *func, S3Status status, char error_details[]) { +static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, + char error_details[]) { if (status < S3StatusErrorAccessDenied) { - uError("%s: %s", __func__, S3_get_status_name(status)); + uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status)); } else { - uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details); } } @@ -230,9 +231,9 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M +#define MULTIPART_CHUNK_SIZE (64 << 20) // multipart is 768M -typedef struct UploadManager { +typedef struct { char err_msg[512]; S3Status status; uint64_t content_length; @@ -307,6 +308,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); + manager->status = S3StatusOK; return S3StatusOK; } @@ -437,7 +439,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan // printListMultipartHeader(data.allDetails); } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); return -1; } @@ -498,7 +500,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(data.status) && should_retry()); if (data.status != S3StatusOK) { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, @@ -508,15 +510,15 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } else { uint64_t totalContentLength = contentLength; uint64_t todoContentLength = contentLength; - UploadManager manager; - manager.upload_id = 0; - manager.gb = 0; + UploadManager manager = {0}; + // manager.upload_id = 0; + // manager.gb = 0; // div round up int seq; - uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; int totalSeq = (contentLength + chunk_size - 1) / chunk_size; - const int max_part_num = 1000; + const int max_part_num = 10000; if (totalSeq > max_part_num) { chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; totalSeq = (contentLength + chunk_size - 1) / chunk_size; @@ -556,7 +558,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.upload_id == 0 || manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -581,7 +583,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { partContentLength, 0, timeoutMsG, &partData); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { - s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -609,7 +611,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { timeoutMsG, &manager); } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -722,7 +724,7 @@ static SArray *getListByPrefix(const char *prefix) { return data.objectArray; } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); } taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); @@ -741,7 +743,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - s3PrintError(__func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } } } @@ -884,7 +886,7 @@ long s3Size(const char *object_name) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } size = cbd.content_length;