xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #23733 from taosdata/fix/TD-27358 

fix(cos/log): dump file name & line no
This commit is contained in:
Hongze Cheng 2023-11-17 17:32:50 +08:00 committed by GitHub
parent 73445d53b1 abcd36b76d
commit f14ac872c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
} }
} }
stage('linux test') { stage('linux test') {
agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() } options { skipDefaultCheckout() }
when { when {
changeRequest() changeRequest()

4
source/common/CMakeLists.txt
View File

@ -60,8 +60,8 @@ if(${BUILD_S3})
find_library(S3_LIBRARY s3) find_library(S3_LIBRARY s3)
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(XML2_LIBRARY xml2) find_library(XML2_LIBRARY xml2)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
target_link_libraries( target_link_libraries(
common common

38
source/common/src/cos.c
View File

@ -66,11 +66,12 @@ static int should_retry() {
return 0; return 0;
} }
static void s3PrintError(const char *func, S3Status status, char error_details[]) { static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
char error_details[]) {
if (status < S3StatusErrorAccessDenied) { if (status < S3StatusErrorAccessDenied) {
uError("%s: %s", __func__, S3_get_status_name(status)); uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status));
} else { } else {
uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details);
} }
} }
@ -230,9 +231,9 @@ typedef struct put_object_callback_data {
int noStatus; int noStatus;
} put_object_callback_data; } put_object_callback_data;
#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M #define MULTIPART_CHUNK_SIZE (64 << 20) // multipart is 768M
typedef struct UploadManager { typedef struct {
char err_msg[512]; char err_msg[512];
S3Status status; S3Status status;
uint64_t content_length; uint64_t content_length;
@ -307,6 +308,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat
S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
UploadManager *manager = (UploadManager *)callbackData; UploadManager *manager = (UploadManager *)callbackData;
manager->upload_id = strdup(upload_id); manager->upload_id = strdup(upload_id);
manager->status = S3StatusOK;
return S3StatusOK; return S3StatusOK;
} }
@ -437,7 +439,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
// printListMultipartHeader(data.allDetails); // printListMultipartHeader(data.allDetails);
} }
} else { } else {
s3PrintError(__func__, data.status, data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
return -1; return -1;
} }
@ -498,7 +500,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} while (S3_status_is_retryable(data.status) && should_retry()); } while (S3_status_is_retryable(data.status) && should_retry());
if (data.status != S3StatusOK) { if (data.status != S3StatusOK) {
s3PrintError(__func__, data.status, data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} else if (data.contentLength) { } else if (data.contentLength) {
uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
@ -508,15 +510,15 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} else { } else {
uint64_t totalContentLength = contentLength; uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength; uint64_t todoContentLength = contentLength;
UploadManager manager; UploadManager manager = {0};
manager.upload_id = 0; // manager.upload_id = 0;
manager.gb = 0; // manager.gb = 0;
// div round up // div round up
int seq; int seq;
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size; int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 1000; const int max_part_num = 10000;
if (totalSeq > max_part_num) { if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size; totalSeq = (contentLength + chunk_size - 1) / chunk_size;
@ -556,7 +558,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) { if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
goto clean; goto clean;
} }
@ -581,7 +583,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
partContentLength, 0, timeoutMsG, &partData); partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) { if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
goto clean; goto clean;
} }
@ -609,7 +611,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
timeoutMsG, &manager); timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) { if (manager.status != S3StatusOK) {
s3PrintError(__func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
goto clean; goto clean;
} }
@ -722,7 +724,7 @@ static SArray *getListByPrefix(const char *prefix) {
return data.objectArray; return data.objectArray;
} }
} else { } else {
s3PrintError(__func__, data.status, data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
} }
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
@ -741,7 +743,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
} while (S3_status_is_retryable(cbd.status) && should_retry()); } while (S3_status_is_retryable(cbd.status) && should_retry());
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
} }
} }
} }
@ -884,7 +886,7 @@ long s3Size(const char *object_name) {
} while (S3_status_is_retryable(cbd.status) && should_retry()); } while (S3_status_is_retryable(cbd.status) && should_retry());
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
} }
size = cbd.content_length; size = cbd.content_length;