xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #23733 from taosdata/fix/TD-27358 

fix(cos/log): dump file name & line no
This commit is contained in:
Hongze Cheng 2023-11-17 17:32:50 +08:00 committed by GitHub
parent 73445d53b1 abcd36b76d
commit f14ac872c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
}
}
stage('linux test') {
agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() }
when {
changeRequest()

4
source/common/CMakeLists.txt
View File

@ -60,8 +60,8 @@ if(${BUILD_S3})
find_library(S3_LIBRARY s3)
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(XML2_LIBRARY xml2)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
target_link_libraries(
common

38
source/common/src/cos.c
View File

@ -66,11 +66,12 @@ static int should_retry() {
return 0;
}
static void s3PrintError(const char *func, S3Status status, char error_details[]) {
static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
char error_details[]) {
if (status < S3StatusErrorAccessDenied) {
uError("%s: %s", __func__, S3_get_status_name(status));
uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status));
} else {
uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details);
}
}
@ -230,9 +231,9 @@ typedef struct put_object_callback_data {
int noStatus;
} put_object_callback_data;
#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M
#define MULTIPART_CHUNK_SIZE (64 << 20) // multipart is 768M
typedef struct UploadManager {
typedef struct {
char err_msg[512];
S3Status status;
uint64_t content_length;
@ -307,6 +308,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat
S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
UploadManager *manager = (UploadManager *)callbackData;
manager->upload_id = strdup(upload_id);
manager->status = S3StatusOK;
return S3StatusOK;
}
@ -437,7 +439,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
// printListMultipartHeader(data.allDetails);
}
} else {
s3PrintError(__func__, data.status, data.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
return -1;
}
@ -498,7 +500,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} while (S3_status_is_retryable(data.status) && should_retry());
if (data.status != S3StatusOK) {
s3PrintError(__func__, data.status, data.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
} else if (data.contentLength) {
uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
@ -508,15 +510,15 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} else {
uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength;
UploadManager manager;
manager.upload_id = 0;
manager.gb = 0;
UploadManager manager = {0};
// manager.upload_id = 0;
// manager.gb = 0;
// div round up
int seq;
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7;
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
const int max_part_num = 1000;
const int max_part_num = 10000;
if (totalSeq > max_part_num) {
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
@ -556,7 +558,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__func__, manager.status, manager.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
@ -581,7 +583,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
partContentLength, 0, timeoutMsG, &partData);
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
@ -609,7 +611,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
timeoutMsG, &manager);
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__func__, manager.status, manager.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
}
@ -722,7 +724,7 @@ static SArray *getListByPrefix(const char *prefix) {
return data.objectArray;
}
} else {
s3PrintError(__func__, data.status, data.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
}
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
@ -741,7 +743,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
} while (S3_status_is_retryable(cbd.status) && should_retry());
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__func__, cbd.status, cbd.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
}
}
}
@ -884,7 +886,7 @@ long s3Size(const char *object_name) {
} while (S3_status_is_retryable(cbd.status) && should_retry());
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
}
size = cbd.content_length;