From 4a77a49a3e3c38d6519390eb6d00df15496a9d9b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Oct 2023 15:25:48 +0800 Subject: [PATCH] prefix/list: list bucket with prefix --- cmake/libs3_CMakeLists.txt.in | 2 +- contrib/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 10 +-- source/dnode/vnode/src/vnd/vnodeCos.c | 93 ++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/cmake/libs3_CMakeLists.txt.in b/cmake/libs3_CMakeLists.txt.in index 93d0eff41a..790a6fe312 100644 --- a/cmake/libs3_CMakeLists.txt.in +++ b/cmake/libs3_CMakeLists.txt.in @@ -13,7 +13,7 @@ endfunction(update_cflags) ExternalProject_Add(libs3 GIT_REPOSITORY https://github.com/bji/libs3 #GIT_TAG v5.0.16 - DEPENDS curl xml2 + DEPENDS curl xml2 openssl SOURCE_DIR "${TD_CONTRIB_DIR}/libs3" #BINARY_DIR "" BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 24ad63db13..63a60fa59a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3") configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/) - cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -192,6 +191,7 @@ if(${BUILD_WITH_S3}) #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include) + cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e554d94e4d..d4ea90f48a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -181,11 +181,11 @@ if(${BUILD_WITH_S3}) vnode # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PRIVATE ${S3_LIBRARY} + PRIVATE ${CURL_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE xml2 ) add_definitions(-DUSE_S3) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/dnode/vnode/src/vnd/vnodeCos.c index 8984d33ad8..cf19a08813 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/dnode/vnode/src/vnd/vnodeCos.c @@ -63,9 +63,17 @@ static int should_retry() { return 0; } +static void s3PrintError(const char *func, S3Status status, char error_details[]) { + if (status < S3StatusErrorAccessDenied) { + vError("%s: %s", __func__, S3_get_status_name(status)); + } else { + vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + } +} + typedef struct { uint64_t content_length; - int status; + S3Status status; char *buf; char err_msg[4096]; } TS3SizeCBD; @@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; } -void s3DeleteObjectsByPrefix(const char *prefix) {} + +typedef struct list_bucket_callback_data { + int isTruncated; + char nextMarker[1024]; + int keyCount; + int allDetails; + S3Status status; + char err_msg[4096]; +} list_bucket_callback_data; + +static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount, + const S3ListBucketContent *contents, int commonPrefixesCount, + const char **commonPrefixes, void *callbackData) { + list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData; + + data->isTruncated = isTruncated; + if ((!nextMarker || !nextMarker[0]) && contentsCount) { + nextMarker = contents[contentsCount - 1].key; + } + if (nextMarker) { + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); + } else { + data->nextMarker[0] = 0; + } + + if (contentsCount && !data->keyCount) { + // printListBucketHeader(data->allDetails); + } + + int i; + for (i = 0; i < contentsCount; ++i) { + const S3ListBucketContent *content = &(contents[i]); + printf("%-50s", content->key); + } + data->keyCount += contentsCount; + + for (i = 0; i < commonPrefixesCount; i++) { + // printf("\nCommon Prefix: %s\n", commonPrefixes[i]); + } + + return S3StatusOK; +} + +void s3DeleteObjectsByPrefix(const char *prefix) { + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + &listBucketCallback}; + + const char *marker = 0, *delimiter = 0; + int maxkeys = 0, allDetails = 0; + list_bucket_callback_data data; + + if (marker) { + snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); + } else { + data.nextMarker[0] = 0; + } + data.keyCount = 0; + data.allDetails = allDetails; + + do { + data.isTruncated = 0; + do { + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, + &data); + } while (S3_status_is_retryable(data.status) && should_retry()); + if (data.status != S3StatusOK) { + break; + } + } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + + if (data.status == S3StatusOK) { + if (!data.keyCount) { + // printListBucketHeader(allDetails); + } + } else { + s3PrintError(__func__, data.status, data.err_msg); + } +} void s3DeleteObjects(const char *object_name[], int nobject) { int status = 0; @@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__func__, cbd.status, cbd.err_msg); } } }