prefix/list: list bucket with prefix

cmake/libs3_CMakeLists.txt.in

@@ -13,7 +13,7 @@ endfunction(update_cflags)
 ExternalProject_Add(libs3
         GIT_REPOSITORY https://github.com/bji/libs3
 	#GIT_TAG v5.0.16
-	DEPENDS curl xml2
+	DEPENDS curl xml2 openssl
         SOURCE_DIR "${TD_CONTRIB_DIR}/libs3"
         #BINARY_DIR ""
         BUILD_IN_SOURCE TRUE

contrib/CMakeLists.txt

@@ -14,7 +14,6 @@ set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
 configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
     file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
-    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
 
 configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -192,6 +191,7 @@ if(${BUILD_WITH_S3})
     #cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     #INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
 
+    cat("${TD_SUPPORT_DIR}/ssl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
     cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})

source/dnode/vnode/CMakeLists.txt

@@ -181,11 +181,11 @@ if(${BUILD_WITH_S3})
     vnode
 
     # s3
-    PUBLIC ${S3_LIBRARY}
+    PRIVATE ${S3_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
+    PRIVATE ${CURL_LIBRARY}
-    PUBLIC ${SSL_LIBRARY}
+    PRIVATE ${SSL_LIBRARY}
-    PUBLIC ${CRYPTO_LIBRARY}
+    PRIVATE ${CRYPTO_LIBRARY}
-    PUBLIC xml2
+    PRIVATE xml2
   )
 
   add_definitions(-DUSE_S3)

source/dnode/vnode/src/vnd/vnodeCos.c

@@ -63,9 +63,17 @@ static int should_retry() {
   return 0;
 }
 
+static void s3PrintError(const char *func, S3Status status, char error_details[]) {
+  if (status < S3StatusErrorAccessDenied) {
+    vError("%s: %s", __func__, S3_get_status_name(status));
+  } else {
+    vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+  }
+}
+
 typedef struct {
   uint64_t content_length;
-  int      status;
+  S3Status status;
   char    *buf;
   char     err_msg[4096];
 } TS3SizeCBD;
@@ -109,7 +117,86 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
 }
 
 int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
-void    s3DeleteObjectsByPrefix(const char *prefix) {}
+
+typedef struct list_bucket_callback_data {
+  int      isTruncated;
+  char     nextMarker[1024];
+  int      keyCount;
+  int      allDetails;
+  S3Status status;
+  char     err_msg[4096];
+} list_bucket_callback_data;
+
+static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
+                                   const S3ListBucketContent *contents, int commonPrefixesCount,
+                                   const char **commonPrefixes, void *callbackData) {
+  list_bucket_callback_data *data = (list_bucket_callback_data *)callbackData;
+
+  data->isTruncated = isTruncated;
+  if ((!nextMarker || !nextMarker[0]) && contentsCount) {
+    nextMarker = contents[contentsCount - 1].key;
+  }
+  if (nextMarker) {
+    snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker);
+  } else {
+    data->nextMarker[0] = 0;
+  }
+
+  if (contentsCount && !data->keyCount) {
+    // printListBucketHeader(data->allDetails);
+  }
+
+  int i;
+  for (i = 0; i < contentsCount; ++i) {
+    const S3ListBucketContent *content = &(contents[i]);
+    printf("%-50s", content->key);
+  }
+  data->keyCount += contentsCount;
+
+  for (i = 0; i < commonPrefixesCount; i++) {
+    // printf("\nCommon Prefix: %s\n", commonPrefixes[i]);
+  }
+
+  return S3StatusOK;
+}
+
+void s3DeleteObjectsByPrefix(const char *prefix) {
+  S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                       0, awsRegionG};
+  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+                                           &listBucketCallback};
+
+  const char               *marker = 0, *delimiter = 0;
+  int                       maxkeys = 0, allDetails = 0;
+  list_bucket_callback_data data;
+
+  if (marker) {
+    snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
+  } else {
+    data.nextMarker[0] = 0;
+  }
+  data.keyCount = 0;
+  data.allDetails = allDetails;
+
+  do {
+    data.isTruncated = 0;
+    do {
+      S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler,
+                     &data);
+    } while (S3_status_is_retryable(data.status) && should_retry());
+    if (data.status != S3StatusOK) {
+      break;
+    }
+  } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
+
+  if (data.status == S3StatusOK) {
+    if (!data.keyCount) {
+      // printListBucketHeader(allDetails);
+    }
+  } else {
+    s3PrintError(__func__, data.status, data.err_msg);
+  }
+}
 
 void s3DeleteObjects(const char *object_name[], int nobject) {
   int               status = 0;
@@ -124,7 +211,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
     } while (S3_status_is_retryable(cbd.status) && should_retry());
 
     if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-      vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+      s3PrintError(__func__, cbd.status, cbd.err_msg);
     }
   }
 }