From 3890def22bd7df670f594c7a1de0b84e20d1e315 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 9 Oct 2024 18:36:06 +0800 Subject: [PATCH 01/41] lib_azure: base on ms ablob sdk --- .gitignore | 1 + cmake/azure_CMakeLists.txt.in | 15 +++++++ cmake/cmake.define | 3 +- contrib/CMakeLists.txt | 9 ++++ contrib/azure-cmake/CMakeLists.txt | 70 ++++++++++++++++++++++++++++++ contrib/test/CMakeLists.txt | 1 + 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 cmake/azure_CMakeLists.txt.in create mode 100644 contrib/azure-cmake/CMakeLists.txt diff --git a/.gitignore b/.gitignore index 1798a920eb..03b30a2224 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ TAGS contrib/* !contrib/CMakeLists.txt !contrib/test +!contrib/azure-cmake sql debug*/ .env diff --git a/cmake/azure_CMakeLists.txt.in b/cmake/azure_CMakeLists.txt.in new file mode 100644 index 0000000000..5aa32b70e5 --- /dev/null +++ b/cmake/azure_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# azure +ExternalProject_Add(azure + URL https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_12.13.0-beta.1.tar.gz + URL_HASH SHA256=3eca486fd60e3522d0a633025ecd652a71515b1e944799b2e8ee31fd590305a9 + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1" + #BUILD_IN_SOURCE TRUE + #BUILD_ALWAYS 1 + #UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/cmake.define b/cmake/cmake.define index eb78b54cae..c7a58cd349 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -208,7 +208,8 @@ ELSE () IF (${BUILD_SANITIZER}) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") + #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") MESSAGE(STATUS "Compile with Address Sanitizer!") ELSEIF (${BUILD_RELEASE}) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 2a38bf74c7..d6ae5497f1 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -152,6 +152,7 @@ if(${BUILD_WITH_S3}) cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/azure_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_S3) # cos @@ -614,9 +615,17 @@ if (${BUILD_PCRE2}) add_subdirectory(pcre2 EXCLUDE_FROM_ALL) endif(${BUILD_PCRE2}) + +if(${TD_LINUX}) + add_subdirectory(azure-cmake EXCLUDE_FROM_ALL) +endif(${TD_LINUX}) + # ================================================================================================ # Build test # ================================================================================================ + +MESSAGE("build with dependency tests: ${BUILD_DEPENDENCY_TESTS}") + if(${BUILD_DEPENDENCY_TESTS}) add_subdirectory(test EXCLUDE_FROM_ALL) endif(${BUILD_DEPENDENCY_TESTS}) diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt new file mode 100644 index 0000000000..e4624361ed --- /dev/null +++ b/contrib/azure-cmake/CMakeLists.txt @@ -0,0 +1,70 @@ +# lib_azure_sdk +set(AZURE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1") +set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk") + +file(GLOB AZURE_SDK_SRC + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/credentials/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/cryptography/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/io/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/tracing/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.cpp" + "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp" +) + +file(GLOB AZURE_SDK_UNIFIED_SRC + ${AZURE_SDK_SRC} +) + +set(AZURE_SDK_INCLUDES + "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/inc/" + "${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/inc/" + "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/inc/" + "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/" +) + +add_library(_azure_sdk STATIC ${AZURE_SDK_UNIFIED_SRC}) +target_compile_definitions(_azure_sdk PRIVATE BUILD_CURL_HTTP_TRANSPORT_ADAPTER) + + target_include_directories( + _azure_sdk + PUBLIC "$ENV{HOME}/.cos-local.2/include" + ) + +find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +#find_library(CURL_LIBRARY curl) +#find_library(XML2_LIBRARY xml2) +find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +#find_library(CoreFoundation_Library CoreFoundation) +#find_library(SystemConfiguration_Library SystemConfiguration) + +target_link_libraries( + _azure_sdk + PRIVATE ${CURL_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE ${XML2_LIBRARY} + #PRIVATE xml2 + PRIVATE zlib +# PRIVATE ${CoreFoundation_Library} +# PRIVATE ${SystemConfiguration_Library} +) + +# Originally, on Windows azure-core is built with bcrypt and crypt32 by default +if (TARGET OpenSSL::SSL) + target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL) +endif() + +# Originally, on Windows azure-core is built with winhttp by default +if (TARGET td_contrib::curl) + target_link_libraries(_azure_sdk PRIVATE td_contrib::curl) +endif() + +target_include_directories(_azure_sdk SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES}) +add_library(td_contrib::azure_sdk ALIAS _azure_sdk) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 1deff5a67e..f544baafde 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -28,5 +28,6 @@ if(${BUILD_WITH_TRAFT}) # add_subdirectory(traft) endif(${BUILD_WITH_TRAFT}) +add_subdirectory(azure) add_subdirectory(tdev) add_subdirectory(lz4) From c6018cbfaaac825036713ed5c27632848abdb7c5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 08:58:02 +0800 Subject: [PATCH 02/41] lib_azure: test case --- contrib/test/azure/CMakeLists.txt | 29 +++++++++++++++++ contrib/test/azure/main.cpp | 54 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 contrib/test/azure/CMakeLists.txt create mode 100644 contrib/test/azure/main.cpp diff --git a/contrib/test/azure/CMakeLists.txt b/contrib/test/azure/CMakeLists.txt new file mode 100644 index 0000000000..b3db1dffce --- /dev/null +++ b/contrib/test/azure/CMakeLists.txt @@ -0,0 +1,29 @@ +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +add_executable ( + azure-test + main.cpp +) + +# Link to Azure SDK +#target_link_libraries(application _azure_sdk) + +find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +#find_library(XML2_LIBRARY xml2) +find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) +#find_library(CoreFoundation_Library CoreFoundation) +#find_library(SystemConfiguration_Library SystemConfiguration) + + target_link_libraries( + azure-test + PRIVATE _azure_sdk + PRIVATE ${CURL_LIBRARY} + PRIVATE ${XML2_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE dl + PRIVATE pthread +) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp new file mode 100644 index 0000000000..5d52801329 --- /dev/null +++ b/contrib/test/azure/main.cpp @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @file + * @brief Application that consumes the Azure SDK for C++. + * + * @remark Set environment variable `STORAGE_CONNECTION_STRING` before running the application. + * + */ + +#include + +#include +#include + +using namespace Azure::Storage::Blobs; + +int main(int argc, char* argv[]) +{ + (void)argc; + (void)argv; + + /**************** Container SDK client ************************/ + /**************** Create container ************************/ + try + { + auto containerClient = BlobContainerClient::CreateFromConnectionString( + std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); + //containerClient.CreateIfNotExists(); + + /**************** Container SDK client ************************/ + /**************** list blobs (one page) ******************/ + //auto response = containerClient.ListBlobsSinglePage(); + //auto response = containerClient.ListBlobs(); + //auto blobListPage = response.Value; + //auto blobListPage = response.Blobs; + for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) + { + for (auto& blob : page.Blobs) + { + std::cout << blob.Name << std::endl; + } + } + + } + catch (const std::exception& ex) + { + std::cout << ex.what(); + return 1; + } + + return 0; +} From fdc5d6c62574edfb3b7e53207e64b7d335c35ec5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 09:56:43 +0800 Subject: [PATCH 03/41] cos: prep cos from tcs integration --- include/common/cos.h | 2 ++ source/common/src/cos.c | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/include/common/cos.h b/include/common/cos.h index b336a1e5ee..53dc161ee5 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -32,6 +32,8 @@ extern int32_t tsS3PageCacheSize; extern int32_t tsS3UploadDelaySec; int32_t s3Init(); +int32_t s3Begin(); +void s3End(); int32_t s3CheckCfg(); int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index c2b9fe34e1..47dc629c73 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -89,20 +89,8 @@ static void s3DumpCfgByEp(int8_t epIndex) { int32_t s3CheckCfg() { int32_t code = 0, lino = 0; - int8_t i = 0; - if (!tsS3Enabled) { - (void)fprintf(stderr, "s3 not configured.\n"); - TAOS_RETURN(code); - } - - code = s3Begin(); - if (code != 0) { - (void)fprintf(stderr, "failed to initialize s3.\n"); - TAOS_RETURN(code); - } - - for (; i < tsS3EpNum; i++) { + for (int8_t i = 0; i < tsS3EpNum; i++) { (void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum); s3DumpCfgByEp(i); @@ -192,7 +180,7 @@ int32_t s3CheckCfg() { (void)fprintf(stdout, "=================================================================\n"); } - s3End(); + // s3End(); TAOS_RETURN(code); } @@ -1529,6 +1517,8 @@ void s3EvictCache(const char *path, long object_size) {} #include "cos_http_io.h" #include "cos_log.h" +int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); } + int32_t s3Init() { if (cos_http_io_initialize(NULL, 0) != COSE_OK) { return -1; @@ -1967,6 +1957,10 @@ long s3Size(const char *object_name) { #else int32_t s3Init() { return 0; } +int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); } + +void s3End() {} +int32_t s3CheckCfg() { return 0; } int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; } From cd9eec15fb40ccd68116b151d24739d5d8043234 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 10:13:55 +0800 Subject: [PATCH 04/41] tcs: unified interface of cs --- include/libs/tcs/tcs.h | 58 ++++ source/libs/CMakeLists.txt | 4 +- source/libs/tcs/CMakeLists.txt | 22 ++ source/libs/tcs/src/tcs.c | 149 +++++++++++ source/libs/tcs/test/CMakeLists.txt | 18 ++ source/libs/tcs/test/tcsTest.c | 395 ++++++++++++++++++++++++++++ 6 files changed, 645 insertions(+), 1 deletion(-) create mode 100644 include/libs/tcs/tcs.h create mode 100644 source/libs/tcs/CMakeLists.txt create mode 100644 source/libs/tcs/src/tcs.c create mode 100644 source/libs/tcs/test/CMakeLists.txt create mode 100644 source/libs/tcs/test/tcsTest.c diff --git a/include/libs/tcs/tcs.h b/include/libs/tcs/tcs.h new file mode 100644 index 0000000000..530a23d9e9 --- /dev/null +++ b/include/libs/tcs/tcs.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TCS_H_ +#define _TD_TCS_H_ + +#include "os.h" +#include "tarray.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int8_t tsS3Enabled; +extern int8_t tsS3EnabledCfg; + +extern int32_t tsS3UploadDelaySec; +extern int32_t tsS3BlockSize; +extern int32_t tsS3BlockCacheSize; +extern int32_t tsS3PageCacheSize; + +extern int8_t tsS3StreamEnabled; + +int32_t tcsInit(); +void tcsUninit(); + +int32_t tcsCheckCfg(); + +int32_t tcsPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size); +int32_t tcsGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); + +void tcsDeleteObjectsByPrefix(const char *prefix); + +int32_t tcsPutObjectFromFile2(const char *file, const char *object, int8_t withcp); +int32_t tcsGetObjectsByPrefix(const char *prefix, const char *path); +int32_t tcsDeleteObjects(const char *object_name[], int nobject); +int32_t tcsGetObjectToFile(const char *object_name, const char *fileName); + +#ifdef __cplusplus +} +#endif + +#endif // _TD_TCS_H_ diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 64209572f4..41a1e99521 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -22,4 +22,6 @@ add_subdirectory(stream) add_subdirectory(planner) add_subdirectory(qworker) add_subdirectory(geometry) -add_subdirectory(command) \ No newline at end of file +add_subdirectory(command) +#add_subdirectory(azure) +add_subdirectory(tcs) diff --git a/source/libs/tcs/CMakeLists.txt b/source/libs/tcs/CMakeLists.txt new file mode 100644 index 0000000000..1c914a18b9 --- /dev/null +++ b/source/libs/tcs/CMakeLists.txt @@ -0,0 +1,22 @@ +aux_source_directory(src TOS_SRC) + +add_library(tcs STATIC ${TOS_SRC}) +target_include_directories( + tcs + PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + tcs + PUBLIC az + PUBLIC common + # PUBLIC cjson + # PUBLIC os + # PUBLIC util + # PUBLIC crypt +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c new file mode 100644 index 0000000000..c5c68c4933 --- /dev/null +++ b/source/libs/tcs/src/tcs.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tcs.h" + +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" + +//#include "az.h" +#include "cos.h" + +extern int8_t tsS3Ablob; + +typedef enum { + TOS_PROTO_NIL, + TOS_PROTO_S3, + TOS_PROTO_ABLOB, +} STosProto; + +typedef struct { + int32_t (*Begin)(); + void (*End)(); + int32_t (*CheckCfg)(); + + int32_t (*PutObjectFromFileOffset)(const char* file, const char* object_name, int64_t offset, int64_t size); + int32_t (*GetObjectBlock)(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock); + + void (*DeleteObjectsByPrefix)(const char* prefix); + + int32_t (*PutObjectFromFile2)(const char* file, const char* object, int8_t withcp); + int32_t (*GetObjectsByPrefix)(const char* prefix, const char* path); + int32_t (*DeleteObjects)(const char* object_name[], int nobject); + int32_t (*GetObjectToFile)(const char* object_name, const char* fileName); +} STcs; + +static STcs tcs; + +int32_t tcsInit() { + int32_t code = 0; + + STosProto proto = tsS3Ablob ? TOS_PROTO_ABLOB : TOS_PROTO_S3; + + if (TOS_PROTO_S3 == proto) { + tcs.Begin = s3Begin; + tcs.End = s3End; + tcs.CheckCfg = s3CheckCfg; + + tcs.PutObjectFromFileOffset = s3PutObjectFromFileOffset; + tcs.GetObjectBlock = s3GetObjectBlock; + + tcs.DeleteObjectsByPrefix = s3DeleteObjectsByPrefix; + + tcs.PutObjectFromFile2 = s3PutObjectFromFile2; + tcs.GetObjectsByPrefix = s3GetObjectsByPrefix; + tcs.DeleteObjects = s3DeleteObjects; + tcs.GetObjectToFile = s3GetObjectToFile; + } else if (TOS_PROTO_ABLOB == proto) { + /* + tcs.Begin = azBegin; + tcs.End = azEnd; + tcs.CheckCfg = azCheckCfg; + + tcs.PutObjectFromFileOffset = azPutObjectFromFileOffset; + tcs.GetObjectBlock = azGetObjectBlock; + + tcs.DeleteObjectsByPrefix = azDeleteObjectsByPrefix; + + tcs.PutObjectFromFile2 = azPutObjectFromFile2; + tcs.GetObjectsByPrefix = azGetObjectsByPrefix; + tcs.DeleteObjects = azDeleteObjects; + tcs.GetObjectToFile = azGetObjectToFile; + */ + } else { + code = TSDB_CODE_INVALID_PARA; + return code; + } + + code = tcs.Begin(); + + return code; +} + +void tcsUninit() { tcs.End(); } + +int32_t tcsCheckCfg() { + int32_t code = 0; + + if (!tsS3Enabled) { + (void)fprintf(stderr, "s3 not configured.\n"); + TAOS_RETURN(code); + } + + code = tcsInit(); + if (code != 0) { + (void)fprintf(stderr, "failed to initialize s3.\n"); + TAOS_RETURN(code); + } + + code = s3Begin(); + if (code != 0) { + (void)fprintf(stderr, "failed to begin s3.\n"); + TAOS_RETURN(code); + } + + code = tcs.CheckCfg(); + if (code != 0) { + (void)fprintf(stderr, "failed to check s3.\n"); + TAOS_RETURN(code); + } + + tcsUninit(); + + return code; +} + +int32_t tcsPutObjectFromFileOffset(const char* file, const char* object_name, int64_t offset, int64_t size) { + return tcs.PutObjectFromFileOffset(file, object_name, offset, size); +} + +int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock) { + return tcs.GetObjectBlock(object_name, offset, size, check, ppBlock); +} + +void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); } + +int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { + return tcs.PutObjectFromFile2(file, object, withcp); +} + +int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } + +int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } + +int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { + return tcs.GetObjectToFile(object_name, fileName); +} diff --git a/source/libs/tcs/test/CMakeLists.txt b/source/libs/tcs/test/CMakeLists.txt new file mode 100644 index 0000000000..656c659476 --- /dev/null +++ b/source/libs/tcs/test/CMakeLists.txt @@ -0,0 +1,18 @@ +aux_source_directory(. TOS_TEST_SRC) + +add_executable(tosTest ${TOS_TEST_SRC}) +target_include_directories(tosTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/tosure" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(tosTest + tcs + gtest_main +) +enable_testing() +add_test( + NAME tos_test + COMMAND tosTest +) diff --git a/source/libs/tcs/test/tcsTest.c b/source/libs/tcs/test/tcsTest.c new file mode 100644 index 0000000000..68b39bd710 --- /dev/null +++ b/source/libs/tcs/test/tcsTest.c @@ -0,0 +1,395 @@ +#include +#include +#include +#include +/* +#include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); +SWalSyncInfo syncMeta = {0}; +class WalCleanEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalKeepEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalRetentionEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 0; + cfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +TEST_F(WalCleanEnv, createNew) { + walRollFileInfo(pWal); + ASSERT(pWal->fileInfoSet != NULL); + ASSERT_EQ(pWal->fileInfoSet->size, 1); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + ASSERT_EQ(pInfo->firstVer, 0); + ASSERT_EQ(pInfo->lastVer, -1); + ASSERT_EQ(pInfo->closeTs, -1); + ASSERT_EQ(pInfo->fileSize, 0); +} +TEST_F(WalCleanEnv, serialize) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + char* ss = NULL; + code = walMetaSerialize(pWal, &ss); + ASSERT(code == 0); + printf("%s\n", ss); + taosMemoryFree(ss); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} +TEST_F(WalCleanEnv, removeOldMeta) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walSaveMeta(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} +TEST_F(WalKeepEnv, readOldMeta) { + walResetEnv(); + int code; + syncMeta.isWeek = -1; + syncMeta.seqNum = UINT64_MAX; + syncMeta.term = UINT64_MAX; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + char* oldss = NULL; + code = walMetaSerialize(pWal, &oldss); + ASSERT(code == 0); + TearDown(); + SetUp(); + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + char* newss = NULL; + code = walMetaSerialize(pWal, &newss); + ASSERT(code == 0); + int len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for (int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } + taosMemoryFree(oldss); + taosMemoryFree(newss); +} +TEST_F(WalCleanEnv, write) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanEnv, rollback) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 2); + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanEnv, rollbackMultiFile) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + if (i == 5) { + walBeginSnapshot(pWal, i, 0); + walEndSnapshot(pWal); + } + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 6); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); + code = walRollback(pWal, 5); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); + code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 6); + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanDeleteEnv, roll) { + int code; + int i; + for (i = 0; i < 100; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); + walEndSnapshot(pWal); + ASSERT_EQ(pWal->vers.snapshotVer, i - 1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); + code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_NE(code, 0); + for (; i < 200; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + code = walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(code, 0); + code = walEndSnapshot(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} +TEST_F(WalRetentionEnv, repairMeta1) { + walResetEnv(); + int code; + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + TearDown(); + // getchar(); + char buf[100]; + sprintf(buf, "%s/meta-ver%d", pathName, 0); + taosRemoveFile(buf); + sprintf(buf, "%s/meta-ver%d", pathName, 1); + taosRemoveFile(buf); + SetUp(); + // getchar(); + ASSERT_EQ(pWal->vers.lastVer, 99); + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + for (i = 100; i < 200; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 200; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} +*/ From 96b121e5d19c365c8c9bb75215b2fc4a6ebebd81 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 12:50:07 +0800 Subject: [PATCH 05/41] tcs/test: use cpp to compile --- source/libs/tcs/test/tcsTest.c | 395 --------------------------------- 1 file changed, 395 deletions(-) delete mode 100644 source/libs/tcs/test/tcsTest.c diff --git a/source/libs/tcs/test/tcsTest.c b/source/libs/tcs/test/tcsTest.c deleted file mode 100644 index 68b39bd710..0000000000 --- a/source/libs/tcs/test/tcsTest.c +++ /dev/null @@ -1,395 +0,0 @@ -#include -#include -#include -#include -/* -#include "walInt.h" -const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); -SWalSyncInfo syncMeta = {0}; -class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalRetentionEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - void SetUp() override { - SWalCfg cfg; - cfg.rollPeriod = -1; - cfg.segSize = -1; - cfg.retentionPeriod = -1; - cfg.retentionSize = 0; - cfg.rollPeriod = 0; - cfg.vgId = 0; - cfg.level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, &cfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -TEST_F(WalCleanEnv, createNew) { - walRollFileInfo(pWal); - ASSERT(pWal->fileInfoSet != NULL); - ASSERT_EQ(pWal->fileInfoSet->size, 1); - SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); - ASSERT_EQ(pInfo->firstVer, 0); - ASSERT_EQ(pInfo->lastVer, -1); - ASSERT_EQ(pInfo->closeTs, -1); - ASSERT_EQ(pInfo->fileSize, 0); -} -TEST_F(WalCleanEnv, serialize) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - char* ss = NULL; - code = walMetaSerialize(pWal, &ss); - ASSERT(code == 0); - printf("%s\n", ss); - taosMemoryFree(ss); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} -TEST_F(WalCleanEnv, removeOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walSaveMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} -TEST_F(WalKeepEnv, readOldMeta) { - walResetEnv(); - int code; - syncMeta.isWeek = -1; - syncMeta.seqNum = UINT64_MAX; - syncMeta.term = UINT64_MAX; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - char* oldss = NULL; - code = walMetaSerialize(pWal, &oldss); - ASSERT(code == 0); - TearDown(); - SetUp(); - ASSERT_EQ(pWal->vers.firstVer, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - char* newss = NULL; - code = walMetaSerialize(pWal, &newss); - ASSERT(code == 0); - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for (int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } - taosMemoryFree(oldss); - taosMemoryFree(newss); -} -TEST_F(WalCleanEnv, write) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanEnv, rollback) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 2); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanEnv, rollbackMultiFile) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - if (i == 5) { - walBeginSnapshot(pWal, i, 0); - walEndSnapshot(pWal); - } - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 6); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walRollback(pWal, 5); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 6); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanDeleteEnv, roll) { - int code; - int i; - for (i = 0; i < 100; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); - walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i - 1); - ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_NE(code, 0); - for (; i < 200; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - code = walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(code, 0); - code = walEndSnapshot(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalKeepEnv, readHandleRead) { - walResetEnv(); - int code; - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -TEST_F(WalRetentionEnv, repairMeta1) { - walResetEnv(); - int code; - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - TearDown(); - // getchar(); - char buf[100]; - sprintf(buf, "%s/meta-ver%d", pathName, 0); - taosRemoveFile(buf); - sprintf(buf, "%s/meta-ver%d", pathName, 1); - taosRemoveFile(buf); - SetUp(); - // getchar(); - ASSERT_EQ(pWal->vers.lastVer, 99); - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - for (i = 100; i < 200; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 200; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -*/ From c5b4cc760b569e2e84ffcd5eff6d4f6246306ae6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 13:15:51 +0800 Subject: [PATCH 06/41] tcs: link az later --- source/libs/tcs/CMakeLists.txt | 2 +- source/libs/tcs/test/CMakeLists.txt | 14 +- source/libs/tcs/test/tcsTest.cpp | 395 ++++++++++++++++++++++++++++ 3 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 source/libs/tcs/test/tcsTest.cpp diff --git a/source/libs/tcs/CMakeLists.txt b/source/libs/tcs/CMakeLists.txt index 1c914a18b9..4d74dedcd0 100644 --- a/source/libs/tcs/CMakeLists.txt +++ b/source/libs/tcs/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( tcs - PUBLIC az + # PUBLIC az PUBLIC common # PUBLIC cjson # PUBLIC os diff --git a/source/libs/tcs/test/CMakeLists.txt b/source/libs/tcs/test/CMakeLists.txt index 656c659476..33fe75c589 100644 --- a/source/libs/tcs/test/CMakeLists.txt +++ b/source/libs/tcs/test/CMakeLists.txt @@ -1,18 +1,18 @@ -aux_source_directory(. TOS_TEST_SRC) +aux_source_directory(. TCS_TEST_SRC) -add_executable(tosTest ${TOS_TEST_SRC}) -target_include_directories(tosTest +add_executable(tcsTest ${TCS_TEST_SRC}) +target_include_directories(tcsTest PUBLIC - "${TD_SOURCE_DIR}/include/libs/tosure" + "${TD_SOURCE_DIR}/include/libs/tcs" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_link_libraries(tosTest +target_link_libraries(tcsTest tcs gtest_main ) enable_testing() add_test( - NAME tos_test - COMMAND tosTest + NAME tcs_test + COMMAND tcsTest ) diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp new file mode 100644 index 0000000000..68b39bd710 --- /dev/null +++ b/source/libs/tcs/test/tcsTest.cpp @@ -0,0 +1,395 @@ +#include +#include +#include +#include +/* +#include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); +SWalSyncInfo syncMeta = {0}; +class WalCleanEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalKeepEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +class WalRetentionEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + static void TearDownTestCase() { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 0; + cfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; +TEST_F(WalCleanEnv, createNew) { + walRollFileInfo(pWal); + ASSERT(pWal->fileInfoSet != NULL); + ASSERT_EQ(pWal->fileInfoSet->size, 1); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + ASSERT_EQ(pInfo->firstVer, 0); + ASSERT_EQ(pInfo->lastVer, -1); + ASSERT_EQ(pInfo->closeTs, -1); + ASSERT_EQ(pInfo->fileSize, 0); +} +TEST_F(WalCleanEnv, serialize) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + char* ss = NULL; + code = walMetaSerialize(pWal, &ss); + ASSERT(code == 0); + printf("%s\n", ss); + taosMemoryFree(ss); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} +TEST_F(WalCleanEnv, removeOldMeta) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walSaveMeta(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} +TEST_F(WalKeepEnv, readOldMeta) { + walResetEnv(); + int code; + syncMeta.isWeek = -1; + syncMeta.seqNum = UINT64_MAX; + syncMeta.term = UINT64_MAX; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + char* oldss = NULL; + code = walMetaSerialize(pWal, &oldss); + ASSERT(code == 0); + TearDown(); + SetUp(); + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + char* newss = NULL; + code = walMetaSerialize(pWal, &newss); + ASSERT(code == 0); + int len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for (int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } + taosMemoryFree(oldss); + taosMemoryFree(newss); +} +TEST_F(WalCleanEnv, write) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanEnv, rollback) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 2); + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanEnv, rollbackMultiFile) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + if (i == 5) { + walBeginSnapshot(pWal, i, 0); + walEndSnapshot(pWal); + } + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 6); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); + code = walRollback(pWal, 5); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); + code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 6); + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalCleanDeleteEnv, roll) { + int code; + int i; + for (i = 0; i < 100; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); + walEndSnapshot(pWal); + ASSERT_EQ(pWal->vers.snapshotVer, i - 1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); + code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_NE(code, 0); + for (; i < 200; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + code = walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(code, 0); + code = walEndSnapshot(pWal); + ASSERT_EQ(code, 0); +} +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} +TEST_F(WalRetentionEnv, repairMeta1) { + walResetEnv(); + int code; + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + TearDown(); + // getchar(); + char buf[100]; + sprintf(buf, "%s/meta-ver%d", pathName, 0); + taosRemoveFile(buf); + sprintf(buf, "%s/meta-ver%d", pathName, 1); + taosRemoveFile(buf); + SetUp(); + // getchar(); + ASSERT_EQ(pWal->vers.lastVer, 99); + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + for (i = 100; i < 200; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 200; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} +*/ From 09cf91f256a9c64f87ac928a093ab83b7ac927fb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 14:52:42 +0800 Subject: [PATCH 07/41] tcs/dnode: use tcs interface for dnode --- source/common/src/tglobal.c | 59 ++++++++++++---------- source/dnode/mgmt/exe/dmMain.c | 6 +-- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 18 +++---- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index af1a8ccfbe..45546f9a59 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -179,12 +179,12 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsMetaCacheMaxSize = -1; // MB -int32_t tsSlowLogThreshold = 10; // seconds -int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds -char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 10; // seconds +int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds +char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY; -char* tsSlowLogScopeString = "query"; +char *tsSlowLogScopeString = "query"; int32_t tsSlowLogMaxLen = 4096; int32_t tsTimeSeriesThreshold = 50; bool tsMultiResultFunctionStarReturnTags = false; @@ -306,6 +306,7 @@ char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; int8_t tsS3Enabled = false; int8_t tsS3EnabledCfg = false; int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false}; +int8_t tsS3Ablob = false; int8_t tsS3StreamEnabled = false; int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true}; @@ -322,7 +323,6 @@ int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d - #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \ @@ -361,7 +361,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, name); char *strDup = NULL; - if ((strDup = taosStrdup(pItem->str))== NULL){ + if ((strDup = taosStrdup(pItem->str)) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -435,6 +435,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL); tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL); + tsS3Ablob = (strstr(tsS3Endpoint[i], ".blob.core.windows.net") != NULL); } if (tsS3BucketName[0] != '<') { @@ -450,7 +451,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { return tsCfg; } +struct SConfig *taosGetCfg() { + return tsCfg; +} static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -572,7 +575,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); @@ -600,7 +604,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); tsNumOfRpcThreads = tsNumOfCores / 2; @@ -1088,9 +1093,9 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope) { int32_t slowScope = 0; - char* scope = NULL; - char *tmp = NULL; - while((scope = strsep(&pScopeStr, "|")) != NULL){ + char *scope = NULL; + char *tmp = NULL; + while ((scope = strsep(&pScopeStr, "|")) != NULL) { taosMemoryFreeClear(tmp); tmp = taosStrdup(scope); if (tmp == NULL) { @@ -1147,13 +1152,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "firstEp"); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &firstEp)); (void)snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "firstEp", tsFirst, pItem->stype, true)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "secondEp"); - SEp secondEp = {0}; + SEp secondEp = {0}; TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &secondEp)); (void)snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); TAOS_CHECK_RETURN(cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, true)); @@ -1653,8 +1658,8 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SConfig *pCfg = NULL; if (tsCfg == NULL) { @@ -1725,7 +1730,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_RETURN(cfgInit(&pCfg)); TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); + TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); @@ -1754,7 +1759,7 @@ _exit: static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); + int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); if (code) { uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code)); TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR); @@ -1859,7 +1864,7 @@ typedef struct { static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { int32_t code = TSDB_CODE_CFG_NOT_FOUND; - char *name = pItem->name; + char *name = pItem->name; for (int32_t d = 0; d < optionSize; ++d) { const char *optName = pOptions[d].optionName; if (strcasecmp(name, optName) != 0) continue; @@ -2054,8 +2059,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'f': { if (strcasecmp("fqdn", name) == 0) { - SConfigItem* pFqdnItem = cfgGetItem(pCfg, "fqdn"); - SConfigItem* pServerPortItem = cfgGetItem(pCfg, "serverPort"); + SConfigItem *pFqdnItem = cfgGetItem(pCfg, "fqdn"); + SConfigItem *pServerPortItem = cfgGetItem(pCfg, "serverPort"); SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); if (pFqdnItem == NULL || pServerPortItem == NULL || pFirstEpItem == NULL) { uError("failed to get fqdn or serverPort or firstEp from cfg"); @@ -2070,7 +2075,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2110,8 +2115,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } case 'l': { if (strcasecmp("locale", name) == 0) { - SConfigItem* pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem* pCharsetItem = cfgGetItem(pCfg, "charset"); + SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); + SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); if (pLocaleItem == NULL || pCharsetItem == NULL) { uError("failed to get locale or charset from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; @@ -2184,7 +2189,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { char defaultFirstEp[TSDB_EP_LEN] = {0}; (void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - SEp firstEp = {0}; + SEp firstEp = {0}; TAOS_CHECK_GOTO( taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino, _out); @@ -2315,7 +2320,7 @@ int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg, // NOTE: set all command does not change the tmrDebugFlag static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag < 0) TAOS_RETURN(TSDB_CODE_INVALID_PARA); - if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore + if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore SArray *noNeedToSetVars = NULL; SConfigItem *pItem = NULL; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 89569d69d6..6069dc33f1 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -23,6 +23,7 @@ #include "jemalloc/jemalloc.h" #endif #include "dmUtil.h" +#include "tcs.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -325,10 +326,9 @@ static int32_t dmCheckS3() { int32_t code = 0; SConfig *pCfg = taosGetCfg(); cfgDumpCfgS3(pCfg, 0, true); -#if defined(USE_S3) - extern int32_t s3CheckCfg(); - code = s3CheckCfg(); +#if defined(USE_S3) + code = tcsCheckCfg(); #endif return code; } diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 82b9384d66..98de62eee1 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw tcs ) IF (TD_ENTERPRISE) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 2d0ad70adf..d72bc79034 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -20,6 +20,7 @@ #include "libs/function/tudf.h" #include "tgrant.h" #include "tcompare.h" +#include "tcs.h" // clang-format on #define DM_INIT_AUDIT() \ @@ -97,9 +98,9 @@ static bool dmDataSpaceAvailable() { static int32_t dmCheckDiskSpace() { // availability int32_t code = 0; - code = osUpdate(); - if(code != 0) { - code = 0; // ignore the error, just log it + code = osUpdate(); + if (code != 0) { + code = 0; // ignore the error, just log it dError("failed to update os info since %s", tstrerror(code)); } if (!dmDataSpaceAvailable()) { @@ -162,13 +163,6 @@ static int32_t dmCheckDataDirVersionWrapper() { } return 0; } -#if defined(USE_S3) - -extern int32_t s3Begin(); -extern void s3End(); -extern int8_t tsS3Enabled; - -#endif int32_t dmInit() { dInfo("start to init dnode env"); @@ -186,7 +180,7 @@ int32_t dmInit() { if ((code = dmInitDnode(dmInstance())) != 0) return code; if ((code = InitRegexCache() != 0)) return code; #if defined(USE_S3) - if ((code = s3Begin()) != 0) return code; + if ((code = tcsInit()) != 0) return code; #endif dInfo("dnode env is initialized"); @@ -219,7 +213,7 @@ void dmCleanup() { DestroyRegexCache(); #if defined(USE_S3) - s3End(); + tcsUninit(); #endif dInfo("dnode env is cleaned up"); From 3905c94f035db22d95d7c2eca7644dfbb004665a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 16:34:37 +0800 Subject: [PATCH 08/41] vnode/tcs: use tcs instead of s3 interface --- source/dnode/vnode/CMakeLists.txt | 3 ++ source/dnode/vnode/src/tsdb/tsdbCache.c | 12 +++--- source/dnode/vnode/src/tsdb/tsdbFile2.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 37 ++----------------- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +- 6 files changed, 18 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index f70a8844ba..83ed98d7b7 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -119,6 +119,7 @@ if (${BUILD_CONTRIB}) vnode PUBLIC "inc" PUBLIC "src/inc" + PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" @@ -129,6 +130,7 @@ else() vnode PUBLIC "inc" PUBLIC "src/inc" + PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" @@ -164,6 +166,7 @@ target_link_libraries( PUBLIC tdb PUBLIC audit PUBLIC crypt + PUBLIC tcs # PUBLIC bdb # PUBLIC scalar diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 85f74b1672..ac5689724f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "cos.h" #include "functionMgt.h" +#include "tcs.h" #include "tsdb.h" #include "tsdbDataFileRW.h" #include "tsdbIter.h" @@ -1251,7 +1251,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) { - SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID}; + SLastCol lastColTmp = { + .rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID}; if ((code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, &lastColTmp)) != TSDB_CODE_SUCCESS) { tsdbError("tsdb/cache: vgId:%d, put rocks failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); @@ -1698,8 +1699,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(code); } @@ -3503,7 +3503,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) { int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; - TAOS_CHECK_RETURN(s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock)); + TAOS_CHECK_RETURN(tcsGetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock)); tsdbTrace("block:%p load from s3", *ppBlock); @@ -3600,4 +3600,4 @@ void tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t * (void)taosThreadMutexUnlock(&pFD->pTsdb->pgMutex); tsdbCacheRelease(pFD->pTsdb->pgCache, handle); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index da78d67db3..ad5f02d601 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -14,7 +14,7 @@ */ #include "tsdbFile2.h" -#include "cos.h" +#include "tcs.h" #include "vnd.h" // to_json @@ -318,7 +318,7 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { } *(dot + 1) = 0; - s3DeleteObjectsByPrefix(object_name_prefix); + tcsDeleteObjectsByPrefix(object_name_prefix); // remove local last chunk file dot = strrchr(lc_path, '.'); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d867318e1c..53e1c57f14 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "cos.h" #include "crypt.h" +#include "tcs.h" #include "tsdb.h" #include "tsdbDef.h" #include "vnd.h" @@ -391,7 +391,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno); - code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); + code = tcsGetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); TSDB_CHECK_CODE(code, lino, _exit); memcpy(buf + n, pBlock, nRead); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index cbe2ab4b8e..0072fd5e7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "cos.h" +#include "tcs.h" #include "tsdb.h" #include "tsdbFS2.h" #include "vnd.h" @@ -426,35 +426,6 @@ static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3Kee } } -static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) { - int32_t code = 0; - int32_t lino = 0; - - char fname[TSDB_FILENAME_LEN]; - TdFilePtr fdFrom = NULL; - // TdFilePtr fdTo = NULL; - - tsdbTFileName(rtner->tsdb, to, fname); - - fdFrom = taosOpenFile(from->fname, TD_FILE_READ); - if (fdFrom == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } - - char *object_name = taosDirEntryBaseName(fname); - TAOS_CHECK_GOTO(s3PutObjectFromFile2(from->fname, object_name, 1), &lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, - tstrerror(code)); - } - if (taosCloseFile(&fdFrom) != 0) { - tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); - } - return code; -} - static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) { int32_t code = 0; int32_t lino = 0; @@ -519,7 +490,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); int64_t c_offset = chunksize * (cn - fobj->f->lcn); - TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit); + TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit); } // copy last chunk @@ -618,7 +589,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); int64_t c_offset = chunksize * (cn - 1); - TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit); + TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit); } // copy last chunk @@ -741,8 +712,6 @@ _exit: int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) { int32_t code = 0; - extern int8_t tsS3EnabledCfg; - int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE); if (expired && tsS3Enabled) { tsdbWarn("s3 grant expired: %d", expired); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 0d04486925..53365303b0 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "cos.h" #include "sync.h" +#include "tcs.h" #include "tsdb.h" #include "vnd.h" @@ -327,7 +327,7 @@ void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) { if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) { char vnode_prefix[TSDB_FILENAME_LEN]; snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId); - s3DeleteObjectsByPrefix(vnode_prefix); + tcsDeleteObjectsByPrefix(vnode_prefix); } } From c0ef07c050bdea314d23e5a7824721e8d3eec670 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 18:48:34 +0800 Subject: [PATCH 09/41] az: libaz module from ablob --- include/libs/azure/az.h | 45 ++ source/libs/CMakeLists.txt | 2 +- source/libs/azure/CMakeLists.txt | 32 + .../libs/azure/inc/td_block_blob_client.hpp | 260 ++++++++ source/libs/azure/src/avro_parser.cpp | 531 +++++++++++++++ source/libs/azure/src/avro_parser.hpp | 198 ++++++ source/libs/azure/src/az.cpp | 402 +++++++++++ .../libs/azure/src/td_block_blob_client.cpp | 625 ++++++++++++++++++ source/libs/azure/test/CMakeLists.txt | 18 + source/libs/azure/test/azTest.cpp | 457 +++++++++++++ source/libs/tcs/CMakeLists.txt | 2 +- source/libs/tcs/src/tcs.c | 5 +- 12 files changed, 2572 insertions(+), 5 deletions(-) create mode 100644 include/libs/azure/az.h create mode 100644 source/libs/azure/CMakeLists.txt create mode 100644 source/libs/azure/inc/td_block_blob_client.hpp create mode 100644 source/libs/azure/src/avro_parser.cpp create mode 100644 source/libs/azure/src/avro_parser.hpp create mode 100644 source/libs/azure/src/az.cpp create mode 100644 source/libs/azure/src/td_block_blob_client.cpp create mode 100644 source/libs/azure/test/CMakeLists.txt create mode 100644 source/libs/azure/test/azTest.cpp diff --git a/include/libs/azure/az.h b/include/libs/azure/az.h new file mode 100644 index 0000000000..55839b0727 --- /dev/null +++ b/include/libs/azure/az.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AZURE_H_ +#define _TD_AZURE_H_ + +#include "os.h" +#include "tarray.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t azBegin(); +void azEnd(); +int32_t azCheckCfg(); +int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size); +int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); +void azDeleteObjectsByPrefix(const char *prefix); + +int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp); +int32_t azGetObjectsByPrefix(const char *prefix, const char *path); +int32_t azGetObjectToFile(const char *object_name, const char *fileName); +int32_t azDeleteObjects(const char *object_name[], int nobject); + +#ifdef __cplusplus +} +#endif + +#endif // _TD_AZURE_H_ diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 41a1e99521..033582f2c0 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -23,5 +23,5 @@ add_subdirectory(planner) add_subdirectory(qworker) add_subdirectory(geometry) add_subdirectory(command) -#add_subdirectory(azure) +add_subdirectory(azure) add_subdirectory(tcs) diff --git a/source/libs/azure/CMakeLists.txt b/source/libs/azure/CMakeLists.txt new file mode 100644 index 0000000000..1d46a2924b --- /dev/null +++ b/source/libs/azure/CMakeLists.txt @@ -0,0 +1,32 @@ +#if(${TD_LINUX}) +aux_source_directory(src AZ_SRC) + +add_library(az STATIC ${AZ_SRC}) + +if(${BUILD_S3}) + add_definitions(-DUSE_S3) + target_link_libraries( + az + PUBLIC _azure_sdk + PUBLIC crypt + ) +endif() + +target_include_directories( + az + PUBLIC "${TD_SOURCE_DIR}/include/libs/azure" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + az + PUBLIC cjson + PUBLIC os + PUBLIC util + PUBLIC common +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) +#endif(${TD_LINUX}) diff --git a/source/libs/azure/inc/td_block_blob_client.hpp b/source/libs/azure/inc/td_block_blob_client.hpp new file mode 100644 index 0000000000..1b00104821 --- /dev/null +++ b/source/libs/azure/inc/td_block_blob_client.hpp @@ -0,0 +1,260 @@ +#pragma once + +#include "azure/storage/blobs/blob_client.hpp" + +#include +#include +#include + +namespace Azure { +namespace Storage { +namespace Files { +namespace DataLake { +class FileClient; +} +} // namespace Files +} // namespace Storage +} // namespace Azure + +namespace Azure { +namespace Storage { +namespace Blobs { + +/** + * @brief The TDBlockBlobClient allows you to manipulate Azure Storage block blobs. + * + * Block blobs let you upload large blobs efficiently. Block blobs are comprised of blocks, each + * of which is identified by a block ID. You create or modify a block blob by writing a set of + * blocks and committing them by their block IDs. Each block can be a different size. + * + * When you upload a block to a blob in your storage account, it is associated with the specified + * block blob, but it does not become part of the blob until you commit a list of blocks that + * includes the new block's ID. New blocks remain in an uncommitted state until they are + * specifically committed or discarded. Writing a block does not update the last modified time of + * an existing blob. + */ +class TDBlockBlobClient final : public BlobClient { + public: + /** + * @brief Initialize a new instance of TDBlockBlobClient. + * + * @param connectionString A connection string includes the authentication information required + * for your application to access data in an Azure Storage account at runtime. + * @param blobContainerName The name of the container containing this blob. + * @param blobName The name of this blob. + * @param options Optional client options that define the transport pipeline policies for + * authentication, retries, etc., that are applied to every request. + * @return A new TDBlockBlobClient instance. + */ + static TDBlockBlobClient CreateFromConnectionString(const std::string& connectionString, + const std::string& blobContainerName, const std::string& blobName, + const BlobClientOptions& options = BlobClientOptions()); + + /** + * @brief Initialize a new instance of TDBlockBlobClient. + * + * @param blobUrl A URL + * referencing the blob that includes the name of the account, the name of the container, and + * the name of the blob. + * @param credential The shared key credential used to sign + * requests. + * @param options Optional client options that define the transport pipeline + * policies for authentication, retries, etc., that are applied to every request. + */ + explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr credential, + const BlobClientOptions& options = BlobClientOptions()); + + /** + * @brief Initialize a new instance of TDBlockBlobClient. + * + * @param blobUrl A URL + * referencing the blob that includes the name of the account, the name of the container, and + * the name of the blob. + * @param credential The token credential used to sign requests. + * @param options Optional client options that define the transport pipeline policies for + * authentication, retries, etc., that are applied to every request. + */ + explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr credential, + const BlobClientOptions& options = BlobClientOptions()); + + /** + * @brief Initialize a new instance of TDBlockBlobClient. + * + * @param blobUrl A URL + * referencing the blob that includes the name of the account, the name of the container, and + * the name of the blob, and possibly also a SAS token. + * @param options Optional client + * options that define the transport pipeline policies for authentication, retries, etc., that + * are applied to every request. + */ + explicit TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options = BlobClientOptions()); + + /** + * @brief Initializes a new instance of the TDBlockBlobClient class with an identical URL + * source but the specified snapshot timestamp. + * + * @param snapshot The snapshot + * identifier. + * @return A new TDBlockBlobClient instance. + * @remarks Pass empty string to remove the snapshot returning the base blob. + */ + TDBlockBlobClient WithSnapshot(const std::string& snapshot) const; + + /** + * @brief Creates a clone of this instance that references a version ID rather than the base + * blob. + * + * @param versionId The version ID returning a URL to the base blob. + * @return A new TDBlockBlobClient instance. + * @remarks Pass empty string to remove the version ID returning the base blob. + */ + TDBlockBlobClient WithVersionId(const std::string& versionId) const; + + /** + * @brief Creates a new block blob, or updates the content of an existing block blob. Updating + * an existing block blob overwrites any existing metadata on the blob. + * + * @param content A BodyStream containing the content to upload. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A UploadBlockBlobResult describing the state of the updated block blob. + */ + Azure::Response Upload( + Azure::Core::IO::BodyStream& content, const UploadBlockBlobOptions& options = UploadBlockBlobOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Creates a new block blob, or updates the content of an existing block blob. Updating + * an existing block blob overwrites any existing metadata on the blob. + * + * @param buffer A memory buffer containing the content to upload. + * @param bufferSize Size of the memory buffer. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A UploadBlockBlobFromResult describing the state of the updated block blob. + */ + Azure::Response UploadFrom( + const uint8_t* buffer, size_t bufferSize, + const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Creates a new block blob, or updates the content of an existing block blob. Updating + * an existing block blob overwrites any existing metadata on the blob. + * + * @param fileName A file containing the content to upload. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A UploadBlockBlobFromResult describing the state of the updated block blob. + */ + Azure::Response UploadFrom( + const std::string& fileName, const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + Azure::Response UploadFrom( + const std::string& fileName, int64_t offset, int64_t size, + const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Creates a new Block Blob where the contents of the blob are read from a given URL. + * + * @param sourceUri Specifies the URL of the source blob. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A UploadBlockBlobFromUriResult describing the state of the updated block blob. + */ + Azure::Response UploadFromUri( + const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options = UploadBlockBlobFromUriOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Creates a new block as part of a block blob's staging area to be eventually + * committed via the CommitBlockList operation. + * + * @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the + * string must be less than or equal to 64 bytes in size. + * @param content A BodyStream containing the content to upload. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A StageBlockResult describing the state of the updated block. + */ + Azure::Response StageBlock( + const std::string& blockId, Azure::Core::IO::BodyStream& content, + const StageBlockOptions& options = StageBlockOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Creates a new block to be committed as part of a blob where the contents are read from + * the sourceUri. + * + * @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the + * string must be less than or equal to 64 bytes in size. + * @param sourceUri Specifies the uri of the source + * blob. The value may be a uri of up to 2 KB in length that specifies a blob. The source blob + * must either be public or must be authenticated via a shared access signature. If the source + * blob is public, no authentication is required to perform the operation. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A StageBlockFromUriResult describing the state of the updated block blob. + */ + Azure::Response StageBlockFromUri( + const std::string& blockId, const std::string& sourceUri, + const StageBlockFromUriOptions& options = StageBlockFromUriOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Writes a blob by specifying the list of block IDs that make up the blob. In order to + * be written as part of a blob, a block must have been successfully written to the server in a + * prior StageBlock operation. You can call CommitBlockList to update a blob by uploading only + * those blocks that have changed, then committing the new and existing blocks together. You can + * do this by specifying whether to commit a block from the committed block list or from the + * uncommitted block list, or to commit the most recently uploaded version of the block, + * whichever list it may belong to. + * + * @param blockIds Base64 encoded block IDs to indicate that make up the blob. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A CommitBlobBlockListResult describing the state of the updated block blob. + */ + Azure::Response CommitBlockList( + const std::vector& blockIds, const CommitBlockListOptions& options = CommitBlockListOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Retrieves the list of blocks that have been uploaded as part of a block blob. There + * are two block lists maintained for a blob. The Committed Block list has blocks that have been + * successfully committed to a given blob with CommitBlockList. The Uncommitted Block list has + * blocks that have been uploaded for a blob using StageBlock, but that have not yet been + * committed. + * + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A GetBlobBlockListResult describing requested block list. + */ + Azure::Response GetBlockList( + const GetBlockListOptions& options = GetBlockListOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Returns the result of a query against the blob. + * + * @param querySqlExpression The query expression in SQL. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A QueryBlobResult describing the query result. + */ + Azure::Response Query(const std::string& querySqlExpression, + const QueryBlobOptions& options = QueryBlobOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + explicit TDBlockBlobClient(BlobClient blobClient); + + private: + friend class BlobClient; + friend class Files::DataLake::DataLakeFileClient; +}; + +} // namespace Blobs +} // namespace Storage +} // namespace Azure diff --git a/source/libs/azure/src/avro_parser.cpp b/source/libs/azure/src/avro_parser.cpp new file mode 100644 index 0000000000..485980e007 --- /dev/null +++ b/source/libs/azure/src/avro_parser.cpp @@ -0,0 +1,531 @@ +#if defined(USE_S3) +#include "avro_parser.hpp" + +#include +#include + +#include +#include + +namespace Azure { +namespace Storage { +namespace Blobs { +namespace _detail { + +namespace { +int64_t parseInt(AvroStreamReader::ReaderPos& data) { + uint64_t r = 0; + int nb = 0; + while (true) { + uint8_t c = (*data.BufferPtr)[data.Offset++]; + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); +} + +AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) { + const static std::map BuiltinNameSchemaMap = { + {"string", AvroSchema::StringSchema}, {"bytes", AvroSchema::BytesSchema}, {"int", AvroSchema::IntSchema}, + {"long", AvroSchema::LongSchema}, {"float", AvroSchema::FloatSchema}, {"double", AvroSchema::DoubleSchema}, + {"boolean", AvroSchema::BoolSchema}, {"null", AvroSchema::NullSchema}, {"string", AvroSchema::StringSchema}, + }; + std::map nameSchemaMap = BuiltinNameSchemaMap; + + std::function parseSchemaFromJsonObject; + parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema { + if (obj.is_string()) { + auto typeName = obj.get(); + return nameSchemaMap.find(typeName)->second; + } else if (obj.is_array()) { + std::vector unionSchemas; + for (const auto& s : obj) { + unionSchemas.push_back(parseSchemaFromJsonObject(s)); + } + return AvroSchema::UnionSchema(std::move(unionSchemas)); + } else if (obj.is_object()) { + if (obj.count("namespace") != 0) { + throw std::runtime_error("Namespace isn't supported yet in Avro schema."); + } + if (obj.count("aliases") != 0) { + throw std::runtime_error("Alias isn't supported yet in Avro schema."); + } + auto typeName = obj["type"].get(); + auto i = nameSchemaMap.find(typeName); + if (i != nameSchemaMap.end()) { + return i->second; + } + if (typeName == "record") { + std::vector> fieldsSchema; + for (const auto& field : obj["fields"]) { + fieldsSchema.push_back( + std::make_pair(field["name"].get(), parseSchemaFromJsonObject(field["type"]))); + } + + const std::string recordName = obj["name"].get(); + auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema)); + nameSchemaMap.insert(std::make_pair(recordName, recordSchema)); + return recordSchema; + } else if (typeName == "enum") { + throw std::runtime_error("Enum type isn't supported yet in Avro schema."); + } else if (typeName == "array") { + return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"])); + } else if (typeName == "map") { + return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"])); + } else if (typeName == "fixed") { + const std::string fixedName = obj["name"].get(); + auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get()); + nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema)); + return fixedSchema; + } else { + throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema."); + } + } + AZURE_UNREACHABLE_CODE(); + }; + + auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end()); + return parseSchemaFromJsonObject(jsonRoot); +} +} // namespace + +int64_t AvroStreamReader::ParseInt(const Core::Context& context) { + uint64_t r = 0; + int nb = 0; + while (true) { + Preload(1, context); + uint8_t c = m_streambuffer[m_pos.Offset++]; + + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); +} + +void AvroStreamReader::Advance(size_t n, const Core::Context& context) { + Preload(n, context); + m_pos.Offset += n; +} + +size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) { + size_t oldAvailable = AvailableBytes(); + while (true) { + size_t newAvailable = TryPreload(n, context); + if (newAvailable >= n) { + return newAvailable; + } + if (oldAvailable == newAvailable) { + throw std::runtime_error("Unexpected EOF of Avro stream."); + } + oldAvailable = newAvailable; + } + AZURE_UNREACHABLE_CODE(); +} + +size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) { + size_t availableBytes = AvailableBytes(); + if (availableBytes >= n) { + return availableBytes; + } + const size_t MinRead = 4096; + size_t tryReadSize = (std::max)(n, MinRead); + size_t currSize = m_streambuffer.size(); + m_streambuffer.resize(m_streambuffer.size() + tryReadSize); + size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context); + m_streambuffer.resize(currSize + actualReadSize); + return AvailableBytes(); +} + +void AvroStreamReader::Discard() { + constexpr size_t MinimumReleaseMemory = 128 * 1024; + if (m_pos.Offset < MinimumReleaseMemory) { + return; + } + const size_t availableBytes = AvailableBytes(); + std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes); + m_streambuffer.resize(availableBytes); + m_pos.Offset = 0; +} + +const AvroSchema AvroSchema::StringSchema(AvroDatumType::String); +const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes); +const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int); +const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long); +const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float); +const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double); +const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool); +const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null); + +AvroSchema AvroSchema::RecordSchema(std::string name, + const std::vector>& fieldsSchema) { + AvroSchema recordSchema(AvroDatumType::Record); + recordSchema.m_name = std::move(name); + recordSchema.m_status = std::make_shared(); + for (auto& i : fieldsSchema) { + recordSchema.m_status->m_keys.push_back(i.first); + recordSchema.m_status->m_schemas.push_back(i.second); + } + return recordSchema; +} + +AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) { + AvroSchema arraySchema(AvroDatumType::Array); + arraySchema.m_status = std::make_shared(); + arraySchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return arraySchema; +} + +AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) { + AvroSchema mapSchema(AvroDatumType::Map); + mapSchema.m_status = std::make_shared(); + mapSchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return mapSchema; +} + +AvroSchema AvroSchema::UnionSchema(std::vector schemas) { + AvroSchema unionSchema(AvroDatumType::Union); + unionSchema.m_status = std::make_shared(); + unionSchema.m_status->m_schemas = std::move(schemas); + return unionSchema; +} + +AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) { + AvroSchema fixedSchema(AvroDatumType::Fixed); + fixedSchema.m_name = std::move(name); + fixedSchema.m_status = std::make_shared(); + fixedSchema.m_status->m_size = size; + return fixedSchema; +} + +void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) { + m_data = reader.m_pos; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + int64_t stringSize = reader.ParseInt(context); + reader.Advance(static_cast(stringSize), context); + } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || + m_schema.Type() == AvroDatumType::Enum) { + reader.ParseInt(context); + } else if (m_schema.Type() == AvroDatumType::Float) { + reader.Advance(4, context); + } else if (m_schema.Type() == AvroDatumType::Double) { + reader.Advance(8, context); + } else if (m_schema.Type() == AvroDatumType::Bool) { + reader.Advance(1, context); + } else if (m_schema.Type() == AvroDatumType::Null) { + reader.Advance(0, context); + } else if (m_schema.Type() == AvroDatumType::Record) { + for (const auto& s : m_schema.FieldSchemas()) { + AvroDatum(s).Fill(reader, context); + } + } else if (m_schema.Type() == AvroDatumType::Array) { + while (true) { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } else { + for (auto i = 0; i < numElementsInBlock; ++i) { + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Map) { + while (true) { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } else { + for (int64_t i = 0; i < numElementsInBlock; ++i) { + AvroDatum(AvroSchema::StringSchema).Fill(reader, context); + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = reader.ParseInt(context); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(reader, context); + } else if (m_schema.Type() == AvroDatumType::Fixed) { + reader.Advance(m_schema.Size(), context); + } else { + AZURE_UNREACHABLE_CODE(); + } +} + +void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) { + m_data = data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + int64_t stringSize = parseInt(data); + data.Offset += static_cast(stringSize); + } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || + m_schema.Type() == AvroDatumType::Enum) { + parseInt(data); + } else if (m_schema.Type() == AvroDatumType::Float) { + data.Offset += 4; + } else if (m_schema.Type() == AvroDatumType::Double) { + data.Offset += 8; + } else if (m_schema.Type() == AvroDatumType::Bool) { + data.Offset += 1; + } else if (m_schema.Type() == AvroDatumType::Null) { + data.Offset += 0; + } else if (m_schema.Type() == AvroDatumType::Record) { + for (const auto& s : m_schema.FieldSchemas()) { + AvroDatum(s).Fill(data); + } + } else if (m_schema.Type() == AvroDatumType::Array) { + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } else { + for (auto i = 0; i < numElementsInBlock; ++i) { + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Map) { + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } else { + for (int64_t i = 0; i < numElementsInBlock; ++i) { + AvroDatum(AvroSchema::StringSchema).Fill(data); + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = parseInt(data); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(data); + } else if (m_schema.Type() == AvroDatumType::Fixed) { + data.Offset += m_schema.Size(); + } else { + AZURE_UNREACHABLE_CODE(); + } +} + +template <> +AvroDatum::StringView AvroDatum::Value() const { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + const int64_t length = parseInt(data); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, static_cast(length)}; + data.Offset += static_cast(length); + return ret; + } + if (m_schema.Type() == AvroDatumType::Fixed) { + const size_t fixedSize = m_schema.Size(); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, fixedSize}; + data.Offset += fixedSize; + return ret; + } + AZURE_UNREACHABLE_CODE(); +} + +template <> +std::string AvroDatum::Value() const { + auto stringView = Value(); + return std::string(stringView.Data, stringView.Data + stringView.Length); +} + +template <> +std::vector AvroDatum::Value() const { + auto stringView = Value(); + return std::vector(stringView.Data, stringView.Data + stringView.Length); +} + +template <> +int64_t AvroDatum::Value() const { + auto data = m_data; + return parseInt(data); +} + +template <> +int32_t AvroDatum::Value() const { + return static_cast(Value()); +} + +template <> +bool AvroDatum::Value() const { + return Value(); +} + +template <> +std::nullptr_t AvroDatum::Value() const { + return nullptr; +} + +template <> +AvroRecord AvroDatum::Value() const { + auto data = m_data; + + AvroRecord r; + r.m_keys = &m_schema.FieldNames(); + for (const auto& schema : m_schema.FieldSchemas()) { + auto datum = AvroDatum(schema); + datum.Fill(data); + r.m_values.push_back(std::move(datum)); + } + + return r; +} + +template <> +AvroMap AvroDatum::Value() const { + auto data = m_data; + + AvroMap m; + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } + if (numElementsInBlock < 0) { + numElementsInBlock = -numElementsInBlock; + parseInt(data); + } + for (int64_t i = 0; i < numElementsInBlock; ++i) { + auto keyDatum = AvroDatum(AvroSchema::StringSchema); + keyDatum.Fill(data); + auto valueDatum = AvroDatum(m_schema.ItemSchema()); + valueDatum.Fill(data); + m[keyDatum.Value()] = valueDatum; + } + } + return m; +} + +template <> +AvroDatum AvroDatum::Value() const { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = parseInt(data); + auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast(i)]); + datum.Fill(data); + return datum; + } + AZURE_UNREACHABLE_CODE(); +} + +AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream) + : m_reader(std::make_unique(stream)) {} + +AvroDatum AvroObjectContainerReader::NextImpl(const AvroSchema* schema, const Core::Context& context) { + AZURE_ASSERT_FALSE(m_eof); + static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16); + if (!schema) { + static AvroSchema FileHeaderSchema = []() { + std::vector> fieldsSchema; + fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4))); + fieldsSchema.push_back(std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema))); + fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema)); + return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema)); + }(); + auto fileHeaderDatum = AvroDatum(FileHeaderSchema); + fileHeaderDatum.Fill(*m_reader, context); + auto fileHeader = fileHeaderDatum.Value(); + if (fileHeader.Field("magic").Value() != "Obj\01") { + throw std::runtime_error("Invalid Avro object container magic."); + } + AvroMap meta = fileHeader.Field("meta").Value(); + std::string objectSchemaJson = meta["avro.schema"].Value(); + std::string codec = "null"; + if (meta.count("avro.codec") != 0) { + codec = meta["avro.codec"].Value(); + } + if (codec != "null") { + throw std::runtime_error("Unsupported Avro codec: " + codec); + } + m_syncMarker = fileHeader.Field("sync").Value(); + m_objectSchema = std::make_unique(ParseSchemaFromJsonString(objectSchemaJson)); + schema = m_objectSchema.get(); + } + + if (m_remainingObjectInCurrentBlock == 0) { + m_reader->Discard(); + m_remainingObjectInCurrentBlock = m_reader->ParseInt(context); + int64_t ObjectsSize = m_reader->ParseInt(context); + m_reader->Preload(static_cast(ObjectsSize), context); + } + + auto objectDatum = AvroDatum(*m_objectSchema); + objectDatum.Fill(*m_reader, context); + if (--m_remainingObjectInCurrentBlock == 0) { + auto markerDatum = AvroDatum(SyncMarkerSchema); + markerDatum.Fill(*m_reader, context); + auto marker = markerDatum.Value(); + if (marker != m_syncMarker) { + throw std::runtime_error("Sync marker doesn't match."); + } + m_eof = m_reader->TryPreload(1, context) == 0; + } + return objectDatum; +} + +size_t AvroStreamParser::OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) { + if (m_parserBuffer.Length != 0) { + size_t bytesToCopy = (std::min)(m_parserBuffer.Length, count); + std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy); + m_parserBuffer.Data += bytesToCopy; + m_parserBuffer.Length -= bytesToCopy; + return bytesToCopy; + } + while (!m_parser.End()) { + auto datum = m_parser.Next(context); + if (datum.Schema().Type() == AvroDatumType::Union) { + datum = datum.Value(); + } + if (datum.Schema().Type() != AvroDatumType::Record) { + continue; + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") { + auto record = datum.Value(); + auto dataDatum = record.Field("data"); + m_parserBuffer = dataDatum.Value(); + return OnRead(buffer, count, context); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" && m_progressCallback) { + auto record = datum.Value(); + auto bytesScanned = record.Field("bytesScanned").Value(); + auto totalBytes = record.Field("totalBytes").Value(); + m_progressCallback(bytesScanned, totalBytes); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" && m_errorCallback) { + auto record = datum.Value(); + BlobQueryError e; + e.Name = record.Field("name").Value(); + e.Description = record.Field("description").Value(); + e.IsFatal = record.Field("fatal").Value(); + e.Position = record.Field("position").Value(); + m_errorCallback(std::move(e)); + } + } + return 0; +} +} // namespace _detail +} // namespace Blobs +} // namespace Storage +} // namespace Azure + +#endif diff --git a/source/libs/azure/src/avro_parser.hpp b/source/libs/azure/src/avro_parser.hpp new file mode 100644 index 0000000000..275d073c85 --- /dev/null +++ b/source/libs/azure/src/avro_parser.hpp @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/blobs/blob_options.hpp" + +#include + +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + enum class AvroDatumType + { + String, + Bytes, + Int, + Long, + Float, + Double, + Bool, + Null, + Record, + Enum, + Array, + Map, + Union, + Fixed, + }; + + class AvroStreamReader final { + public: + // position of a vector that lives through vector resizing + struct ReaderPos final + { + const std::vector* BufferPtr = nullptr; + size_t Offset = 0; + }; + explicit AvroStreamReader(Core::IO::BodyStream& stream) + : m_stream(&stream), m_pos{&m_streambuffer, 0} + { + } + AvroStreamReader(const AvroStreamReader&) = delete; + AvroStreamReader& operator=(const AvroStreamReader&) = delete; + + int64_t ParseInt(const Core::Context& context); + void Advance(size_t n, const Core::Context& context); + // Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes + // available in m_streambuffer; + size_t Preload(size_t n, const Core::Context& context); + size_t TryPreload(size_t n, const Core::Context& context); + // discards data that's before m_pos + void Discard(); + + private: + size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; } + + private: + Core::IO::BodyStream* m_stream; + std::vector m_streambuffer; + ReaderPos m_pos; + + friend class AvroDatum; + }; + + class AvroSchema final { + public: + static const AvroSchema StringSchema; + static const AvroSchema BytesSchema; + static const AvroSchema IntSchema; + static const AvroSchema LongSchema; + static const AvroSchema FloatSchema; + static const AvroSchema DoubleSchema; + static const AvroSchema BoolSchema; + static const AvroSchema NullSchema; + static AvroSchema RecordSchema( + std::string name, + const std::vector>& fieldsSchema); + static AvroSchema ArraySchema(AvroSchema elementSchema); + static AvroSchema MapSchema(AvroSchema elementSchema); + static AvroSchema UnionSchema(std::vector schemas); + static AvroSchema FixedSchema(std::string name, int64_t size); + + const std::string& Name() const { return m_name; } + AvroDatumType Type() const { return m_type; } + const std::vector& FieldNames() const { return m_status->m_keys; } + AvroSchema ItemSchema() const { return m_status->m_schemas[0]; } + const std::vector& FieldSchemas() const { return m_status->m_schemas; } + size_t Size() const { return static_cast(m_status->m_size); } + + private: + explicit AvroSchema(AvroDatumType type) : m_type(type) {} + + private: + AvroDatumType m_type; + std::string m_name; + + struct SharedStatus + { + std::vector m_keys; + std::vector m_schemas; + int64_t m_size = 0; + }; + std::shared_ptr m_status; + }; + + class AvroDatum final { + public: + AvroDatum() : m_schema(AvroSchema::NullSchema) {} + explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {} + + void Fill(AvroStreamReader& reader, const Core::Context& context); + void Fill(AvroStreamReader::ReaderPos& data); + + const AvroSchema& Schema() const { return m_schema; } + + template T Value() const; + struct StringView + { + const uint8_t* Data = nullptr; + size_t Length = 0; + }; + + private: + AvroSchema m_schema; + AvroStreamReader::ReaderPos m_data; + }; + + using AvroMap = std::map; + + class AvroRecord final { + public: + bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); } + const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); } + AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); } + const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); } + AvroDatum& FieldAt(size_t i) { return m_values.at(i); } + + private: + size_t FindField(const std::string& key) const + { + auto i = find(m_keys->begin(), m_keys->end(), key); + return i - m_keys->begin(); + } + const std::vector* m_keys = nullptr; + std::vector m_values; + + friend class AvroDatum; + }; + + class AvroObjectContainerReader final { + public: + explicit AvroObjectContainerReader(Core::IO::BodyStream& stream); + + bool End() const { return m_eof; } + // Calling Next() will invalidates the previous AvroDatum returned by this function and all + // AvroDatums propagated from there. + AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); } + + private: + AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context); + + private: + std::unique_ptr m_reader; + std::unique_ptr m_objectSchema; + std::string m_syncMarker; + int64_t m_remainingObjectInCurrentBlock = 0; + bool m_eof = false; + }; + + class AvroStreamParser final : public Core::IO::BodyStream { + public: + explicit AvroStreamParser( + std::unique_ptr inner, + std::function progressCallback, + std::function errorCallback) + : m_inner(std::move(inner)), m_parser(*m_inner), + m_progressCallback(std::move(progressCallback)), m_errorCallback(std::move(errorCallback)) + { + } + + int64_t Length() const override { return -1; } + void Rewind() override { this->m_inner->Rewind(); } + + private: + size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override; + + private: + std::unique_ptr m_inner; + AvroObjectContainerReader m_parser; + std::function m_progressCallback; + std::function m_errorCallback; + AvroDatum::StringView m_parserBuffer; + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp new file mode 100644 index 0000000000..83cba1c877 --- /dev/null +++ b/source/libs/azure/src/az.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "az.h" + +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" + +#if defined(USE_S3) + +#include +#include +#include "td_block_blob_client.hpp" + +// Add appropriate using namespace directives +using namespace Azure::Storage; +using namespace Azure::Storage::Blobs; + +extern char tsS3Hostname[][TSDB_FQDN_LEN]; +extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; +extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; +extern char tsS3BucketName[TSDB_FQDN_LEN]; + +extern int8_t tsS3Enabled; +extern int8_t tsS3EpNum; + +int32_t azBegin() { return TSDB_CODE_SUCCESS; } + +void azEnd() {} + +static void azDumpCfgByEp(int8_t epIndex) { + // clang-format off + (void)fprintf(stdout, + "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n" + // "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n", + "hostName", tsS3Hostname[epIndex], + "bucketName", tsS3BucketName, + "protocol", "https only", + //"uristyle", (uriStyleG[epIndex] == S3UriStyleVirtualHost ? "virtualhost" : "path"), + "accessKey", tsS3AccessKeyId[epIndex], + "accessKeySecret", tsS3AccessKeySecret[epIndex]); + // clang-format on +} + +static int32_t azListBucket(char const *bucketname) { + int32_t code = 0; + const std::string delimiter = "/"; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; + std::string accountURL = tsS3Hostname[0]; + accountURL = "https://" + accountURL; + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = bucketname; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + Azure::Storage::Blobs::ListBlobsOptions options; + options.Prefix = "s3"; + + (void)fprintf(stderr, "objects:\n"); + // std::set listBlobs; + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + for (const auto &blob : pageResult.Blobs) { + (void)fprintf(stderr, "%s\n", blob.Name.c_str()); + } + } + } catch (const Azure::Core::RequestFailedException &e) { + uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + // uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); + + code = TAOS_SYSTEM_ERROR(EIO); + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +int32_t azCheckCfg() { + int32_t code = 0, lino = 0; + int8_t i = 0; + + // for (; i < tsS3EpNum; i++) { + (void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum); + // s3DumpCfgByEp(i); + azDumpCfgByEp(0); + + // test put + char testdata[17] = "0123456789abcdef"; + const char *objectname[] = {"s3test.txt"}; + char path[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]); + } + + uint8_t *pBlock = NULL; + int c_offset = 10; + int c_len = 6; + char buf[7] = {0}; + + TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (!fp) { + (void)fprintf(stderr, "failed to open test file: %s.\n", path); + // uError("ERROR: %s Failed to open %s", __func__, path); + TAOS_CHECK_GOTO(terrno, &lino, _next); + } + if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { + (void)fprintf(stderr, "failed to write test file: %s.\n", path); + TAOS_CHECK_GOTO(terrno, &lino, _next); + } + if (taosFsyncFile(fp) < 0) { + (void)fprintf(stderr, "failed to fsync test file: %s.\n", path); + TAOS_CHECK_GOTO(terrno, &lino, _next); + } + (void)taosCloseFile(&fp); + + (void)fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata); + code = azPutObjectFromFileOffset(path, objectname[0], 0, 16); + if (code != 0) { + (void)fprintf(stderr, "put object %s : failed.\n", objectname[0]); + TAOS_CHECK_GOTO(code, &lino, _next); + } + (void)fprintf(stderr, "put object %s: success.\n\n", objectname[0]); + + // list buckets + (void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName); + // code = s3ListBucketByEp(tsS3BucketName, i); + code = azListBucket(tsS3BucketName); + if (code != 0) { + (void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); + TAOS_CHECK_GOTO(code, &lino, _next); + } + (void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); + + // test range get + (void)fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len); + code = azGetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); + if (code != 0) { + (void)fprintf(stderr, "get object %s : failed.\n", objectname[0]); + TAOS_CHECK_GOTO(code, &lino, _next); + } + + (void)memcpy(buf, pBlock, c_len); + taosMemoryFree(pBlock); + (void)fprintf(stderr, "object content: %s\n", buf); + (void)fprintf(stderr, "get object %s: success.\n\n", objectname[0]); + + // delete test object + (void)fprintf(stderr, "start to delete object: %s.\n", objectname[0]); + // code = azDeleteObjectsByPrefix(objectname[0]); + azDeleteObjectsByPrefix(objectname[0]); + /* + if (code != 0) { + (void)fprintf(stderr, "delete object %s : failed.\n", objectname[0]); + TAOS_CHECK_GOTO(code, &lino, _next); + } + */ + (void)fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); + +_next: + if (fp) { + (void)taosCloseFile(&fp); + } + + if (TSDB_CODE_SUCCESS != code) { + (void)fprintf(stderr, "s3 check failed, code: %d, line: %d, index: %d.\n", code, lino, i); + } + + (void)fprintf(stdout, "=================================================================\n"); + //} + + // azEnd(); + + TAOS_RETURN(code); +} + +int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { + int32_t code = 0; + + std::string endpointUrl = tsS3Hostname[0]; // GetEndpointUrl(); + std::string accountName = tsS3AccessKeyId[0]; // GetAccountName(); + std::string accountKey = tsS3AccessKeySecret[0]; // GetAccountKey(); + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + std::string accountURL = tsS3Hostname[0]; + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + accountURL = "https://" + accountURL; + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = tsS3BucketName; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + // Create the container if it does not exist + // std::cout << "Creating container: " << containerName << std::endl; + // containerClient.CreateIfNotExists(); + + std::string blobName = "blob.txt"; + uint8_t blobContent[] = "Hello Azure!"; + // Create the block blob client + // BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); + // TDBlockBlobClient blobClient(containerClient.GetBlobClient(blobName)); + TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); + + // Upload the blob + // std::cout << "Uploading blob: " << blobName << std::endl; + // blobClient.UploadFrom(blobContent, sizeof(blobContent)); + blobClient.UploadFrom(file, offset, size); + //(void)_azUploadFrom(blobClient, file, offset, size); + /* + auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); + + // Create some data to upload into the blob. + std::vector data = {1, 2, 3, 4}; + Azure::Core::IO::MemoryBodyStream stream(data); + + Azure::Response response = blockBlobClient.Upload(stream); + + Models::UploadBlockBlobResult model = response.Value; + std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() + << std::endl; + */ + } catch (const Azure::Core::RequestFailedException &e) { + /* + std::cout << "Status Code: " << static_cast(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase + << std::endl; + std::cout << e.what() << std::endl; + */ + code = TAOS_SYSTEM_ERROR(EIO); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + int32_t code = TSDB_CODE_SUCCESS; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; + std::string accountURL = tsS3Hostname[0]; + accountURL = "https://" + accountURL; + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = tsS3BucketName; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); + + uint8_t *buf = (uint8_t *)taosMemoryCalloc(1, size); + if (!buf) { + return terrno; + } + + Blobs::DownloadBlobToOptions options; + // options.TransferOptions.Concurrency = concurrency; + // if (offset.HasValue() || length.HasValue()) { + options.Range = Azure::Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = size; + //} + /* + if (initialChunkSize.HasValue()) { + options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + } + if (chunkSize.HasValue()) { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } + */ + + auto res = blobClient.DownloadTo(buf, size, options); + if (check && res.Value.ContentRange.Length.Value() != size) { + code = TAOS_SYSTEM_ERROR(EIO); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } + + *ppBlock = buf; + } catch (const Azure::Core::RequestFailedException &e) { + uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + code = TAOS_SYSTEM_ERROR(EIO); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +void azDeleteObjectsByPrefix(const char *prefix) { + const std::string delimiter = "/"; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; + std::string accountURL = tsS3Hostname[0]; + accountURL = "https://" + accountURL; + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = tsS3BucketName; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + Azure::Storage::Blobs::ListBlobsOptions options; + options.Prefix = prefix; + + std::set listBlobs; + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + for (const auto &blob : pageResult.Blobs) { + listBlobs.insert(blob.Name); + } + } + + for (auto blobName : listBlobs) { + auto blobClient = containerClient.GetAppendBlobClient(blobName); + blobClient.Delete(); + } + } catch (const Azure::Core::RequestFailedException &e) { + uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + // uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); + } +} + +int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } + +int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; } + +int32_t azGetObjectToFile(const char *object_name, const char *fileName) { return 0; } + +int32_t azDeleteObjects(const char *object_name[], int nobject) { return 0; } + +#else + +int32_t azBegin() { return TSDB_CODE_SUCCESS; } + +void azEnd() {} + +int32_t azCheckCfg() { return TSDB_CODE_SUCCESS; } + +int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { + return TSDB_CODE_SUCCESS; +} + +int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + return TSDB_CODE_SUCCESS; +} + +void azDeleteObjectsByPrefix(const char *prefix) {} + +int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } + +int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; } + +int32_t azGetObjectToFile(const char *object_name, const char *fileName) { return 0; } + +int32_t azDeleteObjects(const char *object_name[], int nobject) { return 0; } + +#endif diff --git a/source/libs/azure/src/td_block_blob_client.cpp b/source/libs/azure/src/td_block_blob_client.cpp new file mode 100644 index 0000000000..e75c6ae17f --- /dev/null +++ b/source/libs/azure/src/td_block_blob_client.cpp @@ -0,0 +1,625 @@ +#if defined(USE_S3) + +#include "td_block_blob_client.hpp" + +#include + +#if defined(AZ_PLATFORM_WINDOWS) +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif +#if !defined(NOMINMAX) +#define NOMINMAX +#endif +#include +#endif + +#include "./avro_parser.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Azure { +namespace Storage { +namespace Blobs { + +TDBlockBlobClient TDBlockBlobClient::CreateFromConnectionString(const std::string& connectionString, + const std::string& blobContainerName, + const std::string& blobName, + const BlobClientOptions& options) { + TDBlockBlobClient newClient( + BlobClient::CreateFromConnectionString(connectionString, blobContainerName, blobName, options)); + return newClient; +} + +TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr credential, + const BlobClientOptions& options) + : BlobClient(blobUrl, std::move(credential), options) {} + +TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, + std::shared_ptr credential, + const BlobClientOptions& options) + : BlobClient(blobUrl, std::move(credential), options) {} + +TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options) + : BlobClient(blobUrl, options) {} + +TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {} + +TDBlockBlobClient TDBlockBlobClient::WithSnapshot(const std::string& snapshot) const { + TDBlockBlobClient newClient(*this); + if (snapshot.empty()) { + newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQuerySnapshot); + } else { + newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQuerySnapshot, + _internal::UrlEncodeQueryParameter(snapshot)); + } + return newClient; +} + +TDBlockBlobClient TDBlockBlobClient::WithVersionId(const std::string& versionId) const { + TDBlockBlobClient newClient(*this); + if (versionId.empty()) { + newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQueryVersionId); + } else { + newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQueryVersionId, + _internal::UrlEncodeQueryParameter(versionId)); + } + return newClient; +} + +Azure::Response TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content, + const UploadBlockBlobOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions; + if (options.TransactionalContentHash.HasValue()) { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { + protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value; + } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { + protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value; + } + } + protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; + protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; + protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; + protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value; + protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition; + protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl; + protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); + protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags); + protocolLayerOptions.Tier = options.AccessTier; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (options.ImmutabilityPolicy.HasValue()) { + protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn; + protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode; + } + protocolLayerOptions.LegalHold = options.HasLegalHold; + + return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context); +} + +Azure::Response TDBlockBlobClient::UploadFrom( + const uint8_t* buffer, size_t bufferSize, const UploadBlockBlobFromOptions& options, + const Azure::Core::Context& context) const { + constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; + constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; + constexpr int64_t MaxBlockNumber = 50000; + constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; + + if (static_cast(options.TransferOptions.SingleUploadThreshold) > (std::numeric_limits::max)()) { + throw Azure::Core::RequestFailedException("Single upload threshold is too big"); + } + if (bufferSize <= static_cast(options.TransferOptions.SingleUploadThreshold)) { + Azure::Core::IO::MemoryBodyStream contentStream(buffer, bufferSize); + UploadBlockBlobOptions uploadBlockBlobOptions; + uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders; + uploadBlockBlobOptions.Metadata = options.Metadata; + uploadBlockBlobOptions.Tags = options.Tags; + uploadBlockBlobOptions.AccessTier = options.AccessTier; + uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold; + return Upload(contentStream, uploadBlockBlobOptions, context); + } + + int64_t chunkSize; + if (options.TransferOptions.ChunkSize.HasValue()) { + chunkSize = options.TransferOptions.ChunkSize.Value(); + } else { + int64_t minChunkSize = (bufferSize + MaxBlockNumber - 1) / MaxBlockNumber; + minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize; + chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize); + } + if (chunkSize > MaxStageBlockSize) { + throw Azure::Core::RequestFailedException("Block size is too big."); + } + + std::vector blockIds; + auto getBlockId = [](int64_t id) { + constexpr size_t BlockIdLength = 64; + std::string blockId = std::to_string(id); + blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId; + return Azure::Core::Convert::Base64Encode(std::vector(blockId.begin(), blockId.end())); + }; + + auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + Azure::Core::IO::MemoryBodyStream contentStream(buffer + offset, static_cast(length)); + StageBlockOptions chunkOptions; + auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context); + if (chunkId == numChunks - 1) { + blockIds.resize(static_cast(numChunks)); + } + }; + + _internal::ConcurrentTransfer(0, bufferSize, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc); + + for (size_t i = 0; i < blockIds.size(); ++i) { + blockIds[i] = getBlockId(static_cast(i)); + } + CommitBlockListOptions commitBlockListOptions; + commitBlockListOptions.HttpHeaders = options.HttpHeaders; + commitBlockListOptions.Metadata = options.Metadata; + commitBlockListOptions.Tags = options.Tags; + commitBlockListOptions.AccessTier = options.AccessTier; + commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + commitBlockListOptions.HasLegalHold = options.HasLegalHold; + auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context); + + Models::UploadBlockBlobFromResult ret; + ret.ETag = std::move(commitBlockListResponse.Value.ETag); + ret.LastModified = std::move(commitBlockListResponse.Value.LastModified); + ret.VersionId = std::move(commitBlockListResponse.Value.VersionId); + ret.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted; + ret.EncryptionKeySha256 = std::move(commitBlockListResponse.Value.EncryptionKeySha256); + ret.EncryptionScope = std::move(commitBlockListResponse.Value.EncryptionScope); + return Azure::Response(std::move(ret), + std::move(commitBlockListResponse.RawResponse)); +} + +Azure::Response TDBlockBlobClient::UploadFrom( + const std::string& fileName, const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { + constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; + constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; + constexpr int64_t MaxBlockNumber = 50000; + constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; + + { + Azure::Core::IO::FileBodyStream contentStream(fileName); + + if (contentStream.Length() <= options.TransferOptions.SingleUploadThreshold) { + UploadBlockBlobOptions uploadBlockBlobOptions; + uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders; + uploadBlockBlobOptions.Metadata = options.Metadata; + uploadBlockBlobOptions.Tags = options.Tags; + uploadBlockBlobOptions.AccessTier = options.AccessTier; + uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold; + return Upload(contentStream, uploadBlockBlobOptions, context); + } + } + + std::vector blockIds; + auto getBlockId = [](int64_t id) { + constexpr size_t BlockIdLength = 64; + std::string blockId = std::to_string(id); + blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId; + return Azure::Core::Convert::Base64Encode(std::vector(blockId.begin(), blockId.end())); + }; + + _internal::FileReader fileReader(fileName); + + auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length); + StageBlockOptions chunkOptions; + auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context); + if (chunkId == numChunks - 1) { + blockIds.resize(static_cast(numChunks)); + } + }; + + int64_t chunkSize; + if (options.TransferOptions.ChunkSize.HasValue()) { + chunkSize = options.TransferOptions.ChunkSize.Value(); + } else { + int64_t minChunkSize = (fileReader.GetFileSize() + MaxBlockNumber - 1) / MaxBlockNumber; + minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize; + chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize); + } + if (chunkSize > MaxStageBlockSize) { + throw Azure::Core::RequestFailedException("Block size is too big."); + } + + _internal::ConcurrentTransfer(0, fileReader.GetFileSize(), chunkSize, options.TransferOptions.Concurrency, + uploadBlockFunc); + + for (size_t i = 0; i < blockIds.size(); ++i) { + blockIds[i] = getBlockId(static_cast(i)); + } + CommitBlockListOptions commitBlockListOptions; + commitBlockListOptions.HttpHeaders = options.HttpHeaders; + commitBlockListOptions.Metadata = options.Metadata; + commitBlockListOptions.Tags = options.Tags; + commitBlockListOptions.AccessTier = options.AccessTier; + commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + commitBlockListOptions.HasLegalHold = options.HasLegalHold; + auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context); + + Models::UploadBlockBlobFromResult result; + result.ETag = commitBlockListResponse.Value.ETag; + result.LastModified = commitBlockListResponse.Value.LastModified; + result.VersionId = commitBlockListResponse.Value.VersionId; + result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted; + result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256; + result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope; + return Azure::Response(std::move(result), + std::move(commitBlockListResponse.RawResponse)); +} + +Azure::Response TDBlockBlobClient::UploadFrom( + const std::string& fileName, int64_t offset, int64_t size, const UploadBlockBlobFromOptions& options, + const Azure::Core::Context& context) const { + _internal::FileReader fileReader(fileName); + + { + Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, size); + + if (size <= options.TransferOptions.SingleUploadThreshold) { + UploadBlockBlobOptions uploadBlockBlobOptions; + uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders; + uploadBlockBlobOptions.Metadata = options.Metadata; + uploadBlockBlobOptions.Tags = options.Tags; + uploadBlockBlobOptions.AccessTier = options.AccessTier; + uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold; + return Upload(contentStream, uploadBlockBlobOptions, context); + } + } + + std::vector blockIds; + auto getBlockId = [](int64_t id) { + constexpr size_t BlockIdLength = 64; + std::string blockId = std::to_string(id); + blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId; + return Azure::Core::Convert::Base64Encode(std::vector(blockId.begin(), blockId.end())); + }; + + auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { + Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length); + StageBlockOptions chunkOptions; + auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context); + if (chunkId == numChunks - 1) { + blockIds.resize(static_cast(numChunks)); + } + }; + + constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL; + constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL; + constexpr int64_t MaxBlockNumber = 50000; + constexpr int64_t BlockGrainSize = 1 * 1024 * 1024; + + int64_t chunkSize; + if (options.TransferOptions.ChunkSize.HasValue()) { + chunkSize = options.TransferOptions.ChunkSize.Value(); + } else { + int64_t minChunkSize = (size + MaxBlockNumber - 1) / MaxBlockNumber; + minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize; + chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize); + } + if (chunkSize > MaxStageBlockSize) { + throw Azure::Core::RequestFailedException("Block size is too big."); + } + + _internal::ConcurrentTransfer(offset, size, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc); + + for (size_t i = 0; i < blockIds.size(); ++i) { + blockIds[i] = getBlockId(static_cast(i)); + } + CommitBlockListOptions commitBlockListOptions; + commitBlockListOptions.HttpHeaders = options.HttpHeaders; + commitBlockListOptions.Metadata = options.Metadata; + commitBlockListOptions.Tags = options.Tags; + commitBlockListOptions.AccessTier = options.AccessTier; + commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; + commitBlockListOptions.HasLegalHold = options.HasLegalHold; + auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context); + + Models::UploadBlockBlobFromResult result; + result.ETag = commitBlockListResponse.Value.ETag; + result.LastModified = commitBlockListResponse.Value.LastModified; + result.VersionId = commitBlockListResponse.Value.VersionId; + result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted; + result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256; + result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope; + return Azure::Response(std::move(result), + std::move(commitBlockListResponse.RawResponse)); +} + +Azure::Response TDBlockBlobClient::UploadFromUri( + const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::UploadBlockBlobFromUriOptions protocolLayerOptions; + protocolLayerOptions.CopySource = sourceUri; + protocolLayerOptions.CopySourceBlobProperties = options.CopySourceBlobProperties; + protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; + protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; + protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; + protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value; + protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl; + protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition; + protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); + protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags); + protocolLayerOptions.Tier = options.AccessTier; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch; + protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch; + protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince; + protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince; + protocolLayerOptions.SourceIfTags = options.SourceAccessConditions.TagConditions; + if (options.TransactionalContentHash.HasValue()) { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { + protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value; + } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { + protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value; + } + } + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + protocolLayerOptions.CopySourceTags = options.CopySourceTagsMode; + if (!options.SourceAuthorization.empty()) { + protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization; + } + + return _detail::BlockBlobClient::UploadFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context); +} + +Azure::Response TDBlockBlobClient::StageBlock(const std::string& blockId, + Azure::Core::IO::BodyStream& content, + const StageBlockOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::StageBlockBlobBlockOptions protocolLayerOptions; + protocolLayerOptions.BlockId = blockId; + if (options.TransactionalContentHash.HasValue()) { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { + protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value; + } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { + protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value; + } + } + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + return _detail::BlockBlobClient::StageBlock(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context); +} + +Azure::Response TDBlockBlobClient::StageBlockFromUri( + const std::string& blockId, const std::string& sourceUri, const StageBlockFromUriOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::StageBlockBlobBlockFromUriOptions protocolLayerOptions; + protocolLayerOptions.BlockId = blockId; + protocolLayerOptions.SourceUrl = sourceUri; + if (options.SourceRange.HasValue()) { + std::string rangeStr = "bytes=" + std::to_string(options.SourceRange.Value().Offset) + "-"; + if (options.SourceRange.Value().Length.HasValue()) { + rangeStr += std::to_string(options.SourceRange.Value().Offset + options.SourceRange.Value().Length.Value() - 1); + } + protocolLayerOptions.SourceRange = rangeStr; + } + if (options.TransactionalContentHash.HasValue()) { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { + protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value; + } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { + protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value; + } + } + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince; + protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince; + protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch; + protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch; + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (!options.SourceAuthorization.empty()) { + protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization; + } + + return _detail::BlockBlobClient::StageBlockFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context); +} + +Azure::Response TDBlockBlobClient::CommitBlockList( + const std::vector& blockIds, const CommitBlockListOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::CommitBlockBlobBlockListOptions protocolLayerOptions; + protocolLayerOptions.Blocks.Latest = blockIds; + protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; + protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; + protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; + protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value; + protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition; + protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl; + protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); + protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags); + protocolLayerOptions.Tier = options.AccessTier; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (options.ImmutabilityPolicy.HasValue()) { + protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn; + protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode; + } + protocolLayerOptions.LegalHold = options.HasLegalHold; + + return _detail::BlockBlobClient::CommitBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, context); +} + +Azure::Response TDBlockBlobClient::GetBlockList(const GetBlockListOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::GetBlockBlobBlockListOptions protocolLayerOptions; + protocolLayerOptions.ListType = options.ListType; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + return _detail::BlockBlobClient::GetBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, + _internal::WithReplicaStatus(context)); +} +/* +Azure::Response TDBlockBlobClient::Query(const std::string& querySqlExpression, + const QueryBlobOptions& options, + const Azure::Core::Context& context) const { +_detail::BlobClient::QueryBlobOptions protocolLayerOptions; +protocolLayerOptions.QueryRequest.QueryType = Models::_detail::QueryRequestQueryType::SQL; +protocolLayerOptions.QueryRequest.Expression = querySqlExpression; +if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.InputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.InputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.InputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.InputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); +} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); +} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); +} else if (options.InputTextConfiguration.m_format.ToString().empty()) { +} else { + AZURE_UNREACHABLE_CODE(); +} +if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.OutputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.OutputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.OutputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.OutputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); +} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); +} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); +} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Arrow) { + Models::_detail::ArrowConfiguration c; + c.Schema = options.OutputTextConfiguration.m_schema; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ArrowConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); +} else if (options.InputTextConfiguration.m_format.ToString().empty()) { +} else { + AZURE_UNREACHABLE_CODE(); +} + +protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; +if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); +} +protocolLayerOptions.EncryptionScope = m_encryptionScope; +protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; +protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; +protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; +protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; +protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; +auto response = + _detail::BlobClient::Query(*m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + +const auto statusCode = response.RawResponse->GetStatusCode(); +const auto reasonPhrase = response.RawResponse->GetReasonPhrase(); +const auto requestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId) + : std::string(); + +const auto clientRequestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderClientRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderClientRequestId) + : std::string(); + +auto defaultErrorHandler = [statusCode, reasonPhrase, requestId, clientRequestId](BlobQueryError e) { + if (e.IsFatal) { + StorageException exception("Fatal " + e.Name + " at " + std::to_string(e.Position)); + exception.StatusCode = statusCode; + exception.ReasonPhrase = reasonPhrase; + exception.RequestId = requestId; + exception.ClientRequestId = clientRequestId; + exception.ErrorCode = e.Name; + exception.Message = e.Description; + + throw exception; + } +}; + +response.Value.BodyStream = + std::make_unique<_detail::AvroStreamParser>(std::move(response.Value.BodyStream), options.ProgressHandler, + options.ErrorHandler ? options.ErrorHandler : defaultErrorHandler); +return response; +} +*/ +} // namespace Blobs +} // namespace Storage +} // namespace Azure + +#endif diff --git a/source/libs/azure/test/CMakeLists.txt b/source/libs/azure/test/CMakeLists.txt new file mode 100644 index 0000000000..01570df730 --- /dev/null +++ b/source/libs/azure/test/CMakeLists.txt @@ -0,0 +1,18 @@ +aux_source_directory(. AZ_TEST_SRC) + +add_executable(azTest ${AZ_TEST_SRC}) +target_include_directories(azTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/azure" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(azTest + az + gtest_main +) +enable_testing() +add_test( + NAME az_test + COMMAND azTest +) diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp new file mode 100644 index 0000000000..9e963508f8 --- /dev/null +++ b/source/libs/azure/test/azTest.cpp @@ -0,0 +1,457 @@ +#include +#include +#include +#include +/* +#include "walInt.h" + +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); +SWalSyncInfo syncMeta = {0}; + +class WalCleanEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +class WalKeepEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + taosMemoryFree(pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +class WalRetentionEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 0; + cfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +TEST_F(WalCleanEnv, createNew) { + walRollFileInfo(pWal); + ASSERT(pWal->fileInfoSet != NULL); + ASSERT_EQ(pWal->fileInfoSet->size, 1); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + ASSERT_EQ(pInfo->firstVer, 0); + ASSERT_EQ(pInfo->lastVer, -1); + ASSERT_EQ(pInfo->closeTs, -1); + ASSERT_EQ(pInfo->fileSize, 0); +} + +TEST_F(WalCleanEnv, serialize) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + char* ss = NULL; + code = walMetaSerialize(pWal, &ss); + ASSERT(code == 0); + printf("%s\n", ss); + taosMemoryFree(ss); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} + +TEST_F(WalCleanEnv, removeOldMeta) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walSaveMeta(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walSaveMeta(pWal); + ASSERT(code == 0); +} + +TEST_F(WalKeepEnv, readOldMeta) { + walResetEnv(); + int code; + + syncMeta.isWeek = -1; + syncMeta.seqNum = UINT64_MAX; + syncMeta.term = UINT64_MAX; + + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + char* oldss = NULL; + code = walMetaSerialize(pWal, &oldss); + ASSERT(code == 0); + + TearDown(); + SetUp(); + + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + + char* newss = NULL; + code = walMetaSerialize(pWal, &newss); + ASSERT(code == 0); + + int len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for (int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } + taosMemoryFree(oldss); + taosMemoryFree(newss); +} + +TEST_F(WalCleanEnv, write) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanEnv, rollback) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 2); + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanEnv, rollbackMultiFile) { + int code; + for (int i = 0; i < 10; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + if (i == 5) { + walBeginSnapshot(pWal, i, 0); + walEndSnapshot(pWal); + } + } + code = walRollback(pWal, 12); + ASSERT_NE(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + code = walRollback(pWal, 9); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 6); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); + code = walRollback(pWal, 5); + ASSERT_NE(code, 0); + + ASSERT_EQ(pWal->vers.lastVer, 5); + + code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 6); + + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanDeleteEnv, roll) { + int code; + int i; + for (i = 0; i < 100; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + + walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); + walEndSnapshot(pWal); + ASSERT_EQ(pWal->vers.snapshotVer, i - 1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); + + code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_NE(code, 0); + + for (; i < 200; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->vers.commitVer, i); + } + + code = walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(code, 0); + code = walEndSnapshot(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} + +TEST_F(WalRetentionEnv, repairMeta1) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + TearDown(); + + // getchar(); + char buf[100]; + sprintf(buf, "%s/meta-ver%d", pathName, 0); + taosRemoveFile(buf); + sprintf(buf, "%s/meta-ver%d", pathName, 1); + taosRemoveFile(buf); + SetUp(); + // getchar(); + + ASSERT_EQ(pWal->vers.lastVer, 99); + + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 100; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + + for (i = 100; i < 200; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + for (int i = 0; i < 1000; i++) { + int ver = taosRand() % 200; + code = walReadVer(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.bodyLen, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + walCloseReader(pRead); +} +*/ diff --git a/source/libs/tcs/CMakeLists.txt b/source/libs/tcs/CMakeLists.txt index 4d74dedcd0..1c914a18b9 100644 --- a/source/libs/tcs/CMakeLists.txt +++ b/source/libs/tcs/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( tcs - # PUBLIC az + PUBLIC az PUBLIC common # PUBLIC cjson # PUBLIC os diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index c5c68c4933..5facffa4ac 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -19,7 +19,7 @@ #include "taoserror.h" #include "tglobal.h" -//#include "az.h" +#include "az.h" #include "cos.h" extern int8_t tsS3Ablob; @@ -68,7 +68,6 @@ int32_t tcsInit() { tcs.DeleteObjects = s3DeleteObjects; tcs.GetObjectToFile = s3GetObjectToFile; } else if (TOS_PROTO_ABLOB == proto) { - /* tcs.Begin = azBegin; tcs.End = azEnd; tcs.CheckCfg = azCheckCfg; @@ -82,7 +81,7 @@ int32_t tcsInit() { tcs.GetObjectsByPrefix = azGetObjectsByPrefix; tcs.DeleteObjects = azDeleteObjects; tcs.GetObjectToFile = azGetObjectToFile; - */ + } else { code = TSDB_CODE_INVALID_PARA; return code; From 8b73975455809869955d771e45e06b4bc98ffce8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 10 Oct 2024 18:54:57 +0800 Subject: [PATCH 10/41] test/ci/scan_file_path: ignore az cpp module --- tests/ci/scan_file_path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 2d4e701012..9244d37456 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -129,7 +129,7 @@ def scan_files_path(source_file_path): def input_files(change_files): # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] - scan_skip_file_list = [f"{TD_project_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{TD_project_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{TD_project_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + scan_skip_file_list = ["tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", "source/libs/parser/src/sql.c", "source/libs/azure", "source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] with open(change_files, 'r') as file: for line in file: file_name = line.strip() @@ -141,7 +141,7 @@ def input_files(change_files): tdc_file_path = os.path.join(TD_project_path, "community/") file_name = os.path.join(tdc_file_path, file_name) all_file_path.append(file_name) - # print(f"all_file_path:{all_file_path}") + print(f"all_file_path:{all_file_path}") logger.info("Found %s files" % len(all_file_path)) file_res_path = "" From b0625b0ab2cc9e737d69f77495c96da9f8c623c5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 08:38:44 +0800 Subject: [PATCH 11/41] test system return 1 --- source/libs/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 033582f2c0..41a1e99521 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -23,5 +23,5 @@ add_subdirectory(planner) add_subdirectory(qworker) add_subdirectory(geometry) add_subdirectory(command) -add_subdirectory(azure) +#add_subdirectory(azure) add_subdirectory(tcs) From 7792906622eec722628ae62ba043cfa0f9b90dab Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 08:55:27 +0800 Subject: [PATCH 12/41] test az system 1 --- source/libs/CMakeLists.txt | 2 +- source/libs/azure/src/az.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 41a1e99521..033582f2c0 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -23,5 +23,5 @@ add_subdirectory(planner) add_subdirectory(qworker) add_subdirectory(geometry) add_subdirectory(command) -#add_subdirectory(azure) +add_subdirectory(azure) add_subdirectory(tcs) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 83cba1c877..39aadb11b3 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tglobal.h" -#if defined(USE_S3) +#if !defined(USE_S3) #include #include From 4d0ca2c4d4453b0248680c25d6cc67c80d76572e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 13:44:59 +0800 Subject: [PATCH 13/41] rsync: return 0 if no errno --- source/common/src/rsync.c | 8 ++++++-- source/libs/azure/src/az.cpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 47a452eab7..b5ffae6845 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -160,7 +160,11 @@ int32_t startRsync() { code = system(cmd); if (code != 0) { uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA); - code = TAOS_SYSTEM_ERROR(errno); + if (errno == 0) { + return 0; + } else { + code = TAOS_SYSTEM_ERROR(errno); + } } else { uInfo("[rsync] cmd:%s start server successful", cmd); } @@ -358,4 +362,4 @@ int32_t deleteRsync(const char* id) { uDebug("[rsync] delete data:%s successful", id); return 0; -} \ No newline at end of file +} diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 39aadb11b3..83cba1c877 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tglobal.h" -#if !defined(USE_S3) +#if defined(USE_S3) #include #include From d081c73a17c3516e0f7f53d20f54ca6a252bd557 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 14:27:58 +0800 Subject: [PATCH 14/41] az: interfaces for stream checkpoints --- source/libs/azure/src/az.cpp | 116 +++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 83cba1c877..453740224d 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -365,13 +365,121 @@ void azDeleteObjectsByPrefix(const char *prefix) { } } -int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } +int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { + int32_t code = 0, lino = 0; + uint64_t contentLength = 0; -int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; } + if (taosStatFile(file, (int64_t *)&contentLength, NULL, NULL) < 0) { + uError("ERROR: %s Failed to stat file %s: ", __func__, file); + TAOS_RETURN(terrno); + } -int32_t azGetObjectToFile(const char *object_name, const char *fileName) { return 0; } + code = azPutObjectFromFileOffset(file, object, 0, contentLength); + if (code != 0) { + uError("ERROR: %s Failed to put file %s: ", __func__, file); + TAOS_CHECK_GOTO(code, &lino, _exit); + } -int32_t azDeleteObjects(const char *object_name[], int nobject) { return 0; } +_exit: + if (code) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + + return 0; +} + +int32_t azGetObjectToFile(const char *object_name, const char *fileName) { + int32_t code = TSDB_CODE_SUCCESS; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; + std::string accountURL = tsS3Hostname[0]; + accountURL = "https://" + accountURL; + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = tsS3BucketName; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); + + auto res = blobClient.DownloadTo(fileName); + if (res.Value.ContentRange.Length.Value() <= 0) { + code = TAOS_SYSTEM_ERROR(EIO); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } + } catch (const Azure::Core::RequestFailedException &e) { + uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + code = TAOS_SYSTEM_ERROR(EIO); + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { + const std::string delimiter = "/"; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; + std::string accountURL = tsS3Hostname[0]; + accountURL = "https://" + accountURL; + + try { + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = tsS3BucketName; + auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); + + Azure::Storage::Blobs::ListBlobsOptions options; + options.Prefix = prefix; + + std::set listBlobs; + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + for (const auto &blob : pageResult.Blobs) { + listBlobs.insert(blob.Name); + } + } + + for (auto blobName : listBlobs) { + const char *tmp = strchr(blobName.c_str(), '/'); + tmp = (tmp == NULL) ? blobName.c_str() : tmp + 1; + char fileName[PATH_MAX] = {0}; + if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) { + (void)snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp); + } else { + (void)snprintf(fileName, PATH_MAX, "%s%s", path, tmp); + } + if (!azGetObjectToFile(blobName.c_str(), fileName)) { + TAOS_RETURN(TSDB_CODE_FAILED); + } + } + } catch (const Azure::Core::RequestFailedException &e) { + uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + TAOS_RETURN(TSDB_CODE_FAILED); + } + + return 0; +} + +int32_t azDeleteObjects(const char *object_name[], int nobject) { + for (int i = 0; i < nobject; ++i) { + azDeleteObjectsByPrefix(object_name[i]); + } + + return 0; +} #else From 3f403569eb7de14c59c4a6f280a4ded70ade2a76 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 15:57:47 +0800 Subject: [PATCH 15/41] tcs/stream: use tcs interface instead of s3 --- source/libs/stream/CMakeLists.txt | 1 + source/libs/stream/src/streamCheckpoint.c | 62 +++++++++++------------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index b63a8b3900..f08b16f836 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(stream STATIC ${STREAM_SRC}) target_include_directories( stream PUBLIC "${TD_SOURCE_DIR}/include/libs/stream" + PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index e44bca123b..3b730e40cc 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "cos.h" #include "rsync.h" #include "streamBackendRocksdb.h" #include "streamInt.h" +#include "tcs.h" static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName); static int32_t deleteCheckpointFile(const char* id, const char* name); @@ -343,7 +343,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock // And if we don't start a new timer, and the lost of checkpoint-trigger message may cause the whole checkpoint // procedure to be stucked. SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; - int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1); + int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1); if (old == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); @@ -351,7 +351,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); pTmrInfo->launchChkptId = pActiveInfo->activeId; - } else { // already launched, do nothing + } else { // already launched, do nothing stError("s-task:%s previous checkpoint-trigger monitor tmr is set, not start new one", pTask->id.idStr); } } @@ -372,10 +372,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); - code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure + code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure } else { // only one task exists, no need to dispatch downstream info - code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, - -1); + code = + appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1); streamFreeQitem((SStreamQueueItem*)pBlock); } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -398,8 +398,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (taskLevel == TASK_LEVEL__SINK) { stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num); streamFreeQitem((SStreamQueueItem*)pBlock); - code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet - } else { // source & agg tasks need to forward the checkpoint msg downwards + code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet + } else { // source & agg tasks need to forward the checkpoint msg downwards stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); if (code) { @@ -444,7 +444,7 @@ static int32_t processCheckpointReadyHelp(SActiveCheckpointInfo* pInfo, int32_t .transId = pInfo->transId, .streamId = streamId, .downstreamNodeId = downstreamNodeId}; - void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); + void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); if (p == NULL) { stError("s-task:%s failed to set checkpoint ready recv msg, code:%s", id, tstrerror(terrno)); return terrno; @@ -559,8 +559,8 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { } streamMutexUnlock(&pInfo->lock); - stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%"PRId64", current checkpointId:%"PRId64, - pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId); + stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64, + pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId); } int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { @@ -574,8 +574,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV if (pReq->checkpointId <= pInfo->checkpointId) { stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " Ver:%" PRId64 - " no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64 - " transId:%d ignored", + " no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64 " transId:%d ignored", id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, pReq->transId); streamMutexUnlock(&pTask->lock); @@ -622,7 +621,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } bool valid = (pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer && - pInfo->processedVer <= pReq->checkpointVer); + pInfo->processedVer <= pReq->checkpointVer); if (!valid) { stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 @@ -907,7 +906,7 @@ static int32_t doChkptStatusCheck(SStreamTask* pTask) { if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64 - ", quit, ref:%d", + ", quit, ref:%d", id, vgId, pTmrInfo->launchChkptId, ref); return -1; } @@ -1004,7 +1003,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { int32_t numOfNotSend = 0; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); @@ -1022,7 +1021,8 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { } if (++pTmrInfo->activeCounter < 50) { - streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "trigger-recv-monitor"); return; } @@ -1200,8 +1200,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; - void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); - if (px == NULL) { // pause the stream task, if memory not enough + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + if (px == NULL) { // pause the stream task, if memory not enough code = terrno; } } else { @@ -1212,8 +1212,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { } STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; - void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); - if (px == NULL) { // pause the stream task, if memory not enough + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + if (px == NULL) { // pause the stream task, if memory not enough code = terrno; break; } @@ -1287,11 +1287,11 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { static int32_t uploadCheckpointToS3(const char* id, const char* path) { int32_t code = 0; int32_t nBytes = 0; - + /* if (s3Init() != 0) { return TSDB_CODE_THIRDPARTY_ERROR; } - + */ TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) { return terrno; @@ -1324,11 +1324,11 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) { break; } - code = s3PutObjectFromFile2(filename, object, 0); + code = tcsPutObjectFromFile2(filename, object, 0); if (code != 0) { - stError("[s3] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code)); + stError("[tcs] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code)); } else { - stDebug("[s3] upload checkpoint:%s", filename); + stDebug("[tcs] upload checkpoint:%s", filename); } } @@ -1354,7 +1354,7 @@ int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char taosMemoryFree(buf); return TSDB_CODE_OUT_OF_RANGE; } - int32_t code = s3GetObjectToFile(buf, dstName); + int32_t code = tcsGetObjectToFile(buf, dstName); if (code != 0) { taosMemoryFree(buf); return TAOS_SYSTEM_ERROR(errno); @@ -1417,7 +1417,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path, int64_t che if (strlen(tsSnodeAddress) != 0) { return downloadByRsync(id, path, checkpointId); } else if (tsS3StreamEnabled) { - return s3GetObjectsByPrefix(id, path); + return tcsGetObjectsByPrefix(id, path); } return 0; @@ -1431,7 +1431,7 @@ int32_t deleteCheckpoint(const char* id) { if (strlen(tsSnodeAddress) != 0) { return deleteRsync(id); } else if (tsS3StreamEnabled) { - s3DeleteObjectsByPrefix(id); + tcsDeleteObjectsByPrefix(id); } return 0; } @@ -1445,7 +1445,7 @@ int32_t deleteCheckpointFile(const char* id, const char* name) { } char* tmp = object; - int32_t code = s3DeleteObjects((const char**)&tmp, 1); + int32_t code = tcsDeleteObjects((const char**)&tmp, 1); if (code != 0) { return TSDB_CODE_THIRDPARTY_ERROR; } @@ -1487,4 +1487,4 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) { streamMutexUnlock(&pTask->lock); return code; -} \ No newline at end of file +} From f6f278d8126ac6db16c5137adb23250ce982c097 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 12 Oct 2024 16:37:31 +0800 Subject: [PATCH 16/41] test/ablob: integration test form az --- tests/army/storage/blob/ablob.py | 344 ++++++++++++++++++++++++++ tests/army/storage/blob/s3Basic.json | 66 +++++ tests/army/storage/blob/s3Basic1.json | 66 +++++ 3 files changed, 476 insertions(+) create mode 100644 tests/army/storage/blob/ablob.py create mode 100644 tests/army/storage/blob/s3Basic.json create mode 100644 tests/army/storage/blob/s3Basic1.json diff --git a/tests/army/storage/blob/ablob.py b/tests/army/storage/blob/ablob.py new file mode 100644 index 0000000000..fae492a3df --- /dev/null +++ b/tests/army/storage/blob/ablob.py @@ -0,0 +1,344 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool +import frame.eos +import frame.eutil + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * +from frame.eos import * + + +class TDTestCase(TBase): + index = eutil.cpuRand(20) + 1 + bucketName = f"ci-bucket{index}" + updatecfgDict = { + "supportVnodes":"1000", + 's3EndPoint': 'https://.blob.core.windows.net', + 's3AccessKey': ':', + 's3BucketName': '', + 's3PageCacheSize': '10240', + "s3UploadDelaySec": "10", + 's3MigrateIntervalSec': '600', + 's3MigrateEnabled': '1' + } + + tdLog.info(f"assign bucketName is {bucketName}\n") + maxFileSize = (128 + 10) * 1014 * 1024 # add 10M buffer + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = etool.curFile(__file__, "s3Basic.json") + etool.benchMark(json=json) + + tdSql.execute(f"use {self.db}") + # come from s3_basic.json + self.childtable_count = 6 + self.insert_rows = 2000000 + self.timestamp_step = 100 + + def createStream(self, sname): + sql = f"create stream {sname} fill_history 1 into stm1 as select count(*) from {self.db}.{self.stb} interval(10s);" + tdSql.execute(sql) + + def migrateDbS3(self): + sql = f"s3migrate database {self.db}" + tdSql.execute(sql, show=True) + + def checkDataFile(self, lines, maxFileSize): + # ls -l + # -rwxrwxrwx 1 root root 41652224 Apr 17 14:47 vnode2/tsdb/v2f1974ver47.3.data + overCnt = 0 + for line in lines: + cols = line.split() + fileSize = int(cols[4]) + fileName = cols[8] + #print(f" filesize={fileSize} fileName={fileName} line={line}") + if fileSize > maxFileSize: + tdLog.info(f"error, {fileSize} over max size({maxFileSize}) {fileName}\n") + overCnt += 1 + else: + tdLog.info(f"{fileName}({fileSize}) check size passed.") + + return overCnt + + def checkUploadToS3(self): + rootPath = sc.clusterRootPath() + cmd = f"ls -l {rootPath}/dnode*/data/vnode/vnode*/tsdb/*.data" + tdLog.info(cmd) + loop = 0 + rets = [] + overCnt = 0 + while loop < 200: + time.sleep(3) + + # check upload to s3 + rets = eos.runRetList(cmd) + cnt = len(rets) + if cnt == 0: + overCnt = 0 + tdLog.info("All data file upload to server over.") + break + overCnt = self.checkDataFile(rets, self.maxFileSize) + if overCnt == 0: + uploadOK = True + tdLog.info(f"All data files({len(rets)}) size bellow {self.maxFileSize}, check upload to s3 ok.") + break + + tdLog.info(f"loop={loop} no upload {overCnt} data files wait 3s retry ...") + if loop == 3: + sc.dnodeStop(1) + time.sleep(2) + sc.dnodeStart(1) + loop += 1 + # migrate + self.migrateDbS3() + + # check can pass + if overCnt > 0: + tdLog.exit(f"s3 have {overCnt} files over size.") + + + def doAction(self): + tdLog.info(f"do action.") + + self.flushDb(show=True) + #self.compactDb(show=True) + + # sleep 70s + self.migrateDbS3() + + # check upload to s3 + self.checkUploadToS3() + + def checkStreamCorrect(self): + sql = f"select count(*) from {self.db}.stm1" + count = 0 + for i in range(120): + tdSql.query(sql) + count = tdSql.getData(0, 0) + if count == 100000 or count == 100001: + return True + time.sleep(1) + + tdLog.exit(f"stream count is not expect . expect = 100000 or 100001 real={count} . sql={sql}") + + + def checkCreateDb(self, keepLocal, chunkSize, compact): + # keyword + kw1 = kw2 = kw3 = "" + if keepLocal is not None: + kw1 = f"s3_keeplocal {keepLocal}" + if chunkSize is not None: + kw2 = f"s3_chunksize {chunkSize}" + if compact is not None: + kw3 = f"s3_compact {compact}" + + sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}" + tdSql.execute(sql, show=True) + #sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';" + sql = f"select * from information_schema.ins_databases where name='db1';" + tdSql.query(sql) + # 29 30 31 -> chunksize keeplocal compact + if chunkSize is not None: + tdSql.checkData(0, 29, chunkSize) + if keepLocal is not None: + keepLocalm = keepLocal * 24 * 60 + tdSql.checkData(0, 30, f"{keepLocalm}m") + if compact is not None: + tdSql.checkData(0, 31, compact) + sql = "drop database db1" + tdSql.execute(sql) + + def checkExcept(self): + # errors + sqls = [ + f"create database db2 s3_keeplocal -1", + f"create database db2 s3_keeplocal 0", + f"create database db2 s3_keeplocal 365001", + f"create database db2 s3_chunksize -1", + f"create database db2 s3_chunksize 0", + f"create database db2 s3_chunksize 900000000", + f"create database db2 s3_compact -1", + f"create database db2 s3_compact 100", + f"create database db2 duration 1d s3_keeplocal 1d" + ] + tdSql.errors(sqls) + + + def checkBasic(self): + # create db + keeps = [1, 256, 1024, 365000, None] + chunks = [131072, 600000, 820000, 1048576, None] + comps = [0, 1, None] + + for keep in keeps: + for chunk in chunks: + for comp in comps: + self.checkCreateDb(keep, chunk, comp) + + + # --checks3 + idx = 1 + taosd = sc.taosdFile(idx) + cfg = sc.dnodeCfgPath(idx) + cmd = f"{taosd} -c {cfg} --checks3" + + eos.exe(cmd) + #output, error = eos.run(cmd) + #print(lines) + + ''' + tips = [ + "put object s3test.txt: success", + "listing bucket ci-bucket: success", + "get object s3test.txt: success", + "delete object s3test.txt: success" + ] + pos = 0 + for tip in tips: + pos = output.find(tip, pos) + #if pos == -1: + # tdLog.exit(f"checks3 failed not found {tip}. cmd={cmd} output={output}") + ''' + + # except + self.checkExcept() + + # + def preDb(self, vgroups): + cnt = int(time.time())%2 + 1 + for i in range(cnt): + vg = eutil.cpuRand(9) + 1 + sql = f"create database predb vgroups {vg}" + tdSql.execute(sql, show=True) + sql = "drop database predb" + tdSql.execute(sql, show=True) + + # history + def insertHistory(self): + tdLog.info(f"insert history data.") + # taosBenchmark run + json = etool.curFile(__file__, "s3Basic1.json") + etool.benchMark(json=json) + + # come from s3_basic.json + self.insert_rows += self.insert_rows/4 + self.timestamp_step = 50 + + # delete + def checkDelete(self): + # del 1000 rows + start = 1600000000000 + drows = 200 + for i in range(1, drows, 2): + sql = f"from {self.db}.{self.stb} where ts = {start + i*500}" + tdSql.execute("delete " + sql, show=True) + tdSql.query("select * " + sql) + tdSql.checkRows(0) + + # delete all 500 step + self.flushDb() + self.compactDb() + self.insert_rows -= drows/2 + sql = f"select count(*) from {self.db}.{self.stb}" + tdSql.checkAgg(sql, self.insert_rows * self.childtable_count) + + # delete 10W rows from 100000 + drows = 100000 + sdel = start + 100000 * self.timestamp_step + edel = start + 100000 * self.timestamp_step + drows * self.timestamp_step + sql = f"from {self.db}.{self.stb} where ts >= {sdel} and ts < {edel}" + tdSql.execute("delete " + sql, show=True) + tdSql.query("select * " + sql) + tdSql.checkRows(0) + + self.insert_rows -= drows + sql = f"select count(*) from {self.db}.{self.stb}" + tdSql.checkAgg(sql, self.insert_rows * self.childtable_count) + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + self.sname = "stream1" + if eos.isArm64Cpu(): + tdLog.success(f"{__file__} arm64 ignore executed") + else: + + self.preDb(10) + + # insert data + self.insertData() + + # creat stream + self.createStream(self.sname) + + # check insert data correct + #self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + + # check stream correct and drop stream + #self.checkStreamCorrect() + + # drop stream + self.dropStream(self.sname) + + # insert history disorder data + self.insertHistory() + + # checkBasic + self.checkBasic() + + #self.checkInsertCorrect() + self.snapshotAgg() + self.doAction() + self.checkAggCorrect() + self.checkInsertCorrect(difCnt=self.childtable_count*1499999) + self.checkDelete() + self.doAction() + + # drop database and free s3 file + self.dropDb() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/storage/blob/s3Basic.json b/tests/army/storage/blob/s3Basic.json new file mode 100644 index 0000000000..ee341b2096 --- /dev/null +++ b/tests/army/storage/blob/s3Basic.json @@ -0,0 +1,66 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 4000, + "prepared_rand": 500, + "thread_count": 4, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"10d", + "s3_keeplocal":"30d", + "s3_chunksize":"131072", + "tsdb_pagesize":"1", + "s3_compact":"1", + "wal_retention_size":"1", + "wal_retention_period":"1", + "flush_each_batch":"no", + "keep": "3650d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 6, + "insert_rows": 2000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 100, + "start_timestamp": 1600000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 50}, + { "type": "nchar", "name": "nch", "len": 100} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/storage/blob/s3Basic1.json b/tests/army/storage/blob/s3Basic1.json new file mode 100644 index 0000000000..02be308443 --- /dev/null +++ b/tests/army/storage/blob/s3Basic1.json @@ -0,0 +1,66 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 5000, + "prepared_rand": 500, + "thread_count": 4, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "no", + "vgroups": 2, + "replica": 1, + "duration":"10d", + "s3_keeplocal":"30d", + "s3_chunksize":"131072", + "tsdb_pagesize":"1", + "s3_compact":"1", + "wal_retention_size":"1", + "wal_retention_period":"1", + "flush_each_batch":"no", + "keep": "3650d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "yes", + "childtable_count": 6, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 50, + "start_timestamp": 1600000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 50}, + { "type": "nchar", "name": "nch", "len": 100} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} From 18cd7fd01cc8459a0e372048132cc247e40dc391 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 14 Oct 2024 15:34:31 +0800 Subject: [PATCH 17/41] az/get object block: use random 1~3 seconds for retries --- source/libs/azure/src/az.cpp | 50 ++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 453740224d..0af669c56a 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -268,18 +268,19 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int TAOS_RETURN(code); } -int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { +int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int32_t code = TSDB_CODE_SUCCESS; std::string accountName = tsS3AccessKeyId[0]; std::string accountKey = tsS3AccessKeySecret[0]; std::string accountURL = tsS3Hostname[0]; - accountURL = "https://" + accountURL; + uint8_t *buf = NULL; try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + accountURL = "https://" + accountURL; BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); std::string containerName = tsS3BucketName; @@ -287,26 +288,15 @@ int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); - uint8_t *buf = (uint8_t *)taosMemoryCalloc(1, size); - if (!buf) { - return terrno; - } - Blobs::DownloadBlobToOptions options; - // options.TransferOptions.Concurrency = concurrency; - // if (offset.HasValue() || length.HasValue()) { options.Range = Azure::Core::Http::HttpRange(); options.Range.Value().Offset = offset; options.Range.Value().Length = size; - //} - /* - if (initialChunkSize.HasValue()) { - options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + + buf = (uint8_t *)taosMemoryCalloc(1, size); + if (!buf) { + return terrno; } - if (chunkSize.HasValue()) { - options.TransferOptions.ChunkSize = chunkSize.Value(); - } - */ auto res = blobClient.DownloadTo(buf, size, options); if (check && res.Value.ContentRange.Length.Value() != size) { @@ -321,12 +311,38 @@ int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, e.ReasonPhrase.c_str()); code = TAOS_SYSTEM_ERROR(EIO); uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + + if (buf) { + taosMemoryFree(buf); + } + *ppBlock = NULL; + TAOS_RETURN(code); } TAOS_RETURN(code); } +int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + int32_t code = TSDB_CODE_SUCCESS; + + // May use an exponential backoff policy for retries with 503 + int retryCount = 0; + static int maxRetryCount = 5; + static int minRetryInterval = 1000; // ms + static int maxRetryInterval = 3000; // ms + +_retry: + code = azGetObjectBlockImpl(object_name, offset, size, check, ppBlock); + if (TSDB_CODE_SUCCESS != code && retryCount++ < maxRetryCount) { + taosMsleep(taosRand() % (maxRetryInterval - minRetryInterval + 1) + minRetryInterval); + uInfo("%s: 0x%x(%s) and retry get object", __func__, code, tstrerror(code)); + goto _retry; + } + + TAOS_RETURN(code); +} + void azDeleteObjectsByPrefix(const char *prefix) { const std::string delimiter = "/"; std::string accountName = tsS3AccessKeyId[0]; From 88d730f7eda456f7be8c7803edcd66f8514045f8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 15 Oct 2024 12:47:10 +0800 Subject: [PATCH 18/41] az: clean up avro --- source/libs/azure/inc/td_avro_parser.h | 191 +++++++ source/libs/azure/src/td_avro_parser.cpp | 531 ++++++++++++++++++ .../libs/azure/src/td_block_blob_client.cpp | 2 +- 3 files changed, 723 insertions(+), 1 deletion(-) create mode 100644 source/libs/azure/inc/td_avro_parser.h create mode 100644 source/libs/azure/src/td_avro_parser.cpp diff --git a/source/libs/azure/inc/td_avro_parser.h b/source/libs/azure/inc/td_avro_parser.h new file mode 100644 index 0000000000..dae5a65dc7 --- /dev/null +++ b/source/libs/azure/inc/td_avro_parser.h @@ -0,0 +1,191 @@ +#pragma once + +#include "azure/storage/blobs/blob_options.hpp" + +#include + +#include +#include +#include + +namespace Azure { +namespace Storage { +namespace Blobs { +namespace _detail { +enum class AvroDatumType { + String, + Bytes, + Int, + Long, + Float, + Double, + Bool, + Null, + Record, + Enum, + Array, + Map, + Union, + Fixed, +}; + +class AvroStreamReader final { + public: + // position of a vector that lives through vector resizing + struct ReaderPos final { + const std::vector* BufferPtr = nullptr; + size_t Offset = 0; + }; + explicit AvroStreamReader(Core::IO::BodyStream& stream) : m_stream(&stream), m_pos{&m_streambuffer, 0} {} + AvroStreamReader(const AvroStreamReader&) = delete; + AvroStreamReader& operator=(const AvroStreamReader&) = delete; + + int64_t ParseInt(const Core::Context& context); + void Advance(size_t n, const Core::Context& context); + // Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes + // available in m_streambuffer; + size_t Preload(size_t n, const Core::Context& context); + size_t TryPreload(size_t n, const Core::Context& context); + // discards data that's before m_pos + void Discard(); + + private: + size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; } + + private: + Core::IO::BodyStream* m_stream; + std::vector m_streambuffer; + ReaderPos m_pos; + + friend class AvroDatum; +}; + +class AvroSchema final { + public: + static const AvroSchema StringSchema; + static const AvroSchema BytesSchema; + static const AvroSchema IntSchema; + static const AvroSchema LongSchema; + static const AvroSchema FloatSchema; + static const AvroSchema DoubleSchema; + static const AvroSchema BoolSchema; + static const AvroSchema NullSchema; + static AvroSchema RecordSchema(std::string name, const std::vector>& fieldsSchema); + static AvroSchema ArraySchema(AvroSchema elementSchema); + static AvroSchema MapSchema(AvroSchema elementSchema); + static AvroSchema UnionSchema(std::vector schemas); + static AvroSchema FixedSchema(std::string name, int64_t size); + + const std::string& Name() const { return m_name; } + AvroDatumType Type() const { return m_type; } + const std::vector& FieldNames() const { return m_status->m_keys; } + AvroSchema ItemSchema() const { return m_status->m_schemas[0]; } + const std::vector& FieldSchemas() const { return m_status->m_schemas; } + size_t Size() const { return static_cast(m_status->m_size); } + + private: + explicit AvroSchema(AvroDatumType type) : m_type(type) {} + + private: + AvroDatumType m_type; + std::string m_name; + + struct SharedStatus { + std::vector m_keys; + std::vector m_schemas; + int64_t m_size = 0; + }; + std::shared_ptr m_status; +}; + +class AvroDatum final { + public: + AvroDatum() : m_schema(AvroSchema::NullSchema) {} + explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {} + + void Fill(AvroStreamReader& reader, const Core::Context& context); + void Fill(AvroStreamReader::ReaderPos& data); + + const AvroSchema& Schema() const { return m_schema; } + + template + T Value() const; + struct StringView { + const uint8_t* Data = nullptr; + size_t Length = 0; + }; + + private: + AvroSchema m_schema; + AvroStreamReader::ReaderPos m_data; +}; + +using AvroMap = std::map; + +class AvroRecord final { + public: + bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); } + const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); } + AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); } + const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); } + AvroDatum& FieldAt(size_t i) { return m_values.at(i); } + + private: + size_t FindField(const std::string& key) const { + auto i = find(m_keys->begin(), m_keys->end(), key); + return i - m_keys->begin(); + } + const std::vector* m_keys = nullptr; + std::vector m_values; + + friend class AvroDatum; +}; + +class AvroObjectContainerReader final { + public: + explicit AvroObjectContainerReader(Core::IO::BodyStream& stream); + + bool End() const { return m_eof; } + // Calling Next() will invalidates the previous AvroDatum returned by this function and all + // AvroDatums propagated from there. + AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); } + + private: + AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context); + + private: + std::unique_ptr m_reader; + std::unique_ptr m_objectSchema; + std::string m_syncMarker; + int64_t m_remainingObjectInCurrentBlock = 0; + bool m_eof = false; +}; + +class AvroStreamParser final : public Core::IO::BodyStream { + public: + explicit AvroStreamParser(std::unique_ptr inner, + std::function progressCallback, + std::function errorCallback) + : m_inner(std::move(inner)), + m_parser(*m_inner), + m_progressCallback(std::move(progressCallback)), + m_errorCallback(std::move(errorCallback)) {} + + int64_t Length() const override { return -1; } + void Rewind() override { this->m_inner->Rewind(); } + + private: + size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override; + + private: + std::unique_ptr m_inner; + AvroObjectContainerReader m_parser; + std::function m_progressCallback; + std::function m_errorCallback; + AvroDatum::StringView m_parserBuffer; +}; + +} // namespace _detail +} // namespace Blobs +} // namespace Storage +} // namespace Azure diff --git a/source/libs/azure/src/td_avro_parser.cpp b/source/libs/azure/src/td_avro_parser.cpp new file mode 100644 index 0000000000..485980e007 --- /dev/null +++ b/source/libs/azure/src/td_avro_parser.cpp @@ -0,0 +1,531 @@ +#if defined(USE_S3) +#include "avro_parser.hpp" + +#include +#include + +#include +#include + +namespace Azure { +namespace Storage { +namespace Blobs { +namespace _detail { + +namespace { +int64_t parseInt(AvroStreamReader::ReaderPos& data) { + uint64_t r = 0; + int nb = 0; + while (true) { + uint8_t c = (*data.BufferPtr)[data.Offset++]; + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); +} + +AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) { + const static std::map BuiltinNameSchemaMap = { + {"string", AvroSchema::StringSchema}, {"bytes", AvroSchema::BytesSchema}, {"int", AvroSchema::IntSchema}, + {"long", AvroSchema::LongSchema}, {"float", AvroSchema::FloatSchema}, {"double", AvroSchema::DoubleSchema}, + {"boolean", AvroSchema::BoolSchema}, {"null", AvroSchema::NullSchema}, {"string", AvroSchema::StringSchema}, + }; + std::map nameSchemaMap = BuiltinNameSchemaMap; + + std::function parseSchemaFromJsonObject; + parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema { + if (obj.is_string()) { + auto typeName = obj.get(); + return nameSchemaMap.find(typeName)->second; + } else if (obj.is_array()) { + std::vector unionSchemas; + for (const auto& s : obj) { + unionSchemas.push_back(parseSchemaFromJsonObject(s)); + } + return AvroSchema::UnionSchema(std::move(unionSchemas)); + } else if (obj.is_object()) { + if (obj.count("namespace") != 0) { + throw std::runtime_error("Namespace isn't supported yet in Avro schema."); + } + if (obj.count("aliases") != 0) { + throw std::runtime_error("Alias isn't supported yet in Avro schema."); + } + auto typeName = obj["type"].get(); + auto i = nameSchemaMap.find(typeName); + if (i != nameSchemaMap.end()) { + return i->second; + } + if (typeName == "record") { + std::vector> fieldsSchema; + for (const auto& field : obj["fields"]) { + fieldsSchema.push_back( + std::make_pair(field["name"].get(), parseSchemaFromJsonObject(field["type"]))); + } + + const std::string recordName = obj["name"].get(); + auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema)); + nameSchemaMap.insert(std::make_pair(recordName, recordSchema)); + return recordSchema; + } else if (typeName == "enum") { + throw std::runtime_error("Enum type isn't supported yet in Avro schema."); + } else if (typeName == "array") { + return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"])); + } else if (typeName == "map") { + return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"])); + } else if (typeName == "fixed") { + const std::string fixedName = obj["name"].get(); + auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get()); + nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema)); + return fixedSchema; + } else { + throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema."); + } + } + AZURE_UNREACHABLE_CODE(); + }; + + auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end()); + return parseSchemaFromJsonObject(jsonRoot); +} +} // namespace + +int64_t AvroStreamReader::ParseInt(const Core::Context& context) { + uint64_t r = 0; + int nb = 0; + while (true) { + Preload(1, context); + uint8_t c = m_streambuffer[m_pos.Offset++]; + + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); +} + +void AvroStreamReader::Advance(size_t n, const Core::Context& context) { + Preload(n, context); + m_pos.Offset += n; +} + +size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) { + size_t oldAvailable = AvailableBytes(); + while (true) { + size_t newAvailable = TryPreload(n, context); + if (newAvailable >= n) { + return newAvailable; + } + if (oldAvailable == newAvailable) { + throw std::runtime_error("Unexpected EOF of Avro stream."); + } + oldAvailable = newAvailable; + } + AZURE_UNREACHABLE_CODE(); +} + +size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) { + size_t availableBytes = AvailableBytes(); + if (availableBytes >= n) { + return availableBytes; + } + const size_t MinRead = 4096; + size_t tryReadSize = (std::max)(n, MinRead); + size_t currSize = m_streambuffer.size(); + m_streambuffer.resize(m_streambuffer.size() + tryReadSize); + size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context); + m_streambuffer.resize(currSize + actualReadSize); + return AvailableBytes(); +} + +void AvroStreamReader::Discard() { + constexpr size_t MinimumReleaseMemory = 128 * 1024; + if (m_pos.Offset < MinimumReleaseMemory) { + return; + } + const size_t availableBytes = AvailableBytes(); + std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes); + m_streambuffer.resize(availableBytes); + m_pos.Offset = 0; +} + +const AvroSchema AvroSchema::StringSchema(AvroDatumType::String); +const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes); +const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int); +const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long); +const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float); +const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double); +const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool); +const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null); + +AvroSchema AvroSchema::RecordSchema(std::string name, + const std::vector>& fieldsSchema) { + AvroSchema recordSchema(AvroDatumType::Record); + recordSchema.m_name = std::move(name); + recordSchema.m_status = std::make_shared(); + for (auto& i : fieldsSchema) { + recordSchema.m_status->m_keys.push_back(i.first); + recordSchema.m_status->m_schemas.push_back(i.second); + } + return recordSchema; +} + +AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) { + AvroSchema arraySchema(AvroDatumType::Array); + arraySchema.m_status = std::make_shared(); + arraySchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return arraySchema; +} + +AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) { + AvroSchema mapSchema(AvroDatumType::Map); + mapSchema.m_status = std::make_shared(); + mapSchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return mapSchema; +} + +AvroSchema AvroSchema::UnionSchema(std::vector schemas) { + AvroSchema unionSchema(AvroDatumType::Union); + unionSchema.m_status = std::make_shared(); + unionSchema.m_status->m_schemas = std::move(schemas); + return unionSchema; +} + +AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) { + AvroSchema fixedSchema(AvroDatumType::Fixed); + fixedSchema.m_name = std::move(name); + fixedSchema.m_status = std::make_shared(); + fixedSchema.m_status->m_size = size; + return fixedSchema; +} + +void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) { + m_data = reader.m_pos; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + int64_t stringSize = reader.ParseInt(context); + reader.Advance(static_cast(stringSize), context); + } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || + m_schema.Type() == AvroDatumType::Enum) { + reader.ParseInt(context); + } else if (m_schema.Type() == AvroDatumType::Float) { + reader.Advance(4, context); + } else if (m_schema.Type() == AvroDatumType::Double) { + reader.Advance(8, context); + } else if (m_schema.Type() == AvroDatumType::Bool) { + reader.Advance(1, context); + } else if (m_schema.Type() == AvroDatumType::Null) { + reader.Advance(0, context); + } else if (m_schema.Type() == AvroDatumType::Record) { + for (const auto& s : m_schema.FieldSchemas()) { + AvroDatum(s).Fill(reader, context); + } + } else if (m_schema.Type() == AvroDatumType::Array) { + while (true) { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } else { + for (auto i = 0; i < numElementsInBlock; ++i) { + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Map) { + while (true) { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } else { + for (int64_t i = 0; i < numElementsInBlock; ++i) { + AvroDatum(AvroSchema::StringSchema).Fill(reader, context); + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = reader.ParseInt(context); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(reader, context); + } else if (m_schema.Type() == AvroDatumType::Fixed) { + reader.Advance(m_schema.Size(), context); + } else { + AZURE_UNREACHABLE_CODE(); + } +} + +void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) { + m_data = data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + int64_t stringSize = parseInt(data); + data.Offset += static_cast(stringSize); + } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || + m_schema.Type() == AvroDatumType::Enum) { + parseInt(data); + } else if (m_schema.Type() == AvroDatumType::Float) { + data.Offset += 4; + } else if (m_schema.Type() == AvroDatumType::Double) { + data.Offset += 8; + } else if (m_schema.Type() == AvroDatumType::Bool) { + data.Offset += 1; + } else if (m_schema.Type() == AvroDatumType::Null) { + data.Offset += 0; + } else if (m_schema.Type() == AvroDatumType::Record) { + for (const auto& s : m_schema.FieldSchemas()) { + AvroDatum(s).Fill(data); + } + } else if (m_schema.Type() == AvroDatumType::Array) { + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } else { + for (auto i = 0; i < numElementsInBlock; ++i) { + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Map) { + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } else if (numElementsInBlock < 0) { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } else { + for (int64_t i = 0; i < numElementsInBlock; ++i) { + AvroDatum(AvroSchema::StringSchema).Fill(data); + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } else if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = parseInt(data); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(data); + } else if (m_schema.Type() == AvroDatumType::Fixed) { + data.Offset += m_schema.Size(); + } else { + AZURE_UNREACHABLE_CODE(); + } +} + +template <> +AvroDatum::StringView AvroDatum::Value() const { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { + const int64_t length = parseInt(data); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, static_cast(length)}; + data.Offset += static_cast(length); + return ret; + } + if (m_schema.Type() == AvroDatumType::Fixed) { + const size_t fixedSize = m_schema.Size(); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, fixedSize}; + data.Offset += fixedSize; + return ret; + } + AZURE_UNREACHABLE_CODE(); +} + +template <> +std::string AvroDatum::Value() const { + auto stringView = Value(); + return std::string(stringView.Data, stringView.Data + stringView.Length); +} + +template <> +std::vector AvroDatum::Value() const { + auto stringView = Value(); + return std::vector(stringView.Data, stringView.Data + stringView.Length); +} + +template <> +int64_t AvroDatum::Value() const { + auto data = m_data; + return parseInt(data); +} + +template <> +int32_t AvroDatum::Value() const { + return static_cast(Value()); +} + +template <> +bool AvroDatum::Value() const { + return Value(); +} + +template <> +std::nullptr_t AvroDatum::Value() const { + return nullptr; +} + +template <> +AvroRecord AvroDatum::Value() const { + auto data = m_data; + + AvroRecord r; + r.m_keys = &m_schema.FieldNames(); + for (const auto& schema : m_schema.FieldSchemas()) { + auto datum = AvroDatum(schema); + datum.Fill(data); + r.m_values.push_back(std::move(datum)); + } + + return r; +} + +template <> +AvroMap AvroDatum::Value() const { + auto data = m_data; + + AvroMap m; + while (true) { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) { + break; + } + if (numElementsInBlock < 0) { + numElementsInBlock = -numElementsInBlock; + parseInt(data); + } + for (int64_t i = 0; i < numElementsInBlock; ++i) { + auto keyDatum = AvroDatum(AvroSchema::StringSchema); + keyDatum.Fill(data); + auto valueDatum = AvroDatum(m_schema.ItemSchema()); + valueDatum.Fill(data); + m[keyDatum.Value()] = valueDatum; + } + } + return m; +} + +template <> +AvroDatum AvroDatum::Value() const { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::Union) { + int64_t i = parseInt(data); + auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast(i)]); + datum.Fill(data); + return datum; + } + AZURE_UNREACHABLE_CODE(); +} + +AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream) + : m_reader(std::make_unique(stream)) {} + +AvroDatum AvroObjectContainerReader::NextImpl(const AvroSchema* schema, const Core::Context& context) { + AZURE_ASSERT_FALSE(m_eof); + static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16); + if (!schema) { + static AvroSchema FileHeaderSchema = []() { + std::vector> fieldsSchema; + fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4))); + fieldsSchema.push_back(std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema))); + fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema)); + return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema)); + }(); + auto fileHeaderDatum = AvroDatum(FileHeaderSchema); + fileHeaderDatum.Fill(*m_reader, context); + auto fileHeader = fileHeaderDatum.Value(); + if (fileHeader.Field("magic").Value() != "Obj\01") { + throw std::runtime_error("Invalid Avro object container magic."); + } + AvroMap meta = fileHeader.Field("meta").Value(); + std::string objectSchemaJson = meta["avro.schema"].Value(); + std::string codec = "null"; + if (meta.count("avro.codec") != 0) { + codec = meta["avro.codec"].Value(); + } + if (codec != "null") { + throw std::runtime_error("Unsupported Avro codec: " + codec); + } + m_syncMarker = fileHeader.Field("sync").Value(); + m_objectSchema = std::make_unique(ParseSchemaFromJsonString(objectSchemaJson)); + schema = m_objectSchema.get(); + } + + if (m_remainingObjectInCurrentBlock == 0) { + m_reader->Discard(); + m_remainingObjectInCurrentBlock = m_reader->ParseInt(context); + int64_t ObjectsSize = m_reader->ParseInt(context); + m_reader->Preload(static_cast(ObjectsSize), context); + } + + auto objectDatum = AvroDatum(*m_objectSchema); + objectDatum.Fill(*m_reader, context); + if (--m_remainingObjectInCurrentBlock == 0) { + auto markerDatum = AvroDatum(SyncMarkerSchema); + markerDatum.Fill(*m_reader, context); + auto marker = markerDatum.Value(); + if (marker != m_syncMarker) { + throw std::runtime_error("Sync marker doesn't match."); + } + m_eof = m_reader->TryPreload(1, context) == 0; + } + return objectDatum; +} + +size_t AvroStreamParser::OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) { + if (m_parserBuffer.Length != 0) { + size_t bytesToCopy = (std::min)(m_parserBuffer.Length, count); + std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy); + m_parserBuffer.Data += bytesToCopy; + m_parserBuffer.Length -= bytesToCopy; + return bytesToCopy; + } + while (!m_parser.End()) { + auto datum = m_parser.Next(context); + if (datum.Schema().Type() == AvroDatumType::Union) { + datum = datum.Value(); + } + if (datum.Schema().Type() != AvroDatumType::Record) { + continue; + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") { + auto record = datum.Value(); + auto dataDatum = record.Field("data"); + m_parserBuffer = dataDatum.Value(); + return OnRead(buffer, count, context); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" && m_progressCallback) { + auto record = datum.Value(); + auto bytesScanned = record.Field("bytesScanned").Value(); + auto totalBytes = record.Field("totalBytes").Value(); + m_progressCallback(bytesScanned, totalBytes); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" && m_errorCallback) { + auto record = datum.Value(); + BlobQueryError e; + e.Name = record.Field("name").Value(); + e.Description = record.Field("description").Value(); + e.IsFatal = record.Field("fatal").Value(); + e.Position = record.Field("position").Value(); + m_errorCallback(std::move(e)); + } + } + return 0; +} +} // namespace _detail +} // namespace Blobs +} // namespace Storage +} // namespace Azure + +#endif diff --git a/source/libs/azure/src/td_block_blob_client.cpp b/source/libs/azure/src/td_block_blob_client.cpp index e75c6ae17f..b5a5c3c189 100644 --- a/source/libs/azure/src/td_block_blob_client.cpp +++ b/source/libs/azure/src/td_block_blob_client.cpp @@ -14,7 +14,7 @@ #include #endif -#include "./avro_parser.hpp" +#include "avro_parser.hpp" #include #include From af6ff0dd6abefd537f6cb3c09a000da9ebcf647b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 16 Oct 2024 14:46:14 +0800 Subject: [PATCH 19/41] tcs: separate stream related stuff into a standalone module --- source/libs/tcs/inc/tcsInt.h | 59 +++++++++++++++++++++++++++++++++ source/libs/tcs/src/tcs.c | 46 ++----------------------- source/libs/tcs/src/tcsStream.c | 29 ++++++++++++++++ 3 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 source/libs/tcs/inc/tcsInt.h create mode 100644 source/libs/tcs/src/tcsStream.c diff --git a/source/libs/tcs/inc/tcsInt.h b/source/libs/tcs/inc/tcsInt.h new file mode 100644 index 0000000000..b24a47aa98 --- /dev/null +++ b/source/libs/tcs/inc/tcsInt.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TCS_INT_H_ +#define _TD_TCS_INT_H_ + +#include "os.h" +#include "tarray.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int8_t tsS3Ablob; + +typedef enum { + TOS_PROTO_NIL, + TOS_PROTO_S3, + TOS_PROTO_ABLOB, +} STosProto; + +typedef struct { + int32_t (*Begin)(); + void (*End)(); + int32_t (*CheckCfg)(); + + int32_t (*PutObjectFromFileOffset)(const char* file, const char* object_name, int64_t offset, int64_t size); + int32_t (*GetObjectBlock)(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock); + + void (*DeleteObjectsByPrefix)(const char* prefix); + + int32_t (*PutObjectFromFile2)(const char* file, const char* object, int8_t withcp); + int32_t (*GetObjectsByPrefix)(const char* prefix, const char* path); + int32_t (*DeleteObjects)(const char* object_name[], int nobject); + int32_t (*GetObjectToFile)(const char* object_name, const char* fileName); +} STcs; + +extern STcs tcs; + +#ifdef __cplusplus +} +#endif + +#endif // _TD_TCS_INT_H_ diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index 5facffa4ac..c0933f68d0 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -14,39 +14,15 @@ */ #include "tcs.h" - #include "os.h" #include "taoserror.h" +#include "tcsInt.h" #include "tglobal.h" #include "az.h" #include "cos.h" -extern int8_t tsS3Ablob; - -typedef enum { - TOS_PROTO_NIL, - TOS_PROTO_S3, - TOS_PROTO_ABLOB, -} STosProto; - -typedef struct { - int32_t (*Begin)(); - void (*End)(); - int32_t (*CheckCfg)(); - - int32_t (*PutObjectFromFileOffset)(const char* file, const char* object_name, int64_t offset, int64_t size); - int32_t (*GetObjectBlock)(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock); - - void (*DeleteObjectsByPrefix)(const char* prefix); - - int32_t (*PutObjectFromFile2)(const char* file, const char* object, int8_t withcp); - int32_t (*GetObjectsByPrefix)(const char* prefix, const char* path); - int32_t (*DeleteObjects)(const char* object_name[], int nobject); - int32_t (*GetObjectToFile)(const char* object_name, const char* fileName); -} STcs; - -static STcs tcs; +STcs tcs; int32_t tcsInit() { int32_t code = 0; @@ -108,12 +84,6 @@ int32_t tcsCheckCfg() { TAOS_RETURN(code); } - code = s3Begin(); - if (code != 0) { - (void)fprintf(stderr, "failed to begin s3.\n"); - TAOS_RETURN(code); - } - code = tcs.CheckCfg(); if (code != 0) { (void)fprintf(stderr, "failed to check s3.\n"); @@ -134,15 +104,3 @@ int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, } void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); } - -int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { - return tcs.PutObjectFromFile2(file, object, withcp); -} - -int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } - -int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } - -int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { - return tcs.GetObjectToFile(object_name, fileName); -} diff --git a/source/libs/tcs/src/tcsStream.c b/source/libs/tcs/src/tcsStream.c new file mode 100644 index 0000000000..7cd4647ddd --- /dev/null +++ b/source/libs/tcs/src/tcsStream.c @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tcs.h" +#include "tcsInt.h" + +int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { + return tcs.PutObjectFromFile2(file, object, withcp); +} + +int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } + +int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } + +int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { + return tcs.GetObjectToFile(object_name, fileName); +} From b05784f524b446ffa282302185962c458838aff4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 16 Oct 2024 16:44:08 +0800 Subject: [PATCH 20/41] tcs: fix mac stb link issue --- source/libs/azure/src/az.cpp | 4 +- source/libs/stream/CMakeLists.txt | 6 +-- source/libs/tcs/CMakeLists.txt | 4 +- source/libs/tcs/src/tcs.c | 7 ++-- source/libs/tcs/test/tcsTest.cpp | 61 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 0af669c56a..80b5fb883e 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -49,13 +49,13 @@ static void azDumpCfgByEp(int8_t epIndex) { "%-24s %s\n" "%-24s %s\n" "%-24s %s\n" - // "%-24s %s\n" + "%-24s %s\n" "%-24s %s\n" "%-24s %s\n", "hostName", tsS3Hostname[epIndex], "bucketName", tsS3BucketName, "protocol", "https only", - //"uristyle", (uriStyleG[epIndex] == S3UriStyleVirtualHost ? "virtualhost" : "path"), + "uristyle", "path only", "accessKey", tsS3AccessKeyId[epIndex], "accessKeySecret", tsS3AccessKeySecret[epIndex]); // clang-format on diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index f08b16f836..53e26469f0 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -13,7 +13,7 @@ if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index + PRIVATE os util transport qcom executor wal index tcs ) target_include_directories( stream @@ -32,13 +32,13 @@ if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index + PRIVATE os util transport qcom executor wal index tcs ) else() target_link_libraries( stream PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index + PRIVATE os util transport qcom executor wal index tcs ) target_include_directories( stream diff --git a/source/libs/tcs/CMakeLists.txt b/source/libs/tcs/CMakeLists.txt index 1c914a18b9..e0de823c7a 100644 --- a/source/libs/tcs/CMakeLists.txt +++ b/source/libs/tcs/CMakeLists.txt @@ -1,6 +1,6 @@ -aux_source_directory(src TOS_SRC) +aux_source_directory(src TCS_SRC) -add_library(tcs STATIC ${TOS_SRC}) +add_library(tcs STATIC ${TCS_SRC}) target_include_directories( tcs PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs" diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index c0933f68d0..db02ca21fa 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -43,6 +43,7 @@ int32_t tcsInit() { tcs.GetObjectsByPrefix = s3GetObjectsByPrefix; tcs.DeleteObjects = s3DeleteObjects; tcs.GetObjectToFile = s3GetObjectToFile; + } else if (TOS_PROTO_ABLOB == proto) { tcs.Begin = azBegin; tcs.End = azEnd; @@ -74,19 +75,19 @@ int32_t tcsCheckCfg() { int32_t code = 0; if (!tsS3Enabled) { - (void)fprintf(stderr, "s3 not configured.\n"); + (void)fprintf(stderr, "tcs not configured.\n"); TAOS_RETURN(code); } code = tcsInit(); if (code != 0) { - (void)fprintf(stderr, "failed to initialize s3.\n"); + (void)fprintf(stderr, "failed to initialize tcs.\n"); TAOS_RETURN(code); } code = tcs.CheckCfg(); if (code != 0) { - (void)fprintf(stderr, "failed to check s3.\n"); + (void)fprintf(stderr, "failed to check tcs.\n"); TAOS_RETURN(code); } diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index 68b39bd710..e5593bfc18 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -2,6 +2,67 @@ #include #include #include + +#include "tcs.h" + +int32_t tcsInitEnv() { + int32_t code = 0; + + extern char tsS3Hostname[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; + extern char tsS3BucketName[TSDB_FQDN_LEN]; + + /* TCS parameter format + tsS3Hostname[0] = "endpoint/.blob.core.windows.net"; + tsS3AccessKeyId[0] = ""; + tsS3AccessKeySecret[0] = ""; + tsS3BucketName = ""; + */ + tsS3Enabled = true; + + return code; +} + +// TEST(TcsTest, DISABLE_InterfaceTest) { +TEST(TcsTest, InterfaceTest) { + int code = 0; + + if (!tsS3Enabled) { + (void)fprintf(stderr, "tcs not configured.\n"); + + return; + } + + code = tcsInit(); + GTEST_ASSERT_EQ(code, 0); + + code = tcsCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + /* + code = tcsPutObjectFromFileOffset(file, object_name, offset, size); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectBlock(object_name, offset, size, check, ppBlock); + GTEST_ASSERT_EQ(code, 0); + + tcsDeleteObjectsByPrefix(prefix); + // list object to check + + code = tcsPutObjectFromFile2(file, object, withcp); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectsByPrefix(prefix, path); + GTEST_ASSERT_EQ(code, 0); + code = tcsDeleteObjects(object_name, nobject); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectToFile(object_name, fileName); + GTEST_ASSERT_EQ(code, 0); + + // GTEST_ASSERT_NE(pEnv, nullptr); + */ + + tcsUninit(); +} + /* #include "walInt.h" const char* ranStr = "tvapq02tcp"; From d81fa80865acab41e92f865bdf7165e5ca2b5a60 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 16 Oct 2024 17:27:32 +0800 Subject: [PATCH 21/41] az: remove unused key newing --- .../mnode/impl/test/arbgroup/CMakeLists.txt | 2 +- source/libs/azure/src/az.cpp | 13 +--------- source/libs/stream/CMakeLists.txt | 13 +++++----- source/libs/stream/test/CMakeLists.txt | 2 +- source/libs/tcs/test/tcsTest.cpp | 24 +++++++++++++++---- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/mnode/impl/test/arbgroup/CMakeLists.txt b/source/dnode/mnode/impl/test/arbgroup/CMakeLists.txt index 44ac305498..0da36e1f67 100644 --- a/source/dnode/mnode/impl/test/arbgroup/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/arbgroup/CMakeLists.txt @@ -4,7 +4,7 @@ aux_source_directory(. MNODE_ARBGROUP_TEST_SRC) add_executable(arbgroupTest ${MNODE_ARBGROUP_TEST_SRC}) target_link_libraries( arbgroupTest - PRIVATE dnode nodes planner gtest qcom + PRIVATE dnode nodes planner gtest qcom tcs ) add_test( diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 80b5fb883e..b05f5be2ca 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -72,8 +72,6 @@ static int32_t azListBucket(char const *bucketname) { try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); std::string containerName = bucketname; @@ -216,8 +214,7 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - std::string accountURL = tsS3Hostname[0]; - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + std::string accountURL = tsS3Hostname[0]; accountURL = "https://" + accountURL; BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); @@ -278,8 +275,6 @@ int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t si try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - accountURL = "https://" + accountURL; BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); @@ -353,8 +348,6 @@ void azDeleteObjectsByPrefix(const char *prefix) { try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); std::string containerName = tsS3BucketName; @@ -414,8 +407,6 @@ int32_t azGetObjectToFile(const char *object_name, const char *fileName) { try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); std::string containerName = tsS3BucketName; @@ -450,8 +441,6 @@ int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); - StorageSharedKeyCredential *pSharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); - BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); std::string containerName = tsS3BucketName; diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 53e26469f0..27f5c46004 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -12,8 +12,8 @@ if(${BUILD_WITH_ROCKSDB}) if (${BUILD_CONTRIB}) target_link_libraries( stream - PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index tcs + PUBLIC rocksdb tdb tcs + PRIVATE os util transport qcom executor wal index ) target_include_directories( stream @@ -31,14 +31,14 @@ if(${BUILD_WITH_ROCKSDB}) ) target_link_libraries( stream - PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index tcs + PUBLIC rocksdb tdb tcs + PRIVATE os util transport qcom executor wal index ) else() target_link_libraries( stream - PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal index tcs + PUBLIC rocksdb tdb tcs + PRIVATE os util transport qcom executor wal index ) target_include_directories( stream @@ -59,4 +59,3 @@ endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) endif(${BUILD_TEST}) - diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index c472207b27..f2c985964d 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -101,4 +101,4 @@ IF(NOT TD_DARWIN) NAME backendTest COMMAND backendTest ) -ENDIF () \ No newline at end of file +ENDIF () diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index e5593bfc18..ed2cb59858 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -4,6 +4,7 @@ #include #include "tcs.h" +#include "tcsInt.h" int32_t tcsInitEnv() { int32_t code = 0; @@ -19,7 +20,21 @@ int32_t tcsInitEnv() { tsS3AccessKeySecret[0] = ""; tsS3BucketName = ""; */ + + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; + + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + + tstrncpy(tsTempDir, "/tmp/", PATH_MAX); + tsS3Enabled = true; + tsS3Ablob = true; return code; } @@ -28,11 +43,10 @@ int32_t tcsInitEnv() { TEST(TcsTest, InterfaceTest) { int code = 0; - if (!tsS3Enabled) { - (void)fprintf(stderr, "tcs not configured.\n"); - - return; - } + code = tcsInitEnv(); + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + GTEST_ASSERT_EQ(tsS3Ablob, 1); code = tcsInit(); GTEST_ASSERT_EQ(code, 0); From 5e14f65fb9071239b0c40fba5a731fcb54c70ebb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 16 Oct 2024 19:01:39 +0800 Subject: [PATCH 22/41] tcs/test: new blob test for checkcfg --- source/libs/tcs/test/CMakeLists.txt | 2 +- source/libs/tcs/test/tcsTest.cpp | 80 +++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/source/libs/tcs/test/CMakeLists.txt b/source/libs/tcs/test/CMakeLists.txt index 33fe75c589..f0a5fb97a6 100644 --- a/source/libs/tcs/test/CMakeLists.txt +++ b/source/libs/tcs/test/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories(tcsTest target_link_libraries(tcsTest tcs - gtest_main + common gtest_main ) enable_testing() add_test( diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index ed2cb59858..e86c9e02b3 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -6,9 +6,11 @@ #include "tcs.h" #include "tcsInt.h" -int32_t tcsInitEnv() { +int32_t tcsInitEnv(int8_t isBlob) { int32_t code = 0; + extern int8_t tsS3EpNum; + extern char tsS3Hostname[][TSDB_FQDN_LEN]; extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; @@ -21,20 +23,38 @@ int32_t tcsInitEnv() { tsS3BucketName = ""; */ - const char *hostname = "endpoint/.blob.core.windows.net"; - const char *accessKeyId = ""; - const char *accessKeySecret = ""; - const char *bucketName = ""; + tsS3Ablob = isBlob; + if (isBlob) { + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; - tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); - tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + + } else { + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; + + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + + // setup s3 env + tsS3EpNum = 1; + } tstrncpy(tsTempDir, "/tmp/", PATH_MAX); tsS3Enabled = true; - tsS3Ablob = true; + if (!tsS3Ablob) { + } return code; } @@ -43,7 +63,7 @@ int32_t tcsInitEnv() { TEST(TcsTest, InterfaceTest) { int code = 0; - code = tcsInitEnv(); + code = tcsInitEnv(true); GTEST_ASSERT_EQ(code, 0); GTEST_ASSERT_EQ(tsS3Enabled, 1); GTEST_ASSERT_EQ(tsS3Ablob, 1); @@ -77,6 +97,44 @@ TEST(TcsTest, InterfaceTest) { tcsUninit(); } +// TEST(TcsTest, DISABLE_InterfaceNonBlobTest) { +TEST(TcsTest, InterfaceNonBlobTest) { + int code = 0; + + code = tcsInitEnv(false); + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + GTEST_ASSERT_EQ(tsS3Ablob, 0); + + code = tcsInit(); + GTEST_ASSERT_EQ(code, 0); + + code = tcsCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + /* + code = tcsPutObjectFromFileOffset(file, object_name, offset, size); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectBlock(object_name, offset, size, check, ppBlock); + GTEST_ASSERT_EQ(code, 0); + + tcsDeleteObjectsByPrefix(prefix); + // list object to check + + code = tcsPutObjectFromFile2(file, object, withcp); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectsByPrefix(prefix, path); + GTEST_ASSERT_EQ(code, 0); + code = tcsDeleteObjects(object_name, nobject); + GTEST_ASSERT_EQ(code, 0); + code = tcsGetObjectToFile(object_name, fileName); + GTEST_ASSERT_EQ(code, 0); + + // GTEST_ASSERT_NE(pEnv, nullptr); + */ + + tcsUninit(); +} + /* #include "walInt.h" const char* ranStr = "tvapq02tcp"; From c377bb3514bb80f2f6807e18fe7f3d75fadb7ca5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Oct 2024 08:44:20 +0800 Subject: [PATCH 23/41] tcs/test: ut for linux only --- source/libs/tcs/test/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/tcs/test/CMakeLists.txt b/source/libs/tcs/test/CMakeLists.txt index f0a5fb97a6..1252736b33 100644 --- a/source/libs/tcs/test/CMakeLists.txt +++ b/source/libs/tcs/test/CMakeLists.txt @@ -1,3 +1,5 @@ +if (TD_LINUX) + aux_source_directory(. TCS_TEST_SRC) add_executable(tcsTest ${TCS_TEST_SRC}) @@ -9,10 +11,12 @@ target_include_directories(tcsTest target_link_libraries(tcsTest tcs - common gtest_main + gtest_main ) enable_testing() add_test( NAME tcs_test COMMAND tcsTest ) + +endif() From 1b4c2faf268d7fd5aa664640ff7b3d89b810ca3e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Oct 2024 09:43:18 +0800 Subject: [PATCH 24/41] tcs/stream: merge stream apis into tcs main module --- source/libs/tcs/inc/tcsInt.h | 2 -- source/libs/tcs/src/tcs.c | 14 ++++++++++++++ source/libs/tcs/src/tcsStream.c | 29 ----------------------------- 3 files changed, 14 insertions(+), 31 deletions(-) delete mode 100644 source/libs/tcs/src/tcsStream.c diff --git a/source/libs/tcs/inc/tcsInt.h b/source/libs/tcs/inc/tcsInt.h index b24a47aa98..a7e304b544 100644 --- a/source/libs/tcs/inc/tcsInt.h +++ b/source/libs/tcs/inc/tcsInt.h @@ -26,8 +26,6 @@ extern "C" { #endif -extern int8_t tsS3Ablob; - typedef enum { TOS_PROTO_NIL, TOS_PROTO_S3, diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index db02ca21fa..1716228d5c 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -24,6 +24,8 @@ STcs tcs; +extern int8_t tsS3Ablob; + int32_t tcsInit() { int32_t code = 0; @@ -105,3 +107,15 @@ int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, } void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); } + +int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { + return tcs.PutObjectFromFile2(file, object, withcp); +} + +int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } + +int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } + +int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { + return tcs.GetObjectToFile(object_name, fileName); +} diff --git a/source/libs/tcs/src/tcsStream.c b/source/libs/tcs/src/tcsStream.c deleted file mode 100644 index 7cd4647ddd..0000000000 --- a/source/libs/tcs/src/tcsStream.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tcs.h" -#include "tcsInt.h" - -int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { - return tcs.PutObjectFromFile2(file, object, withcp); -} - -int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } - -int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } - -int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { - return tcs.GetObjectToFile(object_name, fileName); -} From b6cfa2c7f11f68134a204ffd2bb736f43eeffebc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Oct 2024 09:48:19 +0800 Subject: [PATCH 25/41] tcs/test: extern tsS3Ablob from ut --- source/libs/tcs/inc/tcsInt.h | 2 ++ source/libs/tcs/src/tcs.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tcs/inc/tcsInt.h b/source/libs/tcs/inc/tcsInt.h index a7e304b544..b24a47aa98 100644 --- a/source/libs/tcs/inc/tcsInt.h +++ b/source/libs/tcs/inc/tcsInt.h @@ -26,6 +26,8 @@ extern "C" { #endif +extern int8_t tsS3Ablob; + typedef enum { TOS_PROTO_NIL, TOS_PROTO_S3, diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index 1716228d5c..73cb64c34d 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -24,8 +24,6 @@ STcs tcs; -extern int8_t tsS3Ablob; - int32_t tcsInit() { int32_t code = 0; From 12318f3a6df2aa0c357fc1ed5a5db4fd5516e4f5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Oct 2024 14:40:26 +0800 Subject: [PATCH 26/41] tcs/test: disable test cases for ci ut run --- source/libs/tcs/src/tcs.c | 14 -------------- source/libs/tcs/src/tcsStream.c | 31 +++++++++++++++++++++++++++++++ source/libs/tcs/test/tcsTest.cpp | 8 ++++---- 3 files changed, 35 insertions(+), 18 deletions(-) create mode 100644 source/libs/tcs/src/tcsStream.c diff --git a/source/libs/tcs/src/tcs.c b/source/libs/tcs/src/tcs.c index 73cb64c34d..a668eac60f 100644 --- a/source/libs/tcs/src/tcs.c +++ b/source/libs/tcs/src/tcs.c @@ -22,8 +22,6 @@ #include "az.h" #include "cos.h" -STcs tcs; - int32_t tcsInit() { int32_t code = 0; @@ -105,15 +103,3 @@ int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, } void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); } - -int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { - return tcs.PutObjectFromFile2(file, object, withcp); -} - -int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } - -int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } - -int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { - return tcs.GetObjectToFile(object_name, fileName); -} diff --git a/source/libs/tcs/src/tcsStream.c b/source/libs/tcs/src/tcsStream.c new file mode 100644 index 0000000000..f73bb028ba --- /dev/null +++ b/source/libs/tcs/src/tcsStream.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tcs.h" +#include "tcsInt.h" + +STcs tcs; + +int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) { + return tcs.PutObjectFromFile2(file, object, withcp); +} + +int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); } + +int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); } + +int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) { + return tcs.GetObjectToFile(object_name, fileName); +} diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index e86c9e02b3..0eb0b4d071 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -59,8 +59,8 @@ int32_t tcsInitEnv(int8_t isBlob) { return code; } -// TEST(TcsTest, DISABLE_InterfaceTest) { -TEST(TcsTest, InterfaceTest) { +TEST(TcsTest, DISABLE_InterfaceTest) { + // TEST(TcsTest, InterfaceTest) { int code = 0; code = tcsInitEnv(true); @@ -97,8 +97,8 @@ TEST(TcsTest, InterfaceTest) { tcsUninit(); } -// TEST(TcsTest, DISABLE_InterfaceNonBlobTest) { -TEST(TcsTest, InterfaceNonBlobTest) { +TEST(TcsTest, DISABLE_InterfaceNonBlobTest) { + // TEST(TcsTest, InterfaceNonBlobTest) { int code = 0; code = tcsInitEnv(false); From 69bc051b9a74dab38bb23f7947a5f8d1139f0921 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Oct 2024 15:44:31 +0800 Subject: [PATCH 27/41] tcs/test: cleanup unused cases --- source/libs/tcs/test/tcsTest.cpp | 396 +------------------------------ 1 file changed, 2 insertions(+), 394 deletions(-) diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index 0eb0b4d071..33566f6400 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -59,7 +59,7 @@ int32_t tcsInitEnv(int8_t isBlob) { return code; } -TEST(TcsTest, DISABLE_InterfaceTest) { +TEST(TcsTest, DISABLED_InterfaceTest) { // TEST(TcsTest, InterfaceTest) { int code = 0; @@ -97,7 +97,7 @@ TEST(TcsTest, DISABLE_InterfaceTest) { tcsUninit(); } -TEST(TcsTest, DISABLE_InterfaceNonBlobTest) { +TEST(TcsTest, DISABLED_InterfaceNonBlobTest) { // TEST(TcsTest, InterfaceNonBlobTest) { int code = 0; @@ -134,395 +134,3 @@ TEST(TcsTest, DISABLE_InterfaceNonBlobTest) { tcsUninit(); } - -/* -#include "walInt.h" -const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); -SWalSyncInfo syncMeta = {0}; -class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -class WalRetentionEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - void SetUp() override { - SWalCfg cfg; - cfg.rollPeriod = -1; - cfg.segSize = -1; - cfg.retentionPeriod = -1; - cfg.retentionSize = 0; - cfg.rollPeriod = 0; - cfg.vgId = 0; - cfg.level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, &cfg); - ASSERT(pWal != NULL); - } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; -TEST_F(WalCleanEnv, createNew) { - walRollFileInfo(pWal); - ASSERT(pWal->fileInfoSet != NULL); - ASSERT_EQ(pWal->fileInfoSet->size, 1); - SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); - ASSERT_EQ(pInfo->firstVer, 0); - ASSERT_EQ(pInfo->lastVer, -1); - ASSERT_EQ(pInfo->closeTs, -1); - ASSERT_EQ(pInfo->fileSize, 0); -} -TEST_F(WalCleanEnv, serialize) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - char* ss = NULL; - code = walMetaSerialize(pWal, &ss); - ASSERT(code == 0); - printf("%s\n", ss); - taosMemoryFree(ss); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} -TEST_F(WalCleanEnv, removeOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walSaveMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} -TEST_F(WalKeepEnv, readOldMeta) { - walResetEnv(); - int code; - syncMeta.isWeek = -1; - syncMeta.seqNum = UINT64_MAX; - syncMeta.term = UINT64_MAX; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - char* oldss = NULL; - code = walMetaSerialize(pWal, &oldss); - ASSERT(code == 0); - TearDown(); - SetUp(); - ASSERT_EQ(pWal->vers.firstVer, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - char* newss = NULL; - code = walMetaSerialize(pWal, &newss); - ASSERT(code == 0); - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for (int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } - taosMemoryFree(oldss); - taosMemoryFree(newss); -} -TEST_F(WalCleanEnv, write) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanEnv, rollback) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 2); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanEnv, rollbackMultiFile) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - if (i == 5) { - walBeginSnapshot(pWal, i, 0); - walEndSnapshot(pWal); - } - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 6); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walRollback(pWal, 5); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 6); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalCleanDeleteEnv, roll) { - int code; - int i; - for (i = 0; i < 100; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); - walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i - 1); - ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_NE(code, 0); - for (; i < 200; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - code = walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(code, 0); - code = walEndSnapshot(pWal); - ASSERT_EQ(code, 0); -} -TEST_F(WalKeepEnv, readHandleRead) { - walResetEnv(); - int code; - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -TEST_F(WalRetentionEnv, repairMeta1) { - walResetEnv(); - int code; - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - TearDown(); - // getchar(); - char buf[100]; - sprintf(buf, "%s/meta-ver%d", pathName, 0); - taosRemoveFile(buf); - sprintf(buf, "%s/meta-ver%d", pathName, 1); - taosRemoveFile(buf); - SetUp(); - // getchar(); - ASSERT_EQ(pWal->vers.lastVer, 99); - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - for (i = 100; i < 200; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 200; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -*/ From eeab3a8a8b1641bdb379c397122c2017e7b76f71 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Oct 2024 08:44:38 +0800 Subject: [PATCH 28/41] az/log: use az prefix log instead of u-prefixed ones --- include/util/tlog.h | 1 + source/common/src/tglobal.c | 22 +++++++++++------- source/libs/azure/inc/azInt.h | 42 +++++++++++++++++++++++++++++++++ source/libs/azure/src/az.cpp | 44 +++++++++++++++++------------------ source/util/src/tlog.c | 25 ++++++++++---------- 5 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 source/libs/azure/inc/azInt.h diff --git a/include/util/tlog.h b/include/util/tlog.h index e80e94de32..b8d096b7ce 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -57,6 +57,7 @@ extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; extern int32_t stDebugFlag; extern int32_t wDebugFlag; +extern int32_t azDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8254082561..f9fd0ed0ba 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -542,6 +542,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "azDebugFlag", azDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); @@ -1044,6 +1045,9 @@ static int32_t taosSetServerLogCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "wDebugFlag"); wDebugFlag = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "azDebugFlag"); + azDebugFlag = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "sDebugFlag"); sDebugFlag = pItem->i32; @@ -1729,7 +1733,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); + TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); @@ -1956,13 +1960,14 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { - {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, - {"wDebugFlag", &wDebugFlag}, {"sDebugFlag", &sDebugFlag}, {"tsdbDebugFlag", &tsdbDebugFlag}, - {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"udfDebugFlag", &udfDebugFlag}, - {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, - {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, - {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug}, + {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, + {"wDebugFlag", &wDebugFlag}, {"azDebugFlag", &azDebugFlag}, {"sDebugFlag", &sDebugFlag}, + {"tsdbDebugFlag", &tsdbDebugFlag}, {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, + {"udfDebugFlag", &udfDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, + {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, + {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"tqClientDebug", &tqClientDebug}, }; static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, @@ -2340,6 +2345,7 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&azDebugFlag, "azDebugFlag", flag, noNeedToSetVars); taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); diff --git a/source/libs/azure/inc/azInt.h b/source/libs/azure/inc/azInt.h new file mode 100644 index 0000000000..3538e925c7 --- /dev/null +++ b/source/libs/azure/inc/azInt.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AZ_INT_H_ +#define _TD_AZ_INT_H_ + +#include "os.h" +#include "tarray.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// clang-format off +#define azFatal(...) { if (azDebugFlag & DEBUG_FATAL) { taosPrintLog("AZR FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define azError(...) { if (azDebugFlag & DEBUG_ERROR) { taosPrintLog("AZR ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define azWarn(...) { if (azDebugFlag & DEBUG_WARN) { taosPrintLog("AZR WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define azInfo(...) { if (azDebugFlag & DEBUG_INFO) { taosPrintLog("AZR ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define azDebug(...) { if (azDebugFlag & DEBUG_DEBUG) { taosPrintLog("AZR ", DEBUG_DEBUG, azDebugFlag, __VA_ARGS__); }} +#define azTrace(...) { if (azDebugFlag & DEBUG_TRACE) { taosPrintLog("AZR ", DEBUG_TRACE, azDebugFlag, __VA_ARGS__); }} +// clang-format on + +#ifdef __cplusplus +} +#endif + +#endif // _TD_AZ_INT_H_ diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index b05f5be2ca..3da4b6e808 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #include "az.h" +#include "azInt.h" #include "os.h" #include "taoserror.h" @@ -88,9 +89,9 @@ static int32_t azListBucket(char const *bucketname) { } } } catch (const Azure::Core::RequestFailedException &e) { - uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), - e.ReasonPhrase.c_str()); - // uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); + azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + // azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); code = TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(code); @@ -105,7 +106,6 @@ int32_t azCheckCfg() { // for (; i < tsS3EpNum; i++) { (void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum); - // s3DumpCfgByEp(i); azDumpCfgByEp(0); // test put @@ -131,7 +131,7 @@ int32_t azCheckCfg() { TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (!fp) { (void)fprintf(stderr, "failed to open test file: %s.\n", path); - // uError("ERROR: %s Failed to open %s", __func__, path); + // azError("ERROR: %s Failed to open %s", __func__, path); TAOS_CHECK_GOTO(terrno, &lino, _next); } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { @@ -258,7 +258,7 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int std::cout << e.what() << std::endl; */ code = TAOS_SYSTEM_ERROR(EIO); - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); TAOS_RETURN(code); } @@ -296,16 +296,16 @@ int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t si auto res = blobClient.DownloadTo(buf, size, options); if (check && res.Value.ContentRange.Length.Value() != size) { code = TAOS_SYSTEM_ERROR(EIO); - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); TAOS_RETURN(code); } *ppBlock = buf; } catch (const Azure::Core::RequestFailedException &e) { - uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), - e.ReasonPhrase.c_str()); + azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); code = TAOS_SYSTEM_ERROR(EIO); - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); if (buf) { taosMemoryFree(buf); @@ -368,9 +368,9 @@ void azDeleteObjectsByPrefix(const char *prefix) { blobClient.Delete(); } } catch (const Azure::Core::RequestFailedException &e) { - uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), - e.ReasonPhrase.c_str()); - // uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); + azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); + // azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); } } @@ -379,19 +379,19 @@ int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp uint64_t contentLength = 0; if (taosStatFile(file, (int64_t *)&contentLength, NULL, NULL) < 0) { - uError("ERROR: %s Failed to stat file %s: ", __func__, file); + azError("ERROR: %s Failed to stat file %s: ", __func__, file); TAOS_RETURN(terrno); } code = azPutObjectFromFileOffset(file, object, 0, contentLength); if (code != 0) { - uError("ERROR: %s Failed to put file %s: ", __func__, file); + azError("ERROR: %s Failed to put file %s: ", __func__, file); TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: if (code) { - uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return 0; @@ -417,14 +417,14 @@ int32_t azGetObjectToFile(const char *object_name, const char *fileName) { auto res = blobClient.DownloadTo(fileName); if (res.Value.ContentRange.Length.Value() <= 0) { code = TAOS_SYSTEM_ERROR(EIO); - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); TAOS_RETURN(code); } } catch (const Azure::Core::RequestFailedException &e) { - uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), - e.ReasonPhrase.c_str()); + azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); code = TAOS_SYSTEM_ERROR(EIO); - uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); TAOS_RETURN(code); } @@ -470,8 +470,8 @@ int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { } } } catch (const Azure::Core::RequestFailedException &e) { - uError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), - e.ReasonPhrase.c_str()); + azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), + e.ReasonPhrase.c_str()); TAOS_RETURN(TSDB_CODE_FAILED); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 45c8a2f6c2..3ca148a625 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -26,7 +26,7 @@ #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) #define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 128) -#define LOG_FILE_DAY_LEN 64 +#define LOG_FILE_DAY_LEN 64 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB #define LOG_SLOW_BUF_SIZE (10 * 1024 * 1024) // 10MB @@ -113,6 +113,7 @@ int32_t rpcDebugFlag = 131; int32_t qDebugFlag = 131; int32_t stDebugFlag = 131; int32_t wDebugFlag = 131; +int32_t azDebugFlag = 131; int32_t sDebugFlag = 131; int32_t tsdbDebugFlag = 131; int32_t tdbDebugFlag = 131; @@ -151,7 +152,7 @@ static int32_t taosStartLog() { return 0; } -static void getDay(char* buf, int32_t bufSize){ +static void getDay(char *buf, int32_t bufSize) { time_t t = taosTime(NULL); struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) { @@ -172,7 +173,7 @@ static int64_t getTimestampToday() { return (int64_t)taosMktime(&tm); } -static void getFullPathName(char* fullName, const char* logName){ +static void getFullPathName(char *fullName, const char *logName) { if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; if (lastC == '\\' || lastC == '/') { @@ -225,7 +226,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { } TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); - if (tsc){ + if (tsc) { TAOS_CHECK_RETURN(taosInitSlowLog()); } TAOS_CHECK_RETURN(taosStartLog()); @@ -397,7 +398,7 @@ static int32_t taosOpenNewLogFile() { OldFileKeeper *oldFileKeeper = taosOpenNewFile(); if (!oldFileKeeper) { - TAOS_UNUSED(taosThreadMutexUnlock(&tsLogObj.logMutex)); + TAOS_UNUSED(taosThreadMutexUnlock(&tsLogObj.logMutex)); return terrno; } if (taosThreadCreate(&thread, &attr, taosThreadToCloseOldFile, oldFileKeeper) != 0) { @@ -433,7 +434,7 @@ static void taosOpenNewSlowLogFile() { char day[TD_TIME_STR_LEN] = {0}; getDay(day, sizeof(day)); TdFilePtr pFile = NULL; - char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { @@ -455,7 +456,7 @@ void taosResetLog() { if (tsLogObj.logHandle) { int32_t code = taosOpenNewLogFile(); - if(code != 0){ + if (code != 0) { uError("failed to open new log file, reason:%s", tstrerror(code)); } uInfo("=================================="); @@ -508,12 +509,12 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { } } -static void decideLogFileNameFlag(){ +static void decideLogFileNameFlag() { char name[PATH_MAX + 50] = "\0"; int32_t logstat0_mtime = 0; int32_t logstat1_mtime = 0; - bool log0Exist = false; - bool log1Exist = false; + bool log0Exist = false; + bool log1Exist = false; if (strlen(tsLogObj.logName) < PATH_MAX + 50 - 2) { strcpy(name, tsLogObj.logName); @@ -535,7 +536,7 @@ static void decideLogFileNameFlag(){ } } -static void processLogFileName(const char* logName , int32_t maxFileNum){ +static void processLogFileName(const char *logName, int32_t maxFileNum) { char fullName[PATH_MAX] = {0}; getFullPathName(fullName, logName); decideLogFileName(fullName, maxFileNum); @@ -872,7 +873,7 @@ static int32_t taosGetLogRemainSize(SLogBuff *pLogBuf, int32_t start, int32_t en return rSize >= 0 ? rSize : LOG_BUF_SIZE(pLogBuf) + rSize; } -static void taosWriteSlowLog(SLogBuff *pLogBuf){ +static void taosWriteSlowLog(SLogBuff *pLogBuf) { int32_t lock = atomic_val_compare_exchange_32(&pLogBuf->lock, 0, 1); if (lock == 1) return; taosWriteLog(pLogBuf); From 4cc65adc1fec57d4a61e07dd888f74f56b9d5ccd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Oct 2024 09:57:12 +0800 Subject: [PATCH 29/41] az/log: new debug flag azDebugFlag for az logging --- contrib/test/azure/CMakeLists.txt | 3 --- contrib/test/azure/main.cpp | 39 ++++++++++++------------------- tests/system-test/2-query/db.py | 2 +- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/contrib/test/azure/CMakeLists.txt b/contrib/test/azure/CMakeLists.txt index b3db1dffce..35c87312d0 100644 --- a/contrib/test/azure/CMakeLists.txt +++ b/contrib/test/azure/CMakeLists.txt @@ -6,9 +6,6 @@ add_executable ( main.cpp ) -# Link to Azure SDK -#target_link_libraries(application _azure_sdk) - find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) #find_library(XML2_LIBRARY xml2) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp index 5d52801329..09eadecd69 100644 --- a/contrib/test/azure/main.cpp +++ b/contrib/test/azure/main.cpp @@ -1,6 +1,3 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - /** * @file * @brief Application that consumes the Azure SDK for C++. @@ -16,36 +13,30 @@ using namespace Azure::Storage::Blobs; -int main(int argc, char* argv[]) -{ +int main(int argc, char* argv[]) { (void)argc; (void)argv; /**************** Container SDK client ************************/ /**************** Create container ************************/ - try - { - auto containerClient = BlobContainerClient::CreateFromConnectionString( - std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); - //containerClient.CreateIfNotExists(); + try { + auto containerClient = + BlobContainerClient::CreateFromConnectionString(std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); + // containerClient.CreateIfNotExists(); /**************** Container SDK client ************************/ /**************** list blobs (one page) ******************/ - //auto response = containerClient.ListBlobsSinglePage(); - //auto response = containerClient.ListBlobs(); - //auto blobListPage = response.Value; - //auto blobListPage = response.Blobs; - for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) - { - for (auto& blob : page.Blobs) - { - std::cout << blob.Name << std::endl; - } - } + // auto response = containerClient.ListBlobsSinglePage(); + // auto response = containerClient.ListBlobs(); + // auto blobListPage = response.Value; + // auto blobListPage = response.Blobs; + for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) { + for (auto& blob : page.Blobs) { + std::cout << blob.Name << std::endl; + } + } - } - catch (const std::exception& ex) - { + } catch (const std::exception& ex) { std::cout << ex.what(); return 1; } diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 588609e524..1964cea51f 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -57,7 +57,7 @@ class TDTestCase: tdSql.checkData(0, 2, 0) tdSql.query("show dnode 1 variables like '%debugFlag'") - tdSql.checkRows(23) + tdSql.checkRows(24) tdSql.query("show dnode 1 variables like '____debugFlag'") tdSql.checkRows(2) From 1d26d0fa071ddeca7a3bf39c8cf10ede58801652 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Oct 2024 15:12:53 +0800 Subject: [PATCH 30/41] az/test: test cases for az apis --- contrib/test/azure/CMakeLists.txt | 18 +- contrib/test/azure/main.cpp | 18 ++ source/libs/azure/src/az.cpp | 18 -- source/libs/azure/test/azTest.cpp | 498 ++++-------------------------- 4 files changed, 87 insertions(+), 465 deletions(-) diff --git a/contrib/test/azure/CMakeLists.txt b/contrib/test/azure/CMakeLists.txt index 35c87312d0..68571dce46 100644 --- a/contrib/test/azure/CMakeLists.txt +++ b/contrib/test/azure/CMakeLists.txt @@ -14,13 +14,13 @@ find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos #find_library(CoreFoundation_Library CoreFoundation) #find_library(SystemConfiguration_Library SystemConfiguration) - target_link_libraries( - azure-test - PRIVATE _azure_sdk - PRIVATE ${CURL_LIBRARY} - PRIVATE ${XML2_LIBRARY} - PRIVATE ${SSL_LIBRARY} - PRIVATE ${CRYPTO_LIBRARY} - PRIVATE dl - PRIVATE pthread +target_link_libraries( + azure-test + PRIVATE _azure_sdk + PRIVATE ${CURL_LIBRARY} + PRIVATE ${XML2_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE dl + PRIVATE pthread ) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp index 09eadecd69..badfef623b 100644 --- a/contrib/test/azure/main.cpp +++ b/contrib/test/azure/main.cpp @@ -22,6 +22,9 @@ int main(int argc, char* argv[]) { try { auto containerClient = BlobContainerClient::CreateFromConnectionString(std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); + + // Create the container if it does not exist + // std::cout << "Creating container: " << containerName << std::endl; // containerClient.CreateIfNotExists(); /**************** Container SDK client ************************/ @@ -30,6 +33,21 @@ int main(int argc, char* argv[]) { // auto response = containerClient.ListBlobs(); // auto blobListPage = response.Value; // auto blobListPage = response.Blobs; + //(void)_azUploadFrom(blobClient, file, offset, size); + /* + auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); + + // Create some data to upload into the blob. + std::vector data = {1, 2, 3, 4}; + Azure::Core::IO::MemoryBodyStream stream(data); + + Azure::Response response = blockBlobClient.Upload(stream); + + Models::UploadBlockBlobResult model = response.Value; + std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() + << std::endl; + */ + for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) { for (auto& blob : page.Blobs) { std::cout << blob.Name << std::endl; diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 3da4b6e808..a2411c883d 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -222,10 +222,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int std::string containerName = tsS3BucketName; auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); - // Create the container if it does not exist - // std::cout << "Creating container: " << containerName << std::endl; - // containerClient.CreateIfNotExists(); - std::string blobName = "blob.txt"; uint8_t blobContent[] = "Hello Azure!"; // Create the block blob client @@ -237,20 +233,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int // std::cout << "Uploading blob: " << blobName << std::endl; // blobClient.UploadFrom(blobContent, sizeof(blobContent)); blobClient.UploadFrom(file, offset, size); - //(void)_azUploadFrom(blobClient, file, offset, size); - /* - auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); - - // Create some data to upload into the blob. - std::vector data = {1, 2, 3, 4}; - Azure::Core::IO::MemoryBodyStream stream(data); - - Azure::Response response = blockBlobClient.Upload(stream); - - Models::UploadBlockBlobResult model = response.Value; - std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() - << std::endl; - */ } catch (const Azure::Core::RequestFailedException &e) { /* std::cout << "Status Code: " << static_cast(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index 9e963508f8..2780cddacc 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -2,456 +2,78 @@ #include #include #include -/* -#include "walInt.h" -const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); -SWalSyncInfo syncMeta = {0}; +#include "az.h" -class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } +extern int8_t tsS3Enabled; - static void TearDownTestCase() { walCleanUp(); } +int32_t azInitEnv() { + int32_t code = 0; - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } + extern int8_t tsS3EpNum; - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + extern char tsS3Hostname[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; + extern char tsS3BucketName[TSDB_FQDN_LEN]; - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; + /* TCS parameter format + tsS3Hostname[0] = "endpoint/.blob.core.windows.net"; + tsS3AccessKeyId[0] = ""; + tsS3AccessKeySecret[0] = ""; + tsS3BucketName = ""; + */ -class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; - static void TearDownTestCase() { walCleanUp(); } + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } + tstrncpy(tsTempDir, "/tmp/", PATH_MAX); - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + tsS3Enabled = true; - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - - static void TearDownTestCase() { walCleanUp(); } - - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -class WalRetentionEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - - static void TearDownTestCase() { walCleanUp(); } - - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - - void SetUp() override { - SWalCfg cfg; - cfg.rollPeriod = -1; - cfg.segSize = -1; - cfg.retentionPeriod = -1; - cfg.retentionSize = 0; - cfg.rollPeriod = 0; - cfg.vgId = 0; - cfg.level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, &cfg); - ASSERT(pWal != NULL); - } - - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -TEST_F(WalCleanEnv, createNew) { - walRollFileInfo(pWal); - ASSERT(pWal->fileInfoSet != NULL); - ASSERT_EQ(pWal->fileInfoSet->size, 1); - SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); - ASSERT_EQ(pInfo->firstVer, 0); - ASSERT_EQ(pInfo->lastVer, -1); - ASSERT_EQ(pInfo->closeTs, -1); - ASSERT_EQ(pInfo->fileSize, 0); + return code; } -TEST_F(WalCleanEnv, serialize) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); +// TEST(AzTest, DISABLED_InterfaceTest) { +TEST(AzTest, InterfaceTest) { + int code = 0; - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - char* ss = NULL; - code = walMetaSerialize(pWal, &ss); - ASSERT(code == 0); - printf("%s\n", ss); - taosMemoryFree(ss); - code = walSaveMeta(pWal); - ASSERT(code == 0); + code = azInitEnv(); + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + + code = azBegin(); + GTEST_ASSERT_EQ(code, 0); + + code = azCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + /* + code = azPutObjectFromFileOffset(file, object_name, offset, size); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectBlock(object_name, offset, size, check, ppBlock); + GTEST_ASSERT_EQ(code, 0); + + azDeleteObjectsByPrefix(prefix); + // list object to check + + code = azPutObjectFromFile2(file, object, withcp); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectsByPrefix(prefix, path); + GTEST_ASSERT_EQ(code, 0); + code = azDeleteObjects(object_name, nobject); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectToFile(object_name, fileName); + GTEST_ASSERT_EQ(code, 0); + + // GTEST_ASSERT_NE(pEnv, nullptr); + */ + + azEnd(); } - -TEST_F(WalCleanEnv, removeOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walSaveMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} - -TEST_F(WalKeepEnv, readOldMeta) { - walResetEnv(); - int code; - - syncMeta.isWeek = -1; - syncMeta.seqNum = UINT64_MAX; - syncMeta.term = UINT64_MAX; - - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - char* oldss = NULL; - code = walMetaSerialize(pWal, &oldss); - ASSERT(code == 0); - - TearDown(); - SetUp(); - - ASSERT_EQ(pWal->vers.firstVer, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - - char* newss = NULL; - code = walMetaSerialize(pWal, &newss); - ASSERT(code == 0); - - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for (int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } - taosMemoryFree(oldss); - taosMemoryFree(newss); -} - -TEST_F(WalCleanEnv, write) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanEnv, rollback) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 2); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanEnv, rollbackMultiFile) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - if (i == 5) { - walBeginSnapshot(pWal, i, 0); - walEndSnapshot(pWal); - } - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 6); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walRollback(pWal, 5); - ASSERT_NE(code, 0); - - ASSERT_EQ(pWal->vers.lastVer, 5); - - code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 6); - - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanDeleteEnv, roll) { - int code; - int i; - for (i = 0; i < 100; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - - walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); - walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i - 1); - ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - - code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_NE(code, 0); - - for (; i < 200; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - - code = walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(code, 0); - code = walEndSnapshot(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalKeepEnv, readHandleRead) { - walResetEnv(); - int code; - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} - -TEST_F(WalRetentionEnv, repairMeta1) { - walResetEnv(); - int code; - - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - - TearDown(); - - // getchar(); - char buf[100]; - sprintf(buf, "%s/meta-ver%d", pathName, 0); - taosRemoveFile(buf); - sprintf(buf, "%s/meta-ver%d", pathName, 1); - taosRemoveFile(buf); - SetUp(); - // getchar(); - - ASSERT_EQ(pWal->vers.lastVer, 99); - - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - - for (i = 100; i < 200; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 200; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -*/ From 5b750b350f03ce8c9f546dcead6ba8d205c6c266 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Oct 2024 16:29:21 +0800 Subject: [PATCH 31/41] az/checkCfg: remove duplicate void fprintf --- contrib/test/azure/main.cpp | 15 +++++++ source/libs/azure/src/az.cpp | 81 +++++++++++++++++------------------- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp index badfef623b..943546a5fb 100644 --- a/contrib/test/azure/main.cpp +++ b/contrib/test/azure/main.cpp @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + /** * @file * @brief Application that consumes the Azure SDK for C++. diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index a2411c883d..7e4b711fb4 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -44,21 +44,28 @@ int32_t azBegin() { return TSDB_CODE_SUCCESS; } void azEnd() {} +static void checkPrint(const char *fmt, ...) { + va_list arg_ptr; + va_start(arg_ptr, fmt); + (void)vfprintf(stderr, fmt, arg_ptr); + va_end(arg_ptr); +} + static void azDumpCfgByEp(int8_t epIndex) { // clang-format off - (void)fprintf(stdout, - "%-24s %s\n" - "%-24s %s\n" - "%-24s %s\n" - "%-24s %s\n" - "%-24s %s\n" - "%-24s %s\n", - "hostName", tsS3Hostname[epIndex], - "bucketName", tsS3BucketName, - "protocol", "https only", - "uristyle", "path only", - "accessKey", tsS3AccessKeyId[epIndex], - "accessKeySecret", tsS3AccessKeySecret[epIndex]); + checkPrint( + "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n" + "%-24s %s\n", + "hostName", tsS3Hostname[epIndex], + "bucketName", tsS3BucketName, + "protocol", "https only", + "uristyle", "path only", + "accessKey", tsS3AccessKeyId[epIndex], + "accessKeySecret", tsS3AccessKeySecret[epIndex]); // clang-format on } @@ -81,17 +88,15 @@ static int32_t azListBucket(char const *bucketname) { Azure::Storage::Blobs::ListBlobsOptions options; options.Prefix = "s3"; - (void)fprintf(stderr, "objects:\n"); - // std::set listBlobs; + checkPrint("objects:\n"); for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { for (const auto &blob : pageResult.Blobs) { - (void)fprintf(stderr, "%s\n", blob.Name.c_str()); + checkPrint("%s\n", blob.Name.c_str()); } } } catch (const Azure::Core::RequestFailedException &e) { azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), e.ReasonPhrase.c_str()); - // azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); code = TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(code); @@ -102,10 +107,7 @@ static int32_t azListBucket(char const *bucketname) { int32_t azCheckCfg() { int32_t code = 0, lino = 0; - int8_t i = 0; - // for (; i < tsS3EpNum; i++) { - (void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum); azDumpCfgByEp(0); // test put @@ -130,53 +132,51 @@ int32_t azCheckCfg() { TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (!fp) { - (void)fprintf(stderr, "failed to open test file: %s.\n", path); - // azError("ERROR: %s Failed to open %s", __func__, path); + checkPrint("failed to open test file: %s.\n", path); TAOS_CHECK_GOTO(terrno, &lino, _next); } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { - (void)fprintf(stderr, "failed to write test file: %s.\n", path); + checkPrint("failed to write test file: %s.\n", path); TAOS_CHECK_GOTO(terrno, &lino, _next); } if (taosFsyncFile(fp) < 0) { - (void)fprintf(stderr, "failed to fsync test file: %s.\n", path); + checkPrint("failed to fsync test file: %s.\n", path); TAOS_CHECK_GOTO(terrno, &lino, _next); } (void)taosCloseFile(&fp); - (void)fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata); + checkPrint("\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata); code = azPutObjectFromFileOffset(path, objectname[0], 0, 16); if (code != 0) { - (void)fprintf(stderr, "put object %s : failed.\n", objectname[0]); + checkPrint("put object %s : failed.\n", objectname[0]); TAOS_CHECK_GOTO(code, &lino, _next); } - (void)fprintf(stderr, "put object %s: success.\n\n", objectname[0]); + checkPrint("put object %s: success.\n\n", objectname[0]); // list buckets - (void)fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName); - // code = s3ListBucketByEp(tsS3BucketName, i); + checkPrint("start to list bucket %s by prefix s3.\n", tsS3BucketName); code = azListBucket(tsS3BucketName); if (code != 0) { - (void)fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); + checkPrint("listing bucket %s : failed.\n", tsS3BucketName); TAOS_CHECK_GOTO(code, &lino, _next); } - (void)fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); + checkPrint("listing bucket %s: success.\n\n", tsS3BucketName); // test range get - (void)fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len); + checkPrint("start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len); code = azGetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); if (code != 0) { - (void)fprintf(stderr, "get object %s : failed.\n", objectname[0]); + checkPrint("get object %s : failed.\n", objectname[0]); TAOS_CHECK_GOTO(code, &lino, _next); } (void)memcpy(buf, pBlock, c_len); taosMemoryFree(pBlock); - (void)fprintf(stderr, "object content: %s\n", buf); - (void)fprintf(stderr, "get object %s: success.\n\n", objectname[0]); + checkPrint("object content: %s\n", buf); + checkPrint("get object %s: success.\n\n", objectname[0]); // delete test object - (void)fprintf(stderr, "start to delete object: %s.\n", objectname[0]); + checkPrint("start to delete object: %s.\n", objectname[0]); // code = azDeleteObjectsByPrefix(objectname[0]); azDeleteObjectsByPrefix(objectname[0]); /* @@ -185,7 +185,7 @@ int32_t azCheckCfg() { TAOS_CHECK_GOTO(code, &lino, _next); } */ - (void)fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); + checkPrint("delete object %s: success.\n\n", objectname[0]); _next: if (fp) { @@ -193,13 +193,10 @@ _next: } if (TSDB_CODE_SUCCESS != code) { - (void)fprintf(stderr, "s3 check failed, code: %d, line: %d, index: %d.\n", code, lino, i); + checkPrint("s3 check failed, code: %d, line: %d.\n", code, lino); } - (void)fprintf(stdout, "=================================================================\n"); - //} - - // azEnd(); + checkPrint("=================================================================\n"); TAOS_RETURN(code); } From af95725c763aaa096202d767c49e008bda9999b0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 18 Oct 2024 16:33:27 +0800 Subject: [PATCH 32/41] az/test: ut for linux only --- source/libs/azure/test/CMakeLists.txt | 4 ++++ source/libs/azure/test/azTest.cpp | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/azure/test/CMakeLists.txt b/source/libs/azure/test/CMakeLists.txt index 01570df730..ea91dbd2fc 100644 --- a/source/libs/azure/test/CMakeLists.txt +++ b/source/libs/azure/test/CMakeLists.txt @@ -1,3 +1,5 @@ +if (TD_LINUX) + aux_source_directory(. AZ_TEST_SRC) add_executable(azTest ${AZ_TEST_SRC}) @@ -16,3 +18,5 @@ add_test( NAME az_test COMMAND azTest ) + +endif(TD_LINUX) diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index 2780cddacc..8d428fbb69 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -41,8 +41,8 @@ int32_t azInitEnv() { return code; } -// TEST(AzTest, DISABLED_InterfaceTest) { -TEST(AzTest, InterfaceTest) { +TEST(AzTest, DISABLED_InterfaceTest) { + // TEST(AzTest, InterfaceTest) { int code = 0; code = azInitEnv(); From 35c200e6ecb06af506ec1d398487a135ad5db55e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 21 Oct 2024 17:14:55 +0800 Subject: [PATCH 33/41] tcs/test: ut for s3 based tcs --- source/libs/azure/src/az.cpp | 2 +- source/libs/tcs/test/tcsTest.cpp | 242 ++++++++++++++++++++++++++----- 2 files changed, 209 insertions(+), 35 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 7e4b711fb4..1422705011 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -444,7 +444,7 @@ int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { } else { (void)snprintf(fileName, PATH_MAX, "%s%s", path, tmp); } - if (!azGetObjectToFile(blobName.c_str(), fileName)) { + if (azGetObjectToFile(blobName.c_str(), fileName)) { TAOS_RETURN(TSDB_CODE_FAILED); } } diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index 33566f6400..d07513c644 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -9,8 +9,6 @@ int32_t tcsInitEnv(int8_t isBlob) { int32_t code = 0; - extern int8_t tsS3EpNum; - extern char tsS3Hostname[][TSDB_FQDN_LEN]; extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; @@ -36,10 +34,22 @@ int32_t tcsInitEnv(int8_t isBlob) { tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); } else { + /* const char *hostname = "endpoint/.blob.core.windows.net"; const char *accessKeyId = ""; const char *accessKeySecret = ""; const char *bucketName = ""; + */ + + // const char *hostname = "http://192.168.1.52:9000"; + // const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel"; + // const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX"; + // const char *bucketName = "test-bucket"; + const char *hostname = "192.168.1.52:9000"; + const char *accessKeyId = "fGPPyYjzytw05nw44ViA"; + const char *accessKeySecret = "vK1VcwxgSOykicx6hk8fL1x15uEtyDSFU3w4hTaZ"; + + const char *bucketName = "ci-bucket19"; tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); @@ -47,7 +57,11 @@ int32_t tcsInitEnv(int8_t isBlob) { tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); // setup s3 env + extern int8_t tsS3EpNum; + extern int8_t tsS3Https[TSDB_MAX_EP_NUM]; + tsS3EpNum = 1; + tsS3Https[0] = false; } tstrncpy(tsTempDir, "/tmp/", PATH_MAX); @@ -61,7 +75,9 @@ int32_t tcsInitEnv(int8_t isBlob) { TEST(TcsTest, DISABLED_InterfaceTest) { // TEST(TcsTest, InterfaceTest) { - int code = 0; + int code = 0; + bool check = false; + bool withcp = false; code = tcsInitEnv(true); GTEST_ASSERT_EQ(code, 0); @@ -73,33 +89,113 @@ TEST(TcsTest, DISABLED_InterfaceTest) { code = tcsCheckCfg(); GTEST_ASSERT_EQ(code, 0); - /* - code = tcsPutObjectFromFileOffset(file, object_name, offset, size); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectBlock(object_name, offset, size, check, ppBlock); + + const int size = 4096; + char data[size] = {0}; + for (int i = 0; i < size / 2; ++i) { + data[i * 2 + 1] = 1; + } + + const char object_name[] = "tcsut.bin"; + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); GTEST_ASSERT_EQ(code, 0); - tcsDeleteObjectsByPrefix(prefix); + code = tcsPutObjectFromFileOffset(path, object_name, 0, size); + GTEST_ASSERT_EQ(code, 0); + + uint8_t *pBlock = NULL; + code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(pBlock[i * 2], 0); + GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1); + } + + taosMemoryFree(pBlock); + + code = tcsGetObjectToFile(object_name, path_download); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + tcsDeleteObjectsByPrefix(object_name); // list object to check - code = tcsPutObjectFromFile2(file, object, withcp); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectsByPrefix(prefix, path); - GTEST_ASSERT_EQ(code, 0); - code = tcsDeleteObjects(object_name, nobject); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectToFile(object_name, fileName); + code = tcsPutObjectFromFile2(path, object_name, withcp); GTEST_ASSERT_EQ(code, 0); - // GTEST_ASSERT_NE(pEnv, nullptr); - */ + code = tcsGetObjectsByPrefix(object_name, tsTempDir); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + const char *object_name_arr[] = {object_name}; + code = tcsDeleteObjects(object_name_arr, 1); + GTEST_ASSERT_EQ(code, 0); tcsUninit(); } -TEST(TcsTest, DISABLED_InterfaceNonBlobTest) { - // TEST(TcsTest, InterfaceNonBlobTest) { - int code = 0; +// TEST(TcsTest, DISABLED_InterfaceNonBlobTest) { +TEST(TcsTest, InterfaceNonBlobTest) { + int code = 0; + bool check = false; + bool withcp = false; code = tcsInitEnv(false); GTEST_ASSERT_EQ(code, 0); @@ -111,26 +207,104 @@ TEST(TcsTest, DISABLED_InterfaceNonBlobTest) { code = tcsCheckCfg(); GTEST_ASSERT_EQ(code, 0); - /* - code = tcsPutObjectFromFileOffset(file, object_name, offset, size); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectBlock(object_name, offset, size, check, ppBlock); + + const int size = 4096; + char data[size] = {0}; + for (int i = 0; i < size / 2; ++i) { + data[i * 2 + 1] = 1; + } + + const char object_name[] = "tcsut.bin"; + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); GTEST_ASSERT_EQ(code, 0); - tcsDeleteObjectsByPrefix(prefix); + code = tcsPutObjectFromFileOffset(path, object_name, 0, size); + GTEST_ASSERT_EQ(code, 0); + + uint8_t *pBlock = NULL; + code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(pBlock[i * 2], 0); + GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1); + } + + taosMemoryFree(pBlock); + + code = tcsGetObjectToFile(object_name, path_download); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + tcsDeleteObjectsByPrefix(object_name); // list object to check - code = tcsPutObjectFromFile2(file, object, withcp); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectsByPrefix(prefix, path); - GTEST_ASSERT_EQ(code, 0); - code = tcsDeleteObjects(object_name, nobject); - GTEST_ASSERT_EQ(code, 0); - code = tcsGetObjectToFile(object_name, fileName); + code = tcsPutObjectFromFile2(path, object_name, withcp); GTEST_ASSERT_EQ(code, 0); - // GTEST_ASSERT_NE(pEnv, nullptr); - */ + code = tcsGetObjectsByPrefix(object_name, tsTempDir); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + const char *object_name_arr[] = {object_name}; + code = tcsDeleteObjects(object_name_arr, 1); + GTEST_ASSERT_EQ(code, 0); tcsUninit(); } From aa14186f18b4f68a72c8a7e25d7173cfa65309d1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 09:52:24 +0800 Subject: [PATCH 34/41] test/tcs, _azure_sdk: use env vars for blob ut --- contrib/test/azure/main.cpp | 89 ++++++++++++++++++++------------ source/libs/tcs/test/tcsTest.cpp | 46 +++++++++++++---- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp index 943546a5fb..78ecc8b9f5 100644 --- a/contrib/test/azure/main.cpp +++ b/contrib/test/azure/main.cpp @@ -13,42 +13,67 @@ * along with this program. If not, see . */ -/** - * @file - * @brief Application that consumes the Azure SDK for C++. - * - * @remark Set environment variable `STORAGE_CONNECTION_STRING` before running the application. - * - */ - -#include - -#include #include +// Include the necessary SDK headers +#include +#include + +// Add appropriate using namespace directives +using namespace Azure::Storage; using namespace Azure::Storage::Blobs; -int main(int argc, char* argv[]) { - (void)argc; - (void)argv; +// Secrets should be stored & retrieved from secure locations such as Azure::KeyVault. For +// convenience and brevity of samples, the secrets are retrieved from environment variables. + +std::string GetEndpointUrl() { + // return std::getenv("AZURE_STORAGE_ACCOUNT_URL"); + std::string accountId = getenv("ablob_account_id"); + if (accountId.empty()) { + return accountId; + } + + return accountId + ".blob.core.windows.net"; +} + +std::string GetAccountName() { + // return std::getenv("AZURE_STORAGE_ACCOUNT_NAME"); + return getenv("ablob_account_id"); +} + +std::string GetAccountKey() { + // return std::getenv("AZURE_STORAGE_ACCOUNT_KEY"); + + return getenv("ablob_account_secret"); +} + +int main() { + std::string endpointUrl = GetEndpointUrl(); + std::string accountName = GetAccountName(); + std::string accountKey = GetAccountKey(); - /**************** Container SDK client ************************/ - /**************** Create container ************************/ try { - auto containerClient = - BlobContainerClient::CreateFromConnectionString(std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); + auto sharedKeyCredential = std::make_shared(accountName, accountKey); + + std::string accountURL = "https://fd2d01cd892f844eeaa2273.blob.core.windows.net"; + BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential); + + std::string containerName = "myblobcontainer"; + // auto containerClient = blobServiceClient.GetBlobContainerClient("myblobcontainer"); + auto containerClient = blobServiceClient.GetBlobContainerClient("td-test"); // Create the container if it does not exist - // std::cout << "Creating container: " << containerName << std::endl; + std::cout << "Creating container: " << containerName << std::endl; // containerClient.CreateIfNotExists(); - /**************** Container SDK client ************************/ - /**************** list blobs (one page) ******************/ - // auto response = containerClient.ListBlobsSinglePage(); - // auto response = containerClient.ListBlobs(); - // auto blobListPage = response.Value; - // auto blobListPage = response.Blobs; - //(void)_azUploadFrom(blobClient, file, offset, size); + std::string blobName = "blob.txt"; + uint8_t blobContent[] = "Hello Azure!"; + // Create the block blob client + BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); + + // Upload the blob + std::cout << "Uploading blob: " << blobName << std::endl; + blobClient.UploadFrom(blobContent, sizeof(blobContent)); /* auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); @@ -62,15 +87,11 @@ int main(int argc, char* argv[]) { std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() << std::endl; */ + } catch (const Azure::Core::RequestFailedException& e) { + std::cout << "Status Code: " << static_cast(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase + << std::endl; + std::cout << e.what() << std::endl; - for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) { - for (auto& blob : page.Blobs) { - std::cout << blob.Name << std::endl; - } - } - - } catch (const std::exception& ex) { - std::cout << ex.what(); return 1; } diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index d07513c644..5e17d09fc7 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -1,4 +1,5 @@ #include + #include #include #include @@ -15,7 +16,7 @@ int32_t tcsInitEnv(int8_t isBlob) { extern char tsS3BucketName[TSDB_FQDN_LEN]; /* TCS parameter format - tsS3Hostname[0] = "endpoint/.blob.core.windows.net"; + tsS3Hostname[0] = "/.blob.core.windows.net"; tsS3AccessKeyId[0] = ""; tsS3AccessKeySecret[0] = ""; tsS3BucketName = ""; @@ -23,16 +24,38 @@ int32_t tcsInitEnv(int8_t isBlob) { tsS3Ablob = isBlob; if (isBlob) { - const char *hostname = "endpoint/.blob.core.windows.net"; + const char *hostname = "/.blob.core.windows.net"; const char *accessKeyId = ""; const char *accessKeySecret = ""; const char *bucketName = ""; - tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); - tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + if (hostname[0] != '<') { + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + } else { + const char *accountId = getenv("ablob_account_id"); + if (!accountId) { + return -1; + } + const char *accountSecret = getenv("ablob_account_secret"); + if (!accountSecret) { + return -1; + } + + const char *containerName = getenv("ablob_container"); + if (!containerName) { + return -1; + } + + TAOS_STRCPY(&tsS3Hostname[0][0], accountId); + TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net"); + TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId); + TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret); + TAOS_STRCPY(tsS3BucketName, containerName); + } } else { /* const char *hostname = "endpoint/.blob.core.windows.net"; @@ -67,19 +90,22 @@ int32_t tcsInitEnv(int8_t isBlob) { tstrncpy(tsTempDir, "/tmp/", PATH_MAX); tsS3Enabled = true; - if (!tsS3Ablob) { - } return code; } -TEST(TcsTest, DISABLED_InterfaceTest) { - // TEST(TcsTest, InterfaceTest) { +// TEST(TcsTest, DISABLED_InterfaceTest) { +TEST(TcsTest, InterfaceTest) { int code = 0; bool check = false; bool withcp = false; code = tcsInitEnv(true); + if (code) { + std::cout << "ablob env init failed with: " << code << std::endl; + return; + } + GTEST_ASSERT_EQ(code, 0); GTEST_ASSERT_EQ(tsS3Enabled, 1); GTEST_ASSERT_EQ(tsS3Ablob, 1); From bc247650c5fe42e29882c3f7fa96aef9ccd2e018 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 12:35:32 +0800 Subject: [PATCH 35/41] test/az: use env vars for az testing cases --- source/libs/azure/src/az.cpp | 9 +- source/libs/azure/test/azTest.cpp | 161 ++++++++++++++++++++++++++---- source/libs/tcs/test/tcsTest.cpp | 15 +++ 3 files changed, 157 insertions(+), 28 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 1422705011..b3b5c7704a 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -204,9 +204,9 @@ _next: int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { int32_t code = 0; - std::string endpointUrl = tsS3Hostname[0]; // GetEndpointUrl(); - std::string accountName = tsS3AccessKeyId[0]; // GetAccountName(); - std::string accountKey = tsS3AccessKeySecret[0]; // GetAccountKey(); + std::string endpointUrl = tsS3Hostname[0]; + std::string accountName = tsS3AccessKeyId[0]; + std::string accountKey = tsS3AccessKeySecret[0]; try { auto sharedKeyCredential = std::make_shared(accountName, accountKey); @@ -226,9 +226,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int // TDBlockBlobClient blobClient(containerClient.GetBlobClient(blobName)); TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); - // Upload the blob - // std::cout << "Uploading blob: " << blobName << std::endl; - // blobClient.UploadFrom(blobContent, sizeof(blobContent)); blobClient.UploadFrom(file, offset, size); } catch (const Azure::Core::RequestFailedException &e) { /* diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index 8d428fbb69..bb15bf11ec 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include @@ -24,15 +39,38 @@ int32_t azInitEnv() { tsS3BucketName = ""; */ - const char *hostname = "endpoint/.blob.core.windows.net"; + const char *hostname = "/.blob.core.windows.net"; const char *accessKeyId = ""; const char *accessKeySecret = ""; const char *bucketName = ""; - tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); - tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); - tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + if (hostname[0] != '<') { + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + } else { + const char *accountId = getenv("ablob_account_id"); + if (!accountId) { + return -1; + } + + const char *accountSecret = getenv("ablob_account_secret"); + if (!accountSecret) { + return -1; + } + + const char *containerName = getenv("ablob_container"); + if (!containerName) { + return -1; + } + + TAOS_STRCPY(&tsS3Hostname[0][0], accountId); + TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net"); + TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId); + TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret); + TAOS_STRCPY(tsS3BucketName, containerName); + } tstrncpy(tsTempDir, "/tmp/", PATH_MAX); @@ -41,9 +79,11 @@ int32_t azInitEnv() { return code; } -TEST(AzTest, DISABLED_InterfaceTest) { - // TEST(AzTest, InterfaceTest) { - int code = 0; +// TEST(AzTest, DISABLED_InterfaceTest) { +TEST(AzTest, InterfaceTest) { + int code = 0; + bool check = false; + bool withcp = false; code = azInitEnv(); GTEST_ASSERT_EQ(code, 0); @@ -54,26 +94,103 @@ TEST(AzTest, DISABLED_InterfaceTest) { code = azCheckCfg(); GTEST_ASSERT_EQ(code, 0); - /* - code = azPutObjectFromFileOffset(file, object_name, offset, size); - GTEST_ASSERT_EQ(code, 0); - code = azGetObjectBlock(object_name, offset, size, check, ppBlock); + const int size = 4096; + char data[size] = {0}; + for (int i = 0; i < size / 2; ++i) { + data[i * 2 + 1] = 1; + } + + const char object_name[] = "azut.bin"; + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); GTEST_ASSERT_EQ(code, 0); - azDeleteObjectsByPrefix(prefix); + code = azPutObjectFromFileOffset(path, object_name, 0, size); + GTEST_ASSERT_EQ(code, 0); + + uint8_t *pBlock = NULL; + code = azGetObjectBlock(object_name, 0, size, check, &pBlock); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(pBlock[i * 2], 0); + GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1); + } + + taosMemoryFree(pBlock); + + code = azGetObjectToFile(object_name, path_download); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + azDeleteObjectsByPrefix(object_name); // list object to check - code = azPutObjectFromFile2(file, object, withcp); - GTEST_ASSERT_EQ(code, 0); - code = azGetObjectsByPrefix(prefix, path); - GTEST_ASSERT_EQ(code, 0); - code = azDeleteObjects(object_name, nobject); - GTEST_ASSERT_EQ(code, 0); - code = azGetObjectToFile(object_name, fileName); + code = azPutObjectFromFile2(path, object_name, withcp); GTEST_ASSERT_EQ(code, 0); - // GTEST_ASSERT_NE(pEnv, nullptr); - */ + code = azGetObjectsByPrefix(object_name, tsTempDir); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + const char *object_name_arr[] = {object_name}; + code = azDeleteObjects(object_name_arr, 1); + GTEST_ASSERT_EQ(code, 0); azEnd(); } diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index 5e17d09fc7..4b5afc5b85 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include From 1153a5222de848ca9d2ab682d682fc2c49bdd350 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 12:53:16 +0800 Subject: [PATCH 36/41] az/put from file offset: log error with status code and reason phrase --- source/libs/azure/src/az.cpp | 1 + source/libs/azure/test/azTest.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index b3b5c7704a..26a9e543da 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -233,6 +233,7 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int << std::endl; std::cout << e.what() << std::endl; */ + azError("%s: Status Code: %d, Reason Phrase: %s", __func__, static_cast(e.StatusCode), e.ReasonPhrase.c_str()); code = TAOS_SYSTEM_ERROR(EIO); azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); TAOS_RETURN(code); diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index bb15bf11ec..c1be835150 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -33,7 +33,7 @@ int32_t azInitEnv() { extern char tsS3BucketName[TSDB_FQDN_LEN]; /* TCS parameter format - tsS3Hostname[0] = "endpoint/.blob.core.windows.net"; + tsS3Hostname[0] = "/.blob.core.windows.net"; tsS3AccessKeyId[0] = ""; tsS3AccessKeySecret[0] = ""; tsS3BucketName = ""; From ef730b08349158ae2343676638ef9bdf693059bf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 13:04:38 +0800 Subject: [PATCH 37/41] az/put: remove cpp logs --- source/libs/azure/src/az.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 26a9e543da..d4f93b9af4 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -228,14 +228,11 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int blobClient.UploadFrom(file, offset, size); } catch (const Azure::Core::RequestFailedException &e) { - /* - std::cout << "Status Code: " << static_cast(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase - << std::endl; - std::cout << e.what() << std::endl; - */ azError("%s: Status Code: %d, Reason Phrase: %s", __func__, static_cast(e.StatusCode), e.ReasonPhrase.c_str()); + code = TAOS_SYSTEM_ERROR(EIO); azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + TAOS_RETURN(code); } From 9c7eaa1633920ef1f1c2594c6cd5690f75b9c9bd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 13:17:14 +0800 Subject: [PATCH 38/41] az/header: move header file to inc directory --- source/libs/azure/src/avro_parser.cpp | 531 ------------------ source/libs/azure/src/avro_parser.hpp | 198 ------- source/libs/azure/src/td_avro_parser.cpp | 2 +- .../libs/azure/src/td_block_blob_client.cpp | 2 +- 4 files changed, 2 insertions(+), 731 deletions(-) delete mode 100644 source/libs/azure/src/avro_parser.cpp delete mode 100644 source/libs/azure/src/avro_parser.hpp diff --git a/source/libs/azure/src/avro_parser.cpp b/source/libs/azure/src/avro_parser.cpp deleted file mode 100644 index 485980e007..0000000000 --- a/source/libs/azure/src/avro_parser.cpp +++ /dev/null @@ -1,531 +0,0 @@ -#if defined(USE_S3) -#include "avro_parser.hpp" - -#include -#include - -#include -#include - -namespace Azure { -namespace Storage { -namespace Blobs { -namespace _detail { - -namespace { -int64_t parseInt(AvroStreamReader::ReaderPos& data) { - uint64_t r = 0; - int nb = 0; - while (true) { - uint8_t c = (*data.BufferPtr)[data.Offset++]; - r = r | ((static_cast(c) & 0x7f) << (nb * 7)); - if (c & 0x80) { - ++nb; - continue; - } - break; - } - return static_cast(r >> 1) ^ -static_cast(r & 0x01); -} - -AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) { - const static std::map BuiltinNameSchemaMap = { - {"string", AvroSchema::StringSchema}, {"bytes", AvroSchema::BytesSchema}, {"int", AvroSchema::IntSchema}, - {"long", AvroSchema::LongSchema}, {"float", AvroSchema::FloatSchema}, {"double", AvroSchema::DoubleSchema}, - {"boolean", AvroSchema::BoolSchema}, {"null", AvroSchema::NullSchema}, {"string", AvroSchema::StringSchema}, - }; - std::map nameSchemaMap = BuiltinNameSchemaMap; - - std::function parseSchemaFromJsonObject; - parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema { - if (obj.is_string()) { - auto typeName = obj.get(); - return nameSchemaMap.find(typeName)->second; - } else if (obj.is_array()) { - std::vector unionSchemas; - for (const auto& s : obj) { - unionSchemas.push_back(parseSchemaFromJsonObject(s)); - } - return AvroSchema::UnionSchema(std::move(unionSchemas)); - } else if (obj.is_object()) { - if (obj.count("namespace") != 0) { - throw std::runtime_error("Namespace isn't supported yet in Avro schema."); - } - if (obj.count("aliases") != 0) { - throw std::runtime_error("Alias isn't supported yet in Avro schema."); - } - auto typeName = obj["type"].get(); - auto i = nameSchemaMap.find(typeName); - if (i != nameSchemaMap.end()) { - return i->second; - } - if (typeName == "record") { - std::vector> fieldsSchema; - for (const auto& field : obj["fields"]) { - fieldsSchema.push_back( - std::make_pair(field["name"].get(), parseSchemaFromJsonObject(field["type"]))); - } - - const std::string recordName = obj["name"].get(); - auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema)); - nameSchemaMap.insert(std::make_pair(recordName, recordSchema)); - return recordSchema; - } else if (typeName == "enum") { - throw std::runtime_error("Enum type isn't supported yet in Avro schema."); - } else if (typeName == "array") { - return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"])); - } else if (typeName == "map") { - return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"])); - } else if (typeName == "fixed") { - const std::string fixedName = obj["name"].get(); - auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get()); - nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema)); - return fixedSchema; - } else { - throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema."); - } - } - AZURE_UNREACHABLE_CODE(); - }; - - auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end()); - return parseSchemaFromJsonObject(jsonRoot); -} -} // namespace - -int64_t AvroStreamReader::ParseInt(const Core::Context& context) { - uint64_t r = 0; - int nb = 0; - while (true) { - Preload(1, context); - uint8_t c = m_streambuffer[m_pos.Offset++]; - - r = r | ((static_cast(c) & 0x7f) << (nb * 7)); - if (c & 0x80) { - ++nb; - continue; - } - break; - } - return static_cast(r >> 1) ^ -static_cast(r & 0x01); -} - -void AvroStreamReader::Advance(size_t n, const Core::Context& context) { - Preload(n, context); - m_pos.Offset += n; -} - -size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) { - size_t oldAvailable = AvailableBytes(); - while (true) { - size_t newAvailable = TryPreload(n, context); - if (newAvailable >= n) { - return newAvailable; - } - if (oldAvailable == newAvailable) { - throw std::runtime_error("Unexpected EOF of Avro stream."); - } - oldAvailable = newAvailable; - } - AZURE_UNREACHABLE_CODE(); -} - -size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) { - size_t availableBytes = AvailableBytes(); - if (availableBytes >= n) { - return availableBytes; - } - const size_t MinRead = 4096; - size_t tryReadSize = (std::max)(n, MinRead); - size_t currSize = m_streambuffer.size(); - m_streambuffer.resize(m_streambuffer.size() + tryReadSize); - size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context); - m_streambuffer.resize(currSize + actualReadSize); - return AvailableBytes(); -} - -void AvroStreamReader::Discard() { - constexpr size_t MinimumReleaseMemory = 128 * 1024; - if (m_pos.Offset < MinimumReleaseMemory) { - return; - } - const size_t availableBytes = AvailableBytes(); - std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes); - m_streambuffer.resize(availableBytes); - m_pos.Offset = 0; -} - -const AvroSchema AvroSchema::StringSchema(AvroDatumType::String); -const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes); -const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int); -const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long); -const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float); -const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double); -const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool); -const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null); - -AvroSchema AvroSchema::RecordSchema(std::string name, - const std::vector>& fieldsSchema) { - AvroSchema recordSchema(AvroDatumType::Record); - recordSchema.m_name = std::move(name); - recordSchema.m_status = std::make_shared(); - for (auto& i : fieldsSchema) { - recordSchema.m_status->m_keys.push_back(i.first); - recordSchema.m_status->m_schemas.push_back(i.second); - } - return recordSchema; -} - -AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) { - AvroSchema arraySchema(AvroDatumType::Array); - arraySchema.m_status = std::make_shared(); - arraySchema.m_status->m_schemas.push_back(std::move(elementSchema)); - return arraySchema; -} - -AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) { - AvroSchema mapSchema(AvroDatumType::Map); - mapSchema.m_status = std::make_shared(); - mapSchema.m_status->m_schemas.push_back(std::move(elementSchema)); - return mapSchema; -} - -AvroSchema AvroSchema::UnionSchema(std::vector schemas) { - AvroSchema unionSchema(AvroDatumType::Union); - unionSchema.m_status = std::make_shared(); - unionSchema.m_status->m_schemas = std::move(schemas); - return unionSchema; -} - -AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) { - AvroSchema fixedSchema(AvroDatumType::Fixed); - fixedSchema.m_name = std::move(name); - fixedSchema.m_status = std::make_shared(); - fixedSchema.m_status->m_size = size; - return fixedSchema; -} - -void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) { - m_data = reader.m_pos; - if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { - int64_t stringSize = reader.ParseInt(context); - reader.Advance(static_cast(stringSize), context); - } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || - m_schema.Type() == AvroDatumType::Enum) { - reader.ParseInt(context); - } else if (m_schema.Type() == AvroDatumType::Float) { - reader.Advance(4, context); - } else if (m_schema.Type() == AvroDatumType::Double) { - reader.Advance(8, context); - } else if (m_schema.Type() == AvroDatumType::Bool) { - reader.Advance(1, context); - } else if (m_schema.Type() == AvroDatumType::Null) { - reader.Advance(0, context); - } else if (m_schema.Type() == AvroDatumType::Record) { - for (const auto& s : m_schema.FieldSchemas()) { - AvroDatum(s).Fill(reader, context); - } - } else if (m_schema.Type() == AvroDatumType::Array) { - while (true) { - int64_t numElementsInBlock = reader.ParseInt(context); - if (numElementsInBlock == 0) { - break; - } else if (numElementsInBlock < 0) { - int64_t blockSize = reader.ParseInt(context); - reader.Advance(static_cast(blockSize), context); - } else { - for (auto i = 0; i < numElementsInBlock; ++i) { - AvroDatum(m_schema.ItemSchema()).Fill(reader, context); - } - } - } - } else if (m_schema.Type() == AvroDatumType::Map) { - while (true) { - int64_t numElementsInBlock = reader.ParseInt(context); - if (numElementsInBlock == 0) { - break; - } else if (numElementsInBlock < 0) { - int64_t blockSize = reader.ParseInt(context); - reader.Advance(static_cast(blockSize), context); - } else { - for (int64_t i = 0; i < numElementsInBlock; ++i) { - AvroDatum(AvroSchema::StringSchema).Fill(reader, context); - AvroDatum(m_schema.ItemSchema()).Fill(reader, context); - } - } - } - } else if (m_schema.Type() == AvroDatumType::Union) { - int64_t i = reader.ParseInt(context); - AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(reader, context); - } else if (m_schema.Type() == AvroDatumType::Fixed) { - reader.Advance(m_schema.Size(), context); - } else { - AZURE_UNREACHABLE_CODE(); - } -} - -void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) { - m_data = data; - if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { - int64_t stringSize = parseInt(data); - data.Offset += static_cast(stringSize); - } else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long || - m_schema.Type() == AvroDatumType::Enum) { - parseInt(data); - } else if (m_schema.Type() == AvroDatumType::Float) { - data.Offset += 4; - } else if (m_schema.Type() == AvroDatumType::Double) { - data.Offset += 8; - } else if (m_schema.Type() == AvroDatumType::Bool) { - data.Offset += 1; - } else if (m_schema.Type() == AvroDatumType::Null) { - data.Offset += 0; - } else if (m_schema.Type() == AvroDatumType::Record) { - for (const auto& s : m_schema.FieldSchemas()) { - AvroDatum(s).Fill(data); - } - } else if (m_schema.Type() == AvroDatumType::Array) { - while (true) { - int64_t numElementsInBlock = parseInt(data); - if (numElementsInBlock == 0) { - break; - } else if (numElementsInBlock < 0) { - int64_t blockSize = parseInt(data); - data.Offset += static_cast(blockSize); - } else { - for (auto i = 0; i < numElementsInBlock; ++i) { - AvroDatum(m_schema.ItemSchema()).Fill(data); - } - } - } - } else if (m_schema.Type() == AvroDatumType::Map) { - while (true) { - int64_t numElementsInBlock = parseInt(data); - if (numElementsInBlock == 0) { - break; - } else if (numElementsInBlock < 0) { - int64_t blockSize = parseInt(data); - data.Offset += static_cast(blockSize); - } else { - for (int64_t i = 0; i < numElementsInBlock; ++i) { - AvroDatum(AvroSchema::StringSchema).Fill(data); - AvroDatum(m_schema.ItemSchema()).Fill(data); - } - } - } - } else if (m_schema.Type() == AvroDatumType::Union) { - int64_t i = parseInt(data); - AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(data); - } else if (m_schema.Type() == AvroDatumType::Fixed) { - data.Offset += m_schema.Size(); - } else { - AZURE_UNREACHABLE_CODE(); - } -} - -template <> -AvroDatum::StringView AvroDatum::Value() const { - auto data = m_data; - if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) { - const int64_t length = parseInt(data); - const uint8_t* start = &(*data.BufferPtr)[data.Offset]; - StringView ret{start, static_cast(length)}; - data.Offset += static_cast(length); - return ret; - } - if (m_schema.Type() == AvroDatumType::Fixed) { - const size_t fixedSize = m_schema.Size(); - const uint8_t* start = &(*data.BufferPtr)[data.Offset]; - StringView ret{start, fixedSize}; - data.Offset += fixedSize; - return ret; - } - AZURE_UNREACHABLE_CODE(); -} - -template <> -std::string AvroDatum::Value() const { - auto stringView = Value(); - return std::string(stringView.Data, stringView.Data + stringView.Length); -} - -template <> -std::vector AvroDatum::Value() const { - auto stringView = Value(); - return std::vector(stringView.Data, stringView.Data + stringView.Length); -} - -template <> -int64_t AvroDatum::Value() const { - auto data = m_data; - return parseInt(data); -} - -template <> -int32_t AvroDatum::Value() const { - return static_cast(Value()); -} - -template <> -bool AvroDatum::Value() const { - return Value(); -} - -template <> -std::nullptr_t AvroDatum::Value() const { - return nullptr; -} - -template <> -AvroRecord AvroDatum::Value() const { - auto data = m_data; - - AvroRecord r; - r.m_keys = &m_schema.FieldNames(); - for (const auto& schema : m_schema.FieldSchemas()) { - auto datum = AvroDatum(schema); - datum.Fill(data); - r.m_values.push_back(std::move(datum)); - } - - return r; -} - -template <> -AvroMap AvroDatum::Value() const { - auto data = m_data; - - AvroMap m; - while (true) { - int64_t numElementsInBlock = parseInt(data); - if (numElementsInBlock == 0) { - break; - } - if (numElementsInBlock < 0) { - numElementsInBlock = -numElementsInBlock; - parseInt(data); - } - for (int64_t i = 0; i < numElementsInBlock; ++i) { - auto keyDatum = AvroDatum(AvroSchema::StringSchema); - keyDatum.Fill(data); - auto valueDatum = AvroDatum(m_schema.ItemSchema()); - valueDatum.Fill(data); - m[keyDatum.Value()] = valueDatum; - } - } - return m; -} - -template <> -AvroDatum AvroDatum::Value() const { - auto data = m_data; - if (m_schema.Type() == AvroDatumType::Union) { - int64_t i = parseInt(data); - auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast(i)]); - datum.Fill(data); - return datum; - } - AZURE_UNREACHABLE_CODE(); -} - -AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream) - : m_reader(std::make_unique(stream)) {} - -AvroDatum AvroObjectContainerReader::NextImpl(const AvroSchema* schema, const Core::Context& context) { - AZURE_ASSERT_FALSE(m_eof); - static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16); - if (!schema) { - static AvroSchema FileHeaderSchema = []() { - std::vector> fieldsSchema; - fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4))); - fieldsSchema.push_back(std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema))); - fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema)); - return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema)); - }(); - auto fileHeaderDatum = AvroDatum(FileHeaderSchema); - fileHeaderDatum.Fill(*m_reader, context); - auto fileHeader = fileHeaderDatum.Value(); - if (fileHeader.Field("magic").Value() != "Obj\01") { - throw std::runtime_error("Invalid Avro object container magic."); - } - AvroMap meta = fileHeader.Field("meta").Value(); - std::string objectSchemaJson = meta["avro.schema"].Value(); - std::string codec = "null"; - if (meta.count("avro.codec") != 0) { - codec = meta["avro.codec"].Value(); - } - if (codec != "null") { - throw std::runtime_error("Unsupported Avro codec: " + codec); - } - m_syncMarker = fileHeader.Field("sync").Value(); - m_objectSchema = std::make_unique(ParseSchemaFromJsonString(objectSchemaJson)); - schema = m_objectSchema.get(); - } - - if (m_remainingObjectInCurrentBlock == 0) { - m_reader->Discard(); - m_remainingObjectInCurrentBlock = m_reader->ParseInt(context); - int64_t ObjectsSize = m_reader->ParseInt(context); - m_reader->Preload(static_cast(ObjectsSize), context); - } - - auto objectDatum = AvroDatum(*m_objectSchema); - objectDatum.Fill(*m_reader, context); - if (--m_remainingObjectInCurrentBlock == 0) { - auto markerDatum = AvroDatum(SyncMarkerSchema); - markerDatum.Fill(*m_reader, context); - auto marker = markerDatum.Value(); - if (marker != m_syncMarker) { - throw std::runtime_error("Sync marker doesn't match."); - } - m_eof = m_reader->TryPreload(1, context) == 0; - } - return objectDatum; -} - -size_t AvroStreamParser::OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) { - if (m_parserBuffer.Length != 0) { - size_t bytesToCopy = (std::min)(m_parserBuffer.Length, count); - std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy); - m_parserBuffer.Data += bytesToCopy; - m_parserBuffer.Length -= bytesToCopy; - return bytesToCopy; - } - while (!m_parser.End()) { - auto datum = m_parser.Next(context); - if (datum.Schema().Type() == AvroDatumType::Union) { - datum = datum.Value(); - } - if (datum.Schema().Type() != AvroDatumType::Record) { - continue; - } - if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") { - auto record = datum.Value(); - auto dataDatum = record.Field("data"); - m_parserBuffer = dataDatum.Value(); - return OnRead(buffer, count, context); - } - if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" && m_progressCallback) { - auto record = datum.Value(); - auto bytesScanned = record.Field("bytesScanned").Value(); - auto totalBytes = record.Field("totalBytes").Value(); - m_progressCallback(bytesScanned, totalBytes); - } - if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" && m_errorCallback) { - auto record = datum.Value(); - BlobQueryError e; - e.Name = record.Field("name").Value(); - e.Description = record.Field("description").Value(); - e.IsFatal = record.Field("fatal").Value(); - e.Position = record.Field("position").Value(); - m_errorCallback(std::move(e)); - } - } - return 0; -} -} // namespace _detail -} // namespace Blobs -} // namespace Storage -} // namespace Azure - -#endif diff --git a/source/libs/azure/src/avro_parser.hpp b/source/libs/azure/src/avro_parser.hpp deleted file mode 100644 index 275d073c85..0000000000 --- a/source/libs/azure/src/avro_parser.hpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#pragma once - -#include "azure/storage/blobs/blob_options.hpp" - -#include - -#include -#include -#include - -namespace Azure { namespace Storage { namespace Blobs { namespace _detail { - enum class AvroDatumType - { - String, - Bytes, - Int, - Long, - Float, - Double, - Bool, - Null, - Record, - Enum, - Array, - Map, - Union, - Fixed, - }; - - class AvroStreamReader final { - public: - // position of a vector that lives through vector resizing - struct ReaderPos final - { - const std::vector* BufferPtr = nullptr; - size_t Offset = 0; - }; - explicit AvroStreamReader(Core::IO::BodyStream& stream) - : m_stream(&stream), m_pos{&m_streambuffer, 0} - { - } - AvroStreamReader(const AvroStreamReader&) = delete; - AvroStreamReader& operator=(const AvroStreamReader&) = delete; - - int64_t ParseInt(const Core::Context& context); - void Advance(size_t n, const Core::Context& context); - // Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes - // available in m_streambuffer; - size_t Preload(size_t n, const Core::Context& context); - size_t TryPreload(size_t n, const Core::Context& context); - // discards data that's before m_pos - void Discard(); - - private: - size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; } - - private: - Core::IO::BodyStream* m_stream; - std::vector m_streambuffer; - ReaderPos m_pos; - - friend class AvroDatum; - }; - - class AvroSchema final { - public: - static const AvroSchema StringSchema; - static const AvroSchema BytesSchema; - static const AvroSchema IntSchema; - static const AvroSchema LongSchema; - static const AvroSchema FloatSchema; - static const AvroSchema DoubleSchema; - static const AvroSchema BoolSchema; - static const AvroSchema NullSchema; - static AvroSchema RecordSchema( - std::string name, - const std::vector>& fieldsSchema); - static AvroSchema ArraySchema(AvroSchema elementSchema); - static AvroSchema MapSchema(AvroSchema elementSchema); - static AvroSchema UnionSchema(std::vector schemas); - static AvroSchema FixedSchema(std::string name, int64_t size); - - const std::string& Name() const { return m_name; } - AvroDatumType Type() const { return m_type; } - const std::vector& FieldNames() const { return m_status->m_keys; } - AvroSchema ItemSchema() const { return m_status->m_schemas[0]; } - const std::vector& FieldSchemas() const { return m_status->m_schemas; } - size_t Size() const { return static_cast(m_status->m_size); } - - private: - explicit AvroSchema(AvroDatumType type) : m_type(type) {} - - private: - AvroDatumType m_type; - std::string m_name; - - struct SharedStatus - { - std::vector m_keys; - std::vector m_schemas; - int64_t m_size = 0; - }; - std::shared_ptr m_status; - }; - - class AvroDatum final { - public: - AvroDatum() : m_schema(AvroSchema::NullSchema) {} - explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {} - - void Fill(AvroStreamReader& reader, const Core::Context& context); - void Fill(AvroStreamReader::ReaderPos& data); - - const AvroSchema& Schema() const { return m_schema; } - - template T Value() const; - struct StringView - { - const uint8_t* Data = nullptr; - size_t Length = 0; - }; - - private: - AvroSchema m_schema; - AvroStreamReader::ReaderPos m_data; - }; - - using AvroMap = std::map; - - class AvroRecord final { - public: - bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); } - const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); } - AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); } - const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); } - AvroDatum& FieldAt(size_t i) { return m_values.at(i); } - - private: - size_t FindField(const std::string& key) const - { - auto i = find(m_keys->begin(), m_keys->end(), key); - return i - m_keys->begin(); - } - const std::vector* m_keys = nullptr; - std::vector m_values; - - friend class AvroDatum; - }; - - class AvroObjectContainerReader final { - public: - explicit AvroObjectContainerReader(Core::IO::BodyStream& stream); - - bool End() const { return m_eof; } - // Calling Next() will invalidates the previous AvroDatum returned by this function and all - // AvroDatums propagated from there. - AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); } - - private: - AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context); - - private: - std::unique_ptr m_reader; - std::unique_ptr m_objectSchema; - std::string m_syncMarker; - int64_t m_remainingObjectInCurrentBlock = 0; - bool m_eof = false; - }; - - class AvroStreamParser final : public Core::IO::BodyStream { - public: - explicit AvroStreamParser( - std::unique_ptr inner, - std::function progressCallback, - std::function errorCallback) - : m_inner(std::move(inner)), m_parser(*m_inner), - m_progressCallback(std::move(progressCallback)), m_errorCallback(std::move(errorCallback)) - { - } - - int64_t Length() const override { return -1; } - void Rewind() override { this->m_inner->Rewind(); } - - private: - size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override; - - private: - std::unique_ptr m_inner; - AvroObjectContainerReader m_parser; - std::function m_progressCallback; - std::function m_errorCallback; - AvroDatum::StringView m_parserBuffer; - }; - -}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/source/libs/azure/src/td_avro_parser.cpp b/source/libs/azure/src/td_avro_parser.cpp index 485980e007..62bd3a8151 100644 --- a/source/libs/azure/src/td_avro_parser.cpp +++ b/source/libs/azure/src/td_avro_parser.cpp @@ -1,5 +1,5 @@ #if defined(USE_S3) -#include "avro_parser.hpp" +#include #include #include diff --git a/source/libs/azure/src/td_block_blob_client.cpp b/source/libs/azure/src/td_block_blob_client.cpp index b5a5c3c189..33ac774d0c 100644 --- a/source/libs/azure/src/td_block_blob_client.cpp +++ b/source/libs/azure/src/td_block_blob_client.cpp @@ -14,7 +14,7 @@ #include #endif -#include "avro_parser.hpp" +#include #include #include From 2390532bb01179536b00864127cfeaa423e0aa08 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 13:43:42 +0800 Subject: [PATCH 39/41] az/exception: catch all cpp exception to error code --- source/libs/azure/src/az.cpp | 42 +++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index d4f93b9af4..1d693a6a32 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -201,7 +201,7 @@ _next: TAOS_RETURN(code); } -int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { +static int32_t azPutObjectFromFileOffsetImpl(const char *file, const char *object_name, int64_t offset, int64_t size) { int32_t code = 0; std::string endpointUrl = tsS3Hostname[0]; @@ -239,7 +239,25 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int TAOS_RETURN(code); } -int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { +int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { + int32_t code = 0; + + try { + code = azPutObjectFromFileOffsetImpl(file, object_name, offset, size); + } catch (const std::exception &e) { + azError("%s: Reason Phrase: %s", __func__, e.what()); + + code = TAOS_SYSTEM_ERROR(EIO); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +static int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t size, bool check, + uint8_t **ppBlock) { int32_t code = TSDB_CODE_SUCCESS; std::string accountName = tsS3AccessKeyId[0]; std::string accountKey = tsS3AccessKeySecret[0]; @@ -292,7 +310,8 @@ int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t si TAOS_RETURN(code); } -int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { +static int32_t azGetObjectBlockRetry(const char *object_name, int64_t offset, int64_t size, bool check, + uint8_t **ppBlock) { int32_t code = TSDB_CODE_SUCCESS; // May use an exponential backoff policy for retries with 503 @@ -312,6 +331,23 @@ _retry: TAOS_RETURN(code); } +int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + int32_t code = TSDB_CODE_SUCCESS; + + try { + code = azGetObjectBlockRetry(object_name, offset, size, check, ppBlock); + } catch (const std::exception &e) { + azError("%s: Reason Phrase: %s", __func__, e.what()); + + code = TAOS_SYSTEM_ERROR(EIO); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + void azDeleteObjectsByPrefix(const char *prefix) { const std::string delimiter = "/"; std::string accountName = tsS3AccessKeyId[0]; From 6dbee3d08b4ee196cc1a4e04971adad233e12a40 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 15:02:10 +0800 Subject: [PATCH 40/41] az/test: skip ut if no env vars found --- source/libs/azure/test/azTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index c1be835150..0459cb5f6a 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -86,6 +86,11 @@ TEST(AzTest, InterfaceTest) { bool withcp = false; code = azInitEnv(); + if (code) { + std::cout << "ablob env init failed with: " << code << std::endl; + return; + } + GTEST_ASSERT_EQ(code, 0); GTEST_ASSERT_EQ(tsS3Enabled, 1); From 3fd1ad6e824daa13278cab7405aa4e1f85bd7eea Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Oct 2024 16:00:46 +0800 Subject: [PATCH 41/41] az/delete: catch all cpp exceptions --- source/libs/azure/src/az.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 1d693a6a32..831694356a 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -348,7 +348,7 @@ int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, TAOS_RETURN(code); } -void azDeleteObjectsByPrefix(const char *prefix) { +static void azDeleteObjectsByPrefixImpl(const char *prefix) { const std::string delimiter = "/"; std::string accountName = tsS3AccessKeyId[0]; std::string accountKey = tsS3AccessKeySecret[0]; @@ -380,7 +380,16 @@ void azDeleteObjectsByPrefix(const char *prefix) { } catch (const Azure::Core::RequestFailedException &e) { azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast(e.StatusCode), e.ReasonPhrase.c_str()); - // azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TAOS_SYSTEM_ERROR(EIO))); + } +} + +void azDeleteObjectsByPrefix(const char *prefix) { + int32_t code = TSDB_CODE_SUCCESS; + + try { + azDeleteObjectsByPrefixImpl(prefix); + } catch (const std::exception &e) { + azError("%s: Reason Phrase: %s", __func__, e.what()); } }