Merge branch '3.0' into merge/3340
This commit is contained in:
commit
63da0865a2
|
@ -121,6 +121,7 @@ TAGS
|
|||
contrib/*
|
||||
!contrib/CMakeLists.txt
|
||||
!contrib/test
|
||||
!contrib/azure-cmake
|
||||
sql
|
||||
debug*/
|
||||
.env
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# azure
|
||||
ExternalProject_Add(azure
|
||||
URL https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_12.13.0-beta.1.tar.gz
|
||||
URL_HASH SHA256=3eca486fd60e3522d0a633025ecd652a71515b1e944799b2e8ee31fd590305a9
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1"
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
#BUILD_ALWAYS 1
|
||||
#UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
|
@ -220,20 +220,20 @@ ELSE ()
|
|||
|
||||
IF ("${SIMD_SUPPORT}" MATCHES "true")
|
||||
IF (COMPILER_SUPPORT_FMA)
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
|
||||
MESSAGE(STATUS "FMA instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
IF (COMPILER_SUPPORT_AVX)
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
|
||||
MESSAGE(STATUS "AVX instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
IF (COMPILER_SUPPORT_AVX2)
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
|
||||
MESSAGE(STATUS "AVX2 instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
MESSAGE(STATUS "FMA instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
IF (COMPILER_SUPPORT_AVX)
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
|
||||
MESSAGE(STATUS "AVX instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
IF (COMPILER_SUPPORT_AVX2)
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
|
||||
MESSAGE(STATUS "AVX2 instructions is ACTIVATED")
|
||||
ENDIF()
|
||||
|
||||
IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true")
|
||||
|
@ -256,7 +256,8 @@ ELSE ()
|
|||
|
||||
IF (${BUILD_SANITIZER})
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
MESSAGE(STATUS "Compile with Address Sanitizer!")
|
||||
ELSEIF (${BUILD_RELEASE})
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
# taosadapter
|
||||
ExternalProject_Add(taosadapter
|
||||
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
|
||||
GIT_TAG main
|
||||
GIT_TAG 3.0
|
||||
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
# taos-tools
|
||||
ExternalProject_Add(taos-tools
|
||||
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
|
||||
GIT_TAG main
|
||||
GIT_TAG 3.0
|
||||
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
@ -152,6 +152,7 @@ if(${BUILD_WITH_S3})
|
|||
cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/azure_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
add_definitions(-DUSE_S3)
|
||||
|
||||
# cos
|
||||
|
@ -614,9 +615,16 @@ if (${BUILD_PCRE2})
|
|||
add_subdirectory(pcre2 EXCLUDE_FROM_ALL)
|
||||
endif(${BUILD_PCRE2})
|
||||
|
||||
if(${TD_LINUX} AND ${BUILD_WITH_S3})
|
||||
add_subdirectory(azure-cmake EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
|
||||
# ================================================================================================
|
||||
# Build test
|
||||
# ================================================================================================
|
||||
|
||||
MESSAGE("build with dependency tests: ${BUILD_DEPENDENCY_TESTS}")
|
||||
|
||||
if(${BUILD_DEPENDENCY_TESTS})
|
||||
add_subdirectory(test EXCLUDE_FROM_ALL)
|
||||
endif(${BUILD_DEPENDENCY_TESTS})
|
||||
|
|
|
@ -0,0 +1,70 @@
|
|||
# lib_azure_sdk
|
||||
set(AZURE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1")
|
||||
set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk")
|
||||
|
||||
file(GLOB AZURE_SDK_SRC
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/credentials/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/cryptography/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/io/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/tracing/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.cpp"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp"
|
||||
)
|
||||
|
||||
file(GLOB AZURE_SDK_UNIFIED_SRC
|
||||
${AZURE_SDK_SRC}
|
||||
)
|
||||
|
||||
set(AZURE_SDK_INCLUDES
|
||||
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/inc/"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/inc/"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/inc/"
|
||||
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/"
|
||||
)
|
||||
|
||||
add_library(_azure_sdk STATIC ${AZURE_SDK_UNIFIED_SRC})
|
||||
target_compile_definitions(_azure_sdk PRIVATE BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
|
||||
|
||||
target_include_directories(
|
||||
_azure_sdk
|
||||
PUBLIC "$ENV{HOME}/.cos-local.2/include"
|
||||
)
|
||||
|
||||
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
#find_library(CURL_LIBRARY curl)
|
||||
#find_library(XML2_LIBRARY xml2)
|
||||
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
#find_library(CoreFoundation_Library CoreFoundation)
|
||||
#find_library(SystemConfiguration_Library SystemConfiguration)
|
||||
|
||||
target_link_libraries(
|
||||
_azure_sdk
|
||||
PRIVATE ${CURL_LIBRARY}
|
||||
PRIVATE ${SSL_LIBRARY}
|
||||
PRIVATE ${CRYPTO_LIBRARY}
|
||||
PRIVATE ${XML2_LIBRARY}
|
||||
#PRIVATE xml2
|
||||
PRIVATE zlib
|
||||
# PRIVATE ${CoreFoundation_Library}
|
||||
# PRIVATE ${SystemConfiguration_Library}
|
||||
)
|
||||
|
||||
# Originally, on Windows azure-core is built with bcrypt and crypt32 by default
|
||||
if (TARGET OpenSSL::SSL)
|
||||
target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL)
|
||||
endif()
|
||||
|
||||
# Originally, on Windows azure-core is built with winhttp by default
|
||||
if (TARGET td_contrib::curl)
|
||||
target_link_libraries(_azure_sdk PRIVATE td_contrib::curl)
|
||||
endif()
|
||||
|
||||
target_include_directories(_azure_sdk SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES})
|
||||
add_library(td_contrib::azure_sdk ALIAS _azure_sdk)
|
|
@ -28,5 +28,6 @@ if(${BUILD_WITH_TRAFT})
|
|||
# add_subdirectory(traft)
|
||||
endif(${BUILD_WITH_TRAFT})
|
||||
|
||||
add_subdirectory(azure)
|
||||
add_subdirectory(tdev)
|
||||
add_subdirectory(lz4)
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
set(CMAKE_CXX_STANDARD 14)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED True)
|
||||
|
||||
add_executable (
|
||||
azure-test
|
||||
main.cpp
|
||||
)
|
||||
|
||||
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
#find_library(XML2_LIBRARY xml2)
|
||||
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
|
||||
#find_library(CoreFoundation_Library CoreFoundation)
|
||||
#find_library(SystemConfiguration_Library SystemConfiguration)
|
||||
|
||||
target_link_libraries(
|
||||
azure-test
|
||||
PRIVATE _azure_sdk
|
||||
PRIVATE ${CURL_LIBRARY}
|
||||
PRIVATE ${XML2_LIBRARY}
|
||||
PRIVATE ${SSL_LIBRARY}
|
||||
PRIVATE ${CRYPTO_LIBRARY}
|
||||
PRIVATE dl
|
||||
PRIVATE pthread
|
||||
)
|
|
@ -0,0 +1,99 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
// Include the necessary SDK headers
|
||||
#include <azure/core.hpp>
|
||||
#include <azure/storage/blobs.hpp>
|
||||
|
||||
// Add appropriate using namespace directives
|
||||
using namespace Azure::Storage;
|
||||
using namespace Azure::Storage::Blobs;
|
||||
|
||||
// Secrets should be stored & retrieved from secure locations such as Azure::KeyVault. For
|
||||
// convenience and brevity of samples, the secrets are retrieved from environment variables.
|
||||
|
||||
std::string GetEndpointUrl() {
|
||||
// return std::getenv("AZURE_STORAGE_ACCOUNT_URL");
|
||||
std::string accountId = getenv("ablob_account_id");
|
||||
if (accountId.empty()) {
|
||||
return accountId;
|
||||
}
|
||||
|
||||
return accountId + ".blob.core.windows.net";
|
||||
}
|
||||
|
||||
std::string GetAccountName() {
|
||||
// return std::getenv("AZURE_STORAGE_ACCOUNT_NAME");
|
||||
return getenv("ablob_account_id");
|
||||
}
|
||||
|
||||
std::string GetAccountKey() {
|
||||
// return std::getenv("AZURE_STORAGE_ACCOUNT_KEY");
|
||||
|
||||
return getenv("ablob_account_secret");
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::string endpointUrl = GetEndpointUrl();
|
||||
std::string accountName = GetAccountName();
|
||||
std::string accountKey = GetAccountKey();
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
std::string accountURL = "https://fd2d01cd892f844eeaa2273.blob.core.windows.net";
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = "myblobcontainer";
|
||||
// auto containerClient = blobServiceClient.GetBlobContainerClient("myblobcontainer");
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient("td-test");
|
||||
|
||||
// Create the container if it does not exist
|
||||
std::cout << "Creating container: " << containerName << std::endl;
|
||||
// containerClient.CreateIfNotExists();
|
||||
|
||||
std::string blobName = "blob.txt";
|
||||
uint8_t blobContent[] = "Hello Azure!";
|
||||
// Create the block blob client
|
||||
BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
|
||||
|
||||
// Upload the blob
|
||||
std::cout << "Uploading blob: " << blobName << std::endl;
|
||||
blobClient.UploadFrom(blobContent, sizeof(blobContent));
|
||||
/*
|
||||
auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential);
|
||||
|
||||
// Create some data to upload into the blob.
|
||||
std::vector<uint8_t> data = {1, 2, 3, 4};
|
||||
Azure::Core::IO::MemoryBodyStream stream(data);
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobResult> response = blockBlobClient.Upload(stream);
|
||||
|
||||
Models::UploadBlockBlobResult model = response.Value;
|
||||
std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString()
|
||||
<< std::endl;
|
||||
*/
|
||||
} catch (const Azure::Core::RequestFailedException& e) {
|
||||
std::cout << "Status Code: " << static_cast<int>(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase
|
||||
<< std::endl;
|
||||
std::cout << e.what() << std::endl;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
|
@ -19,7 +19,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.3.3</version>
|
||||
<version>3.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
|
|
|
@ -18,7 +18,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.3.3</version>
|
||||
<version>3.4.0</version>
|
||||
</dependency>
|
||||
<!-- druid -->
|
||||
<dependency>
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.3.3</version>
|
||||
<version>3.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
|
|
|
@ -67,7 +67,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.3.3</version>
|
||||
<version>3.4.0</version>
|
||||
<!-- <scope>system</scope>-->
|
||||
<!-- <systemPath>${project.basedir}/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar</systemPath>-->
|
||||
</dependency>
|
||||
|
|
|
@ -22,7 +22,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.3.3</version>
|
||||
<version>3.4.0</version>
|
||||
</dependency>
|
||||
<!-- ANCHOR_END: dep-->
|
||||
|
||||
|
|
|
@ -17,8 +17,8 @@ public class SchemalessWsTest {
|
|||
private static final String jsonDemo = "{\"metric\": \"metric_json\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}";
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
final String url = "jdbc:TAOS-RS://" + host + ":6041?user=root&password=taosdata&batchfetch=true";
|
||||
try(Connection connection = DriverManager.getConnection(url)){
|
||||
final String url = "jdbc:TAOS-WS://" + host + ":6041?user=root&password=taosdata";
|
||||
try (Connection connection = DriverManager.getConnection(url)) {
|
||||
init(connection);
|
||||
AbstractConnection conn = connection.unwrap(AbstractConnection.class);
|
||||
|
||||
|
|
|
@ -12,9 +12,9 @@ public class WSConnectExample {
|
|||
public static void main(String[] args) throws Exception {
|
||||
// use
|
||||
// String jdbcUrl =
|
||||
// "jdbc:TAOS-RS://localhost:6041/dbName?user=root&password=taosdata&batchfetch=true";
|
||||
// "jdbc:TAOS-WS://localhost:6041/dbName?user=root&password=taosdata";
|
||||
// if you want to connect a specified database named "dbName".
|
||||
String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata&batchfetch=true";
|
||||
String jdbcUrl = "jdbc:TAOS-WS://localhost:6041?user=root&password=taosdata";
|
||||
Properties connProps = new Properties();
|
||||
connProps.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true");
|
||||
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
|
||||
|
|
|
@ -15,7 +15,7 @@ public class WSParameterBindingBasicDemo {
|
|||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
|
||||
String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true";
|
||||
String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041";
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) {
|
||||
init(conn);
|
||||
|
||||
|
@ -40,7 +40,7 @@ public class WSParameterBindingBasicDemo {
|
|||
pstmt.setFloat(4, random.nextFloat());
|
||||
pstmt.addBatch();
|
||||
}
|
||||
int [] exeResult = pstmt.executeBatch();
|
||||
int[] exeResult = pstmt.executeBatch();
|
||||
// you can check exeResult here
|
||||
System.out.println("Successfully inserted " + exeResult.length + " rows to power.meters.");
|
||||
}
|
||||
|
@ -60,7 +60,8 @@ public class WSParameterBindingBasicDemo {
|
|||
try (Statement stmt = conn.createStatement()) {
|
||||
stmt.execute("CREATE DATABASE IF NOT EXISTS power");
|
||||
stmt.execute("USE power");
|
||||
stmt.execute("CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
|
||||
stmt.execute(
|
||||
"CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -40,7 +40,7 @@ public class WSParameterBindingFullDemo {
|
|||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
|
||||
String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true";
|
||||
String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041/";
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) {
|
||||
|
||||
|
@ -51,8 +51,10 @@ public class WSParameterBindingFullDemo {
|
|||
stmtAll(conn);
|
||||
|
||||
} catch (SQLException ex) {
|
||||
// handle any errors, please refer to the JDBC specifications for detailed exceptions info
|
||||
System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: " + ex.getMessage());
|
||||
// handle any errors, please refer to the JDBC specifications for detailed
|
||||
// exceptions info
|
||||
System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: "
|
||||
+ ex.getMessage());
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
System.out.println("Failed to insert data using stmt, ErrMessage: " + ex.getMessage());
|
||||
|
@ -104,30 +106,29 @@ public class WSParameterBindingFullDemo {
|
|||
pstmt.setTagBoolean(3, true);
|
||||
pstmt.setTagString(4, "binary_value");
|
||||
pstmt.setTagNString(5, "nchar_value");
|
||||
pstmt.setTagVarbinary(6, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e});
|
||||
pstmt.setTagGeometry(7, new byte[]{
|
||||
pstmt.setTagVarbinary(6, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e });
|
||||
pstmt.setTagGeometry(7, new byte[] {
|
||||
0x01, 0x01, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x59,
|
||||
0x40, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x59, 0x40});
|
||||
0x00, 0x00, 0x00, 0x59, 0x40 });
|
||||
|
||||
long current = System.currentTimeMillis();
|
||||
|
||||
|
||||
pstmt.setTimestamp(1, new Timestamp(current));
|
||||
pstmt.setInt(2, 1);
|
||||
pstmt.setDouble(3, 1.1);
|
||||
pstmt.setBoolean(4, true);
|
||||
pstmt.setString(5, "binary_value");
|
||||
pstmt.setNString(6, "nchar_value");
|
||||
pstmt.setVarbinary(7, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e});
|
||||
pstmt.setGeometry(8, new byte[]{
|
||||
pstmt.setVarbinary(7, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e });
|
||||
pstmt.setGeometry(8, new byte[] {
|
||||
0x01, 0x01, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x59,
|
||||
0x40, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x59, 0x40});
|
||||
0x00, 0x00, 0x00, 0x59, 0x40 });
|
||||
pstmt.addBatch();
|
||||
pstmt.executeBatch();
|
||||
System.out.println("Successfully inserted rows to example_all_type_stmt.ntb");
|
||||
|
|
|
@ -3,6 +3,6 @@
|
|||
```
|
||||
|
||||
:::note
|
||||
对于 Rust 连接器, 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性,那么只有 Websocket 的实现会被编译进来。
|
||||
对于 Rust 连接器, 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性,那么只有 WebSocket 的实现会被编译进来。
|
||||
|
||||
:::
|
||||
|
|
|
@ -28,7 +28,7 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
|
|||
|
||||
1. 通过客户端驱动程序 taosc 直接与服务端程序 taosd 建立连接,这种连接方式下文中简称 “原生连接”。
|
||||
2. 通过 taosAdapter 组件提供的 REST API 建立与 taosd 的连接,这种连接方式下文中简称 “REST 连接”
|
||||
3. 通过 taosAdapter 组件提供的 Websocket API 建立与 taosd 的连接,这种连接方式下文中简称 “Websocket 连接”
|
||||
3. 通过 taosAdapter 组件提供的 WebSocket API 建立与 taosd 的连接,这种连接方式下文中简称 “WebSocket 连接”
|
||||
|
||||

|
||||
|
||||
|
@ -38,9 +38,9 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
|
|||
关键不同点在于:
|
||||
|
||||
1. 使用 原生连接,需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本配套。
|
||||
2. 使用 REST 连接,用户无需安装客户端驱动程序 taosc,具有跨平台易用的优势,但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 Websocket 连接相比,REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。
|
||||
3. 使用 Websocket 连接,用户也无需安装客户端驱动程序 taosc。
|
||||
4. 连接云服务实例,必须使用 REST 连接 或 Websocket 连接。
|
||||
2. 使用 REST 连接,用户无需安装客户端驱动程序 taosc,具有跨平台易用的优势,但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 WebSocket 连接相比,REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。
|
||||
3. 使用 WebSocket 连接,用户也无需安装客户端驱动程序 taosc。
|
||||
4. 连接云服务实例,必须使用 REST 连接 或 WebSocket 连接。
|
||||
|
||||
**推荐使用 WebSocket 连接**
|
||||
|
||||
|
@ -126,7 +126,7 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
|
|||
```bash
|
||||
pip3 install taos-ws-py
|
||||
```
|
||||
:::note 此安装包为 Websocket 连接器
|
||||
:::note 此安装包为 WebSocket 连接器
|
||||
- 同时安装 `taospy` 和 `taos-ws-py`
|
||||
```bash
|
||||
pip3 install taospy[ws]
|
||||
|
@ -182,7 +182,7 @@ taos = { version = "*"}
|
|||
```
|
||||
|
||||
:::info
|
||||
Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 Websocket 连接,如果仅需要建立 Websocket 连接,可设置 `ws` 特性:
|
||||
Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 WebSocket 连接,如果仅需要建立 WebSocket 连接,可设置 `ws` 特性:
|
||||
|
||||
```toml
|
||||
taos = { version = "*", default-features = false, features = ["ws"] }
|
||||
|
@ -201,7 +201,7 @@ taos = { version = "*", default-features = false, features = ["ws"] }
|
|||
```
|
||||
npm install @tdengine/websocket
|
||||
```
|
||||
:::note Node.js 目前只支持 Websocket 连接
|
||||
:::note Node.js 目前只支持 WebSocket 连接
|
||||
- **安装验证**
|
||||
- 新建安装验证目录,例如:`~/tdengine-test`,下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/nodejsChecker.js)到本地。
|
||||
- 在命令行中执行以下命令。
|
||||
|
@ -271,12 +271,10 @@ dotnet add package TDengine.Connector
|
|||
<TabItem label="Java" value="java">
|
||||
Java 连接器建立连接的参数有 URL 和 Properties。
|
||||
TDengine 的 JDBC URL 规范格式为:
|
||||
`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
|
||||
URL 和 Properties 的详细参数说明和如何使用详见 [url 规范](../../reference/connector/java/#url-规范)
|
||||
|
||||
**注**:REST 连接中增加 `batchfetch` 参数并设置为 true,将开启 WebSocket 连接。
|
||||
|
||||
</TabItem>
|
||||
<TabItem label="Python" value="python">
|
||||
Python 连接器使用 `connect()` 方法来建立连接,下面是连接参数的具体说明:
|
||||
|
@ -387,8 +385,8 @@ DSN 的详细说明和如何使用详见 [连接功能](../../reference/connecto
|
|||
- `reconnectIntervalMs`:重连间隔毫秒时间,默认为 2000。
|
||||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
**Websocket 连接**
|
||||
C/C++ 语言连接器 Websocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串,其基本结构如下:
|
||||
**WebSocket 连接**
|
||||
C/C++ 语言连接器 WebSocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串,其基本结构如下:
|
||||
|
||||
```text
|
||||
<driver>[+<protocol>]://[[<username>:<password>@]<host>:<port>][/<database>][?<p1>=<v1>[&<p2>=<v2>]]
|
||||
|
@ -417,8 +415,8 @@ C/C++ 语言连接器原生连接方式使用 `taos_connect()` 函数用于建
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Websocket 连接
|
||||
下面是各语言连接器建立 Websocket 连接代码样例。演示了如何使用 Websocket 连接方式连接到 TDengine 数据库,并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。
|
||||
### WebSocket 连接
|
||||
下面是各语言连接器建立 WebSocket 连接代码样例。演示了如何使用 WebSocket 连接方式连接到 TDengine 数据库,并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。
|
||||
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem label="Java" value="java">
|
||||
|
|
|
@ -33,7 +33,7 @@ REST API:直接调用 `taosadapter` 提供的 REST API 接口,进行数据
|
|||
</TabItem>
|
||||
<TabItem label="Python" value="python">
|
||||
|
||||
```python title="Websocket 连接"
|
||||
```python title="WebSocket 连接"
|
||||
{{#include docs/examples/python/create_db_ws.py}}
|
||||
```
|
||||
|
||||
|
@ -69,7 +69,7 @@ REST API:直接调用 `taosadapter` 提供的 REST API 接口,进行数据
|
|||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
|
||||
```c title="Websocket 连接"
|
||||
```c title="WebSocket 连接"
|
||||
{{#include docs/examples/c-ws/create_db_demo.c:create_db_and_table}}
|
||||
```
|
||||
|
||||
|
@ -114,7 +114,7 @@ NOW 为系统内部函数,默认为客户端所在计算机当前时间。 NOW
|
|||
</TabItem>
|
||||
<TabItem label="Python" value="python">
|
||||
|
||||
```python title="Websocket 连接"
|
||||
```python title="WebSocket 连接"
|
||||
{{#include docs/examples/python/insert_ws.py}}
|
||||
```
|
||||
|
||||
|
@ -151,7 +151,7 @@ NOW 为系统内部函数,默认为客户端所在计算机当前时间。 NOW
|
|||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
|
||||
```c title="Websocket 连接"
|
||||
```c title="WebSocket 连接"
|
||||
{{#include docs/examples/c-ws/insert_data_demo.c:insert_data}}
|
||||
```
|
||||
|
||||
|
@ -189,7 +189,7 @@ curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \
|
|||
</TabItem>
|
||||
<TabItem label="Python" value="python">
|
||||
|
||||
```python title="Websocket 连接"
|
||||
```python title="WebSocket 连接"
|
||||
{{#include docs/examples/python/query_ws.py}}
|
||||
```
|
||||
|
||||
|
@ -230,7 +230,7 @@ rust 连接器还支持使用 **serde** 进行反序列化行为结构体的结
|
|||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
|
||||
```c title="Websocket 连接"
|
||||
```c title="WebSocket 连接"
|
||||
{{#include docs/examples/c-ws/query_data_demo.c:query_data}}
|
||||
```
|
||||
|
||||
|
@ -273,7 +273,7 @@ reqId 可用于请求链路追踪,reqId 就像分布式系统中的 traceId
|
|||
</TabItem>
|
||||
<TabItem label="Python" value="python">
|
||||
|
||||
```python title="Websocket 连接"
|
||||
```python title="WebSocket 连接"
|
||||
{{#include docs/examples/python/reqid_ws.py}}
|
||||
```
|
||||
|
||||
|
@ -310,7 +310,7 @@ reqId 可用于请求链路追踪,reqId 就像分布式系统中的 traceId
|
|||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
|
||||
```c "Websocket 连接"
|
||||
```c "WebSocket 连接"
|
||||
{{#include docs/examples/c-ws/with_reqid_demo.c:with_reqid}}
|
||||
```
|
||||
|
||||
|
|
|
@ -191,7 +191,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c6="passit" 1626006833640000000
|
|||
|
||||
:::
|
||||
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
|
|
@ -23,7 +23,7 @@ import TabItem from "@theme/TabItem";
|
|||
- 执行批量插入操作,将这些数据行插入到对应的子表中。
|
||||
3. 最后打印实际插入表中的行数。
|
||||
|
||||
## Websocket 连接
|
||||
## WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
```java
|
||||
|
|
|
@ -94,7 +94,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
|
||||
</TabItem>
|
||||
<TabItem label="C" value="c">
|
||||
- Websocket 连接: 因为使用 dsn,不需要 `td.connect.ip`,`td.connect.port`,`td.connect.user` 和 `td.connect.pass` 四个配置项,其余同通用配置项。
|
||||
- WebSocket 连接: 因为使用 dsn,不需要 `td.connect.ip`,`td.connect.port`,`td.connect.user` 和 `td.connect.pass` 四个配置项,其余同通用配置项。
|
||||
- 原生连接: 同通用基础配置项。
|
||||
|
||||
</TabItem>
|
||||
|
@ -103,8 +103,8 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Websocket 连接
|
||||
介绍各语言连接器使用 Websocket 连接方式创建消费者。指定连接的服务器地址,设置自动提交,从最新消息开始消费,指定 `group.id` 和 `client.id` 等信息。有的语言的连接器还支持反序列化参数。
|
||||
### WebSocket 连接
|
||||
介绍各语言连接器使用 WebSocket 连接方式创建消费者。指定连接的服务器地址,设置自动提交,从最新消息开始消费,指定 `group.id` 和 `client.id` 等信息。有的语言的连接器还支持反序列化参数。
|
||||
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
@ -234,7 +234,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
|
||||
## 订阅消费数据
|
||||
消费者订阅主题后,可以开始接收并处理这些主题中的消息。订阅消费数据的示例代码如下:
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
||||
|
@ -403,7 +403,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
## 指定订阅的 Offset
|
||||
消费者可以指定从特定 Offset 开始读取分区中的消息,这允许消费者重读消息或跳过已处理的消息。下面展示各语言连接器如何指定订阅的 Offset。
|
||||
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
||||
|
@ -549,7 +549,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
|
||||
**注意**:手工提交消费进度前确保消息正常处理完成,否则处理出错的消息不会被再次消费。自动提交是在本次 `poll` 消息时可能会提交上次消息的消费进度,因此请确保消息处理完毕再进行下一次 `poll` 或消息获取。
|
||||
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
||||
|
@ -663,7 +663,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
## 取消订阅和关闭消费
|
||||
消费者可以取消对主题的订阅,停止接收消息。当消费者不再需要时,应该关闭消费者实例,以释放资源和断开与 TDengine 服务器的连接。
|
||||
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
|
||||
|
@ -766,7 +766,7 @@ Rust 连接器创建消费者的参数为 DSN, 可以设置的参数列表请
|
|||
|
||||
|
||||
## 完整示例
|
||||
### Websocket 连接
|
||||
### WebSocket 连接
|
||||
<Tabs defaultValue="java" groupId="lang">
|
||||
<TabItem value="java" label="Java">
|
||||
<details>
|
||||
|
|
|
@ -30,9 +30,8 @@ toc_max_heading_level: 4
|
|||
目前只有 Java 连接器在 WebSocket 连接模式下支持双活,其配置示例如下
|
||||
|
||||
```java
|
||||
url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
|
||||
url = "jdbc:TAOS-WS://" + host + ":6041/?user=root&password=taosdata";
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD, "true");
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_HOST, "192.168.1.11");
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_PORT, "6041");
|
||||
properties.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true");
|
||||
|
@ -43,13 +42,13 @@ connection = DriverManager.getConnection(url, properties);
|
|||
|
||||
其中的配置属性及含义如下表
|
||||
|
||||
| 属性名 | 含义 |
|
||||
| ----------------- | ------------------ |
|
||||
| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip,默认空 |
|
||||
| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号,默认空 |
|
||||
| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。双活场景下请设置为 true |
|
||||
| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔,单位毫秒:默认 2000 毫秒,也就是 2 秒;最小值为 0, 表示立即重试;最大值不做限制 |
|
||||
| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数:默认值为 3;最小值为 0,表示不进行重试;最大值不做限制 |
|
||||
| 属性名 | 含义 |
|
||||
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
|
||||
| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip,默认空 |
|
||||
| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号,默认空 |
|
||||
| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。双活场景下请设置为 true |
|
||||
| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔,单位毫秒:默认 2000 毫秒,也就是 2 秒;最小值为 0, 表示立即重试;最大值不做限制 |
|
||||
| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数:默认值为 3;最小值为 0,表示不进行重试;最大值不做限制 |
|
||||
|
||||
### 约束条件
|
||||
|
||||
|
|
|
@ -306,7 +306,7 @@ http 返回内容:
|
|||
|
||||
## taosAdapter 监控指标
|
||||
|
||||
taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标上报给 taosKeeper,这些监控指标会被 taosKeeper 写入监控数据库,默认是 `log` 库,可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
|
||||
taosAdapter 采集 REST/WebSocket 相关请求的监控指标。将监控指标上报给 taosKeeper,这些监控指标会被 taosKeeper 写入监控数据库,默认是 `log` 库,可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
|
||||
|
||||
#### adapter\_requests 表
|
||||
|
||||
|
@ -331,7 +331,7 @@ taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标
|
|||
| query\_in\_process | INT UNSIGNED | | 正在处理查询请求数 |
|
||||
| write\_in\_process | INT UNSIGNED | | 正在处理写入请求数 |
|
||||
| endpoint | VARCHAR | | 请求端点 |
|
||||
| req\_type | NCHAR | TAG | 请求类型:0 为 REST,1 为 Websocket |
|
||||
| req\_type | NCHAR | TAG | 请求类型:0 为 REST,1 为 WebSocket |
|
||||
|
||||
## 结果返回条数限制
|
||||
|
||||
|
|
|
@ -5,14 +5,14 @@ toc_max_heading_level: 4
|
|||
---
|
||||
|
||||
C/C++ 开发人员可以使用 TDengine 的客户端驱动,即 C/C++连接器 (以下都用 TDengine 客户端驱动表示),开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时,需要包含 TDengine 头文件,里面列出了提供的 API 的函数原型;应用程序还要链接到所在平台上对应的动态库。
|
||||
TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库,分别支持 Websocket 连接和原生连接。 Websocket 连接和原生连接的区别是 Websocket 连接方式不要求客户端和服务端版本完全匹配,而原生连接要求,在性能上 Websocket 连接方式也接近于原生连接,一般我们推荐使用 Websocket 连接方式。
|
||||
TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库,分别支持 WebSocket 连接和原生连接。 WebSocket 连接和原生连接的区别是 WebSocket 连接方式不要求客户端和服务端版本完全匹配,而原生连接要求,在性能上 WebSocket 连接方式也接近于原生连接,一般我们推荐使用 WebSocket 连接方式。
|
||||
|
||||
下面我们分开介绍两种连接方式的使用方法。
|
||||
|
||||
|
||||
## Websocket 连接方式
|
||||
## WebSocket 连接方式
|
||||
|
||||
Websocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。
|
||||
WebSocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。
|
||||
|
||||
```c
|
||||
#include <taosws.h>
|
||||
|
@ -44,7 +44,7 @@ TDengine 客户端驱动的动态库位于:
|
|||
### 错误码
|
||||
|
||||
在 C 接口的设计中,错误码采用整数类型表示,每个错误码都对应一个特定的错误状态。如未特别说明,当 API 的返回值是整数时,_0_ 代表成功,其它是代表失败原因的错误码,当返回值是指针时, _NULL_ 表示失败。
|
||||
Websocket 连接方式单独的错误码在 `taosws.h` 中,
|
||||
WebSocket 连接方式单独的错误码在 `taosws.h` 中,
|
||||
|
||||
|
||||
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
|
||||
|
@ -82,7 +82,7 @@ WebSocket 连接方式错误码只保留了原生连接错误码的后两个字
|
|||
|
||||
#### DSN
|
||||
|
||||
C/C++ Websocket 连接器通过 DSN 连接描述字符串来表示连接信息。
|
||||
C/C++ WebSocket 连接器通过 DSN 连接描述字符串来表示连接信息。
|
||||
DSN 描述字符串基本结构如下:
|
||||
|
||||
```text
|
||||
|
@ -96,16 +96,16 @@ DSN 描述字符串基本结构如下:
|
|||
- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接,支持如下驱动名:
|
||||
- **taos**: 默认驱动,支持 SQL 执行,参数绑定,无模式写入。
|
||||
- **tmq**: 使用 TMQ 订阅数据。
|
||||
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。
|
||||
- **http/ws**: 使用 Websocket 协议。
|
||||
- **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 协议。
|
||||
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。
|
||||
- **http/ws**: 使用 WebSocket 协议。
|
||||
- **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 协议。
|
||||
|
||||
- **username/password**: 用于创建连接的用户名及密码。
|
||||
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时 Websocket 连接默认为 `localhost:6041` 。
|
||||
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时 WebSocket 连接默认为 `localhost:6041` 。
|
||||
- **database**: 指定默认连接的数据库名,可选参数。
|
||||
- **params**:其他可选参数。
|
||||
|
||||
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test`, 表示使用 Websocket(`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
|
||||
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test`, 表示使用 WebSocket(`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
|
||||
|
||||
#### 基础 API
|
||||
|
||||
|
|
|
@ -33,14 +33,15 @@ REST 连接支持所有能运行 Java 的平台。
|
|||
|
||||
| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 |
|
||||
| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: |
|
||||
| 3.3.3 | 1. 解决了 Websocket statement 关闭导致的内存泄漏 | - |
|
||||
| 3.3.2 | 1. 优化 Websocket 连接下的参数绑定性能;2. 优化了对 mybatis 的支持 | - |
|
||||
| 3.3.0 | 1. 优化 Websocket 连接下的数据传输性能;2. 支持跳过 SSL 验证,默认关闭 | 3.3.2.0 及更高版本 |
|
||||
| 3.4.0 | 1. 使用 jackson 库替换 fastjson 库;2. WebSocket 采用独立协议标识;3. 优化后台拉取线程使用,避免用户误用导致超时。 | - |
|
||||
| 3.3.3 | 1. 解决了 WebSocket statement 关闭导致的内存泄漏 | - |
|
||||
| 3.3.2 | 1. 优化 WebSocket 连接下的参数绑定性能;2. 优化了对 mybatis 的支持 | - |
|
||||
| 3.3.0 | 1. 优化 WebSocket 连接下的数据传输性能;2. 支持跳过 SSL 验证,默认关闭 | 3.3.2.0 及更高版本 |
|
||||
| 3.2.11 | 解决了 Native 连接关闭结果集 bug | - |
|
||||
| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩;2. Websocket 自动重连机制,默认关闭;3. Connection 类提供无模式写入的方法;4. 优化了原生连接的数据拉取性能;5. 修复了一些已知问题;6.元数据获取函数可以返回支持的函数列表。 | - |
|
||||
| 3.2.9 | 解决了 Websocket prepareStatement 关闭 bug | - |
|
||||
| 3.2.8 | 优化了自动提交, 解决了 websocket 手动提交 bug, 优化 Websocket prepareStatement 使用一个连接, 元数据支持视图 | - |
|
||||
| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型,增加 native 连接的时区设置支持。增加 websocket 自动重连功能。 | 3.2.0.0 及更高版本 |
|
||||
| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩;2. WebSocket 自动重连机制,默认关闭;3. Connection 类提供无模式写入的方法;4. 优化了原生连接的数据拉取性能;5. 修复了一些已知问题;6.元数据获取函数可以返回支持的函数列表。 | - |
|
||||
| 3.2.9 | 解决了 WebSocket prepareStatement 关闭 bug | - |
|
||||
| 3.2.8 | 优化了自动提交, 解决了 WebSocket 手动提交 bug, 优化 WebSocket prepareStatement 使用一个连接, 元数据支持视图 | - |
|
||||
| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型,增加 native 连接的时区设置支持。增加 WebSocket 自动重连功能。 | 3.2.0.0 及更高版本 |
|
||||
| 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 |
|
||||
| 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数,以及 unsubscribe() 方法。 | - |
|
||||
| 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - |
|
||||
|
@ -195,17 +196,14 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w
|
|||
## API 参考
|
||||
### JDBC 驱动
|
||||
|
||||
taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口,提供了两个实现类:RestfulDriver 和 TSDBDriver。
|
||||
Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
|
||||
|
||||
taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口,提供了 3 个实现类。
|
||||
- WebSocket 连接使用驱动类 `com.taosdata.jdbc.ws.WebSocketDriver`。
|
||||
- 原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
|
||||
- REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。
|
||||
|
||||
#### URL 规范
|
||||
TDengine 的 JDBC URL 规范格式为:
|
||||
`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
|
||||
对于建立连接,原生连接与 REST 连接有细微不同。 Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
|
||||
|
||||
**注**:REST 连接中增加 `batchfetch` 参数并设置为 true,将开启 WebSocket 连接。
|
||||
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
|
||||
**原生连接**
|
||||
`jdbc:TAOS://taosdemo.com:6030/power?user=root&password=taosdata`,使用了 JDBC 原生连接的 TSDBDriver,建立了到 hostname 为 taosdemo.com,端口为 6030(TDengine 的默认端口),数据库名为 power 的连接。这个 URL
|
||||
|
@ -234,23 +232,38 @@ TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可
|
|||
|
||||
> **注意**:这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
|
||||
|
||||
**WebSocket 连接**
|
||||
使用 JDBC WebSocket 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
|
||||
|
||||
**Websocket 和 REST 连接**
|
||||
使用 JDBC Websocket 或 REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
|
||||
1. driverClass 指定为“com.taosdata.jdbc.ws.WebSocketDriver”;
|
||||
2. jdbcUrl 以“jdbc:TAOS-WS://”开头;
|
||||
3. 使用 6041 作为连接端口。
|
||||
|
||||
对于 WebSocket 连接,url 中的配置参数如下:
|
||||
- user:登录 TDengine 用户名,默认值 'root'。
|
||||
- password:用户登录密码,默认值 'taosdata'。
|
||||
- charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。
|
||||
- batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false:不再执行失败 SQL 后的任何语句。默认值为:false。
|
||||
- httpConnectTimeout: 连接超时时间,单位 ms, 默认值为 60000。
|
||||
- messageWaitTimeout: 消息超时时间, 单位 ms, 默认值为 60000。
|
||||
- useSSL: 连接中是否使用 SSL。
|
||||
|
||||
**注意**:部分配置项(比如:locale、timezone)在 WebSocket 连接中不生效。
|
||||
|
||||
**REST 连接**
|
||||
使用 JDBC REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
|
||||
|
||||
1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”;
|
||||
2. jdbcUrl 以“jdbc:TAOS-RS://”开头;
|
||||
3. 使用 6041 作为连接端口。
|
||||
|
||||
对于 Websocket 和 REST 连接,url 中的配置参数如下:
|
||||
对于 REST 连接,url 中的配置参数如下:
|
||||
- user:登录 TDengine 用户名,默认值 'root'。
|
||||
- password:用户登录密码,默认值 'taosdata'。
|
||||
- batchfetch: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP,WebSocket 可以使 JDBC REST 连接支持大数据量查询,并提升查询性能。
|
||||
- charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。
|
||||
- batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false:不再执行失败 SQL 后的任何语句。默认值为:false。
|
||||
- httpConnectTimeout: 连接超时时间,单位 ms, 默认值为 60000。
|
||||
- httpSocketTimeout: socket 超时时间,单位 ms,默认值为 60000。仅在 batchfetch 设置为 false 时生效。
|
||||
- messageWaitTimeout: 消息超时时间, 单位 ms, 默认值为 60000。 仅在 batchfetch 设置为 true 时生效。
|
||||
- httpSocketTimeout: socket 超时时间,单位 ms,默认值为 60000。
|
||||
- useSSL: 连接中是否使用 SSL。
|
||||
- httpPoolSize: REST 并发请求大小,默认 20。
|
||||
|
||||
|
@ -272,7 +285,7 @@ TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可
|
|||
properties 中的配置参数如下:
|
||||
- TSDBDriver.PROPERTY_KEY_USER:登录 TDengine 用户名,默认值 'root'。
|
||||
- TSDBDriver.PROPERTY_KEY_PASSWORD:用户登录密码,默认值 'taosdata'。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:false。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:false。因历史原因使用 REST 连接时,若设置此参数为 true 会变成 WebSocket 连接。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 sq 了。false:不再执行失败 SQL 后的任何语句。默认值为:false。
|
||||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR:仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径,Linux OS 上默认值 `/etc/taos`,Windows OS 上默认值 `C:/TDengine/cfg`。
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET:客户端使用的字符集,默认值为系统字符集。
|
||||
|
@ -280,16 +293,16 @@ properties 中的配置参数如下:
|
|||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE:仅在使用 JDBC 原生连接时生效。 客户端使用的时区,默认值为系统当前时区。因为历史的原因,我们只支持POSIX标准的部分规范,如UTC-8(代表中国上上海), GMT-8,Asia/Shanghai 这几种形式。
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间,单位 ms, 默认值为 60000。仅在 REST 连接时生效。
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间,单位 ms,默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms, 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms, 默认值为 60000。 仅 WebSocket 连接下有效。
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 WebSocket/REST 连接时生效。
|
||||
- TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小,默认 20。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
> **注意**:启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效,对后面的 `use db` 语句切换数据库无效。
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔,单位毫秒,默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数,默认值 3,仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
|
||||
此外对 JDBC 原生连接,通过指定 URL 和 Properties 还可以指定其他参数,比如日志级别、SQL 长度等。
|
||||
|
||||
|
@ -1154,7 +1167,7 @@ JDBC 驱动支持标准的 ResultSet 接口,提供了用于读取结果集中
|
|||
PreparedStatement 允许使用预编译的 SQL 语句,这可以提高性能并提供参数化查询的能力,从而增加安全性。
|
||||
JDBC 驱动提供了实现 PreparedStatement 接口的两个类:
|
||||
1. 对应原生连接的 TSDBPreparedStatement
|
||||
2. 对应 Websocket 连接的 TSWSPreparedStatement
|
||||
2. 对应 WebSocket 连接的 TSWSPreparedStatement
|
||||
|
||||
因 JDBC 标准没有高性能绑定数据的接口,TSDBPreparedStatement 和 TSWSPreparedStatement 都新增了一些方法,用来扩展参数绑定能力。
|
||||
> **注意**:由于 PreparedStatement 继承了 Statement 接口,因此对于这部分重复的接口不再赘述,请参考 Statement 接口中对应描述。
|
||||
|
@ -1347,8 +1360,8 @@ JDBC 标准不支持数据订阅,因此本章所有接口都是扩展接口。
|
|||
- httpConnectTimeout: 创建连接超时参数,单位 ms,默认为 5000 ms。仅在 WebSocket 连接下有效。
|
||||
- messageWaitTimeout: 数据传输超时参数,单位 ms,默认为 10000 ms。仅在 WebSocket 连接下有效。
|
||||
- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用,false: 不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔,单位毫秒,默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数,默认值 3,仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
|
||||
|
|
|
@ -18,9 +18,9 @@ import RequestId from "./_request_id.mdx";
|
|||
|
||||
## 连接方式
|
||||
|
||||
`taos` 提供两种建立连接的方式。一般我们推荐使用 **Websocket 连接**。
|
||||
`taos` 提供两种建立连接的方式。一般我们推荐使用 **WebSocket 连接**。
|
||||
- **原生连接**,它通过 TDengine 客户端驱动程序(taosc)连接 TDengine 运行实例。
|
||||
- **Websocket 连接**,它通过 taosAdapter 的 Websocket 接口连接 TDengine 运行实例。
|
||||
- **WebSocket 连接**,它通过 taosAdapter 的 WebSocket 接口连接 TDengine 运行实例。
|
||||
|
||||
你可以通过不同的 “特性(即 Cargo 关键字 `features`)” 来指定使用哪种连接器(默认同时支持)。
|
||||
|
||||
|
@ -29,13 +29,13 @@ import RequestId from "./_request_id.mdx";
|
|||
## 支持的平台
|
||||
|
||||
原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。
|
||||
Websocket 连接支持所有能运行 Rust 的平台。
|
||||
WebSocket 连接支持所有能运行 Rust 的平台。
|
||||
|
||||
## 版本历史
|
||||
|
||||
| Rust 连接器版本 | TDengine 版本 | 主要功能 |
|
||||
| :----------------: | :--------------: | :--------------------------------------------------: |
|
||||
| v0.12.3 | 3.3.0.0 or later | 优化了 Websocket 查询和插入性能,支持了 VARBINARY 和 GEOMETRY 类型 |
|
||||
| v0.12.3 | 3.3.0.0 or later | 优化了 WebSocket 查询和插入性能,支持了 VARBINARY 和 GEOMETRY 类型 |
|
||||
| v0.12.0 | 3.2.3.0 or later | WS 支持压缩。 |
|
||||
| v0.11.0 | 3.2.0.0 | TMQ 功能优化。 |
|
||||
| v0.10.0 | 3.1.0.0 | WS endpoint 变更。 |
|
||||
|
@ -115,15 +115,15 @@ DSN 描述字符串基本结构如下:
|
|||
- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接,支持如下驱动名:
|
||||
- **taos**: 使用 TDengine 连接器驱动,默认是使用 taos 驱动。
|
||||
- **tmq**: 使用 TMQ 订阅数据。
|
||||
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。
|
||||
- **http/ws**: 使用 Websocket 创建连接。
|
||||
- **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 连接。
|
||||
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。
|
||||
- **http/ws**: 使用 WebSocket 创建连接。
|
||||
- **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 连接。
|
||||
- **username/password**: 用于创建连接的用户名及密码。
|
||||
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时(`taos://`),原生连接默认为 `localhost:6030`,Websocket 连接默认为 `localhost:6041` 。
|
||||
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时(`taos://`),原生连接默认为 `localhost:6030`,WebSocket 连接默认为 `localhost:6041` 。
|
||||
- **database**: 指定默认连接的数据库名,可选参数。
|
||||
- **params**:其他可选参数。
|
||||
|
||||
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test`, 表示使用 Websocket(`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
|
||||
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test`, 表示使用 WebSocket(`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
|
||||
|
||||
#### TaosBuilder
|
||||
TaosBuilder 结构体主要提供了根据 DSN 构建 Taos 对象的方法,还提供了检查连接,以及获取客户端版本号等功能。
|
||||
|
|
|
@ -14,10 +14,10 @@ import RequestId from "./_request_id.mdx";
|
|||
Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-python)。
|
||||
|
||||
## 连接方式
|
||||
`taospy`主要提供三种形式的连接器。一般我们推荐使用 **Websocket 连接**。
|
||||
`taospy`主要提供三种形式的连接器。一般我们推荐使用 **WebSocket 连接**。
|
||||
- **原生连接**,对应 `taospy` 包的 `taos` 模块。通过 TDengine 客户端驱动程序(taosc)原生连接 TDengine 实例,支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。
|
||||
- **REST 连接**,对应 `taospy` 包的 `taosrest` 模块。通过 taosAdapter 提供的 HTTP 接口连接 TDengine 实例,不支持 schemaless 和数据订阅等特性。
|
||||
- **Websocket 连接**,对应 `taos-ws-py` 包,可以选装。通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例,WebSocket 连接实现的功能集合和原生连接有少量不同。
|
||||
- **WebSocket 连接**,对应 `taos-ws-py` 包,可以选装。通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例,WebSocket 连接实现的功能集合和原生连接有少量不同。
|
||||
|
||||
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)
|
||||
|
||||
|
@ -48,9 +48,9 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con
|
|||
|2.7.9|数据订阅支持获取消费进度和重置消费进度|
|
||||
|2.7.8|新增 `execute_many`|
|
||||
|
||||
|Python Websocket Connector 版本|主要变化|
|
||||
|Python WebSocket Connector 版本|主要变化|
|
||||
|:----------------------------:|:-----:|
|
||||
|0.3.2|优化 Websocket sql 查询和插入性能,修改 readme 和 文档,修复已知问题|
|
||||
|0.3.2|优化 WebSocket sql 查询和插入性能,修改 readme 和 文档,修复已知问题|
|
||||
|0.2.9|已知问题修复|
|
||||
|0.2.5|1. 数据订阅支持获取消费进度和重置消费进度 <br/> 2. 支持 schemaless <br/> 3. 支持 STMT|
|
||||
|0.2.4|数据订阅新增取消订阅方法|
|
||||
|
|
|
@ -14,7 +14,7 @@ Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-conne
|
|||
|
||||
## 连接方式
|
||||
|
||||
Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。
|
||||
Node.js 连接器目前仅支持 WebSocket 连接器, 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。
|
||||
|
||||
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)
|
||||
|
||||
|
@ -48,7 +48,7 @@ Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter
|
|||
| 107 | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 |
|
||||
| 108 | connection has been closed | 连接已经关闭,请检查 Connection 是否关闭后再次使用,或是连接是否正常。 |
|
||||
| 109 | fetch block data parse fail | 获取到的查询数据,解析失败 |
|
||||
| 110 | websocket connection has reached its maximum limit | Websocket 连接达到上限 |
|
||||
| 110 | websocket connection has reached its maximum limit | WebSocket 连接达到上限 |
|
||||
- [TDengine Node.js Connector Error Code](https://github.com/taosdata/taos-connector-node/blob/main/nodejs/src/common/wsError.ts)
|
||||
- TDengine 其他功能模块的报错,请参考 [错误码](../../../reference/error-code)
|
||||
|
||||
|
@ -104,7 +104,7 @@ Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter
|
|||
|
||||
## API 参考
|
||||
|
||||
Node.js 连接器(`@tdengine/websocket`), 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。
|
||||
Node.js 连接器(`@tdengine/websocket`), 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。
|
||||
|
||||
### URL 规范
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ import RequestId from "./_request_id.mdx";
|
|||
|
||||
`TDengine.Connector` 提供两种形式的连接器
|
||||
* **原生连接**,通过 TDengine 客户端驱动程序(taosc)原生连接 TDengine 实例,支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。
|
||||
* **Websocket 连接**,通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例,WebSocket 连接实现的功能集合和原生连接有少量不同。(自 v3.0.1 起)
|
||||
* **WebSocket 连接**,通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例,WebSocket 连接实现的功能集合和原生连接有少量不同。(自 v3.0.1 起)
|
||||
|
||||
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)
|
||||
|
||||
|
|
|
@ -109,7 +109,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
|
|||
|
||||
## 版本历史
|
||||
|
||||
| taos_odbc版本 | 主要变化 | TDengine 版本 |
|
||||
| taos_odbc 版本 | 主要变化 | TDengine 版本 |
|
||||
| :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- |
|
||||
| v1.1.0 | 1. 支持视图功能;<br/>2. 支持 VARBINARY/GEOMETRY 数据类型; | 3.3.3.0及更高版本 |
|
||||
| v1.0.2 | 支持 CP1252 字符编码; | 3.2.3.0及更高版本 |
|
||||
|
@ -145,7 +145,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
|
|||
|
||||
## API 参考
|
||||
|
||||
本节按功能分类汇总了 ODBC API,关于完整的 ODBC API 参考,请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的ODBC程序员参考页面。
|
||||
本节按功能分类汇总了 ODBC API,关于完整的 ODBC API 参考,请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的 ODBC 程序员参考页面。
|
||||
|
||||
### 数据源和驱动程序管理
|
||||
|
||||
|
|
|
@ -62,7 +62,7 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器
|
|||
| **连接管理** | 支持 | 支持 | 支持 |
|
||||
| **执行 SQL** | 支持 | 支持 | 支持 |
|
||||
|
||||
### 使用 Websocket 接口
|
||||
### 使用 WebSocket 接口
|
||||
|
||||
| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** |
|
||||
| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | --------- |
|
||||
|
|
|
@ -32,6 +32,8 @@ extern int32_t tsS3PageCacheSize;
|
|||
extern int32_t tsS3UploadDelaySec;
|
||||
|
||||
int32_t s3Init();
|
||||
int32_t s3Begin();
|
||||
void s3End();
|
||||
int32_t s3CheckCfg();
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object);
|
||||
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
|
||||
|
|
|
@ -42,21 +42,21 @@ typedef enum {
|
|||
} EEncryptScope;
|
||||
|
||||
// cluster
|
||||
extern char tsFirst[];
|
||||
extern char tsSecond[];
|
||||
extern char tsLocalFqdn[];
|
||||
extern char tsLocalEp[];
|
||||
extern char tsVersionName[];
|
||||
extern uint16_t tsServerPort;
|
||||
extern int32_t tsVersion;
|
||||
extern int32_t tsStatusInterval;
|
||||
extern int32_t tsNumOfSupportVnodes;
|
||||
extern char tsEncryptAlgorithm[];
|
||||
extern char tsEncryptScope[];
|
||||
extern EEncryptAlgor tsiEncryptAlgorithm;
|
||||
extern EEncryptScope tsiEncryptScope;
|
||||
//extern char tsAuthCode[];
|
||||
extern char tsEncryptKey[];
|
||||
extern char tsFirst[];
|
||||
extern char tsSecond[];
|
||||
extern char tsLocalFqdn[];
|
||||
extern char tsLocalEp[];
|
||||
extern char tsVersionName[];
|
||||
extern uint16_t tsServerPort;
|
||||
extern int32_t tsVersion;
|
||||
extern int32_t tsStatusInterval;
|
||||
extern int32_t tsNumOfSupportVnodes;
|
||||
extern char tsEncryptAlgorithm[];
|
||||
extern char tsEncryptScope[];
|
||||
extern EEncryptAlgor tsiEncryptAlgorithm;
|
||||
extern EEncryptScope tsiEncryptScope;
|
||||
// extern char tsAuthCode[];
|
||||
extern char tsEncryptKey[];
|
||||
|
||||
// common
|
||||
extern int32_t tsMaxShellConns;
|
||||
|
@ -71,6 +71,8 @@ extern int32_t tsTagFilterResCacheSize;
|
|||
// queue & threads
|
||||
extern int32_t tsNumOfRpcThreads;
|
||||
extern int32_t tsNumOfRpcSessions;
|
||||
extern int32_t tsShareConnLimit;
|
||||
extern int32_t tsReadTimeout;
|
||||
extern int32_t tsTimeToGetAvailableConn;
|
||||
extern int32_t tsKeepAliveIdle;
|
||||
extern int32_t tsNumOfCommitThreads;
|
||||
|
@ -264,8 +266,8 @@ extern bool tsExperimental;
|
|||
|
||||
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
|
||||
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
|
||||
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
|
||||
const char *envFile, char *apolloUrl, SArray *pArgs);
|
||||
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl,
|
||||
SArray *pArgs);
|
||||
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
|
||||
bool tsc);
|
||||
void taosCleanupCfg();
|
||||
|
|
|
@ -983,6 +983,7 @@ typedef struct SEpSet {
|
|||
SEp eps[TSDB_MAX_REPLICA];
|
||||
} SEpSet;
|
||||
|
||||
|
||||
int32_t tEncodeSEpSet(SEncoder* pEncoder, const SEpSet* pEp);
|
||||
int32_t tDecodeSEpSet(SDecoder* pDecoder, SEpSet* pEp);
|
||||
int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
|
||||
|
|
|
@ -329,6 +329,7 @@
|
|||
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_RELEASE, "task-release", NULL, NULL)
|
||||
TD_CLOSE_MSG_SEG(TDMT_SCH_MSG)
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_AZURE_H_
|
||||
#define _TD_AZURE_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tarray.h"
|
||||
#include "tdef.h"
|
||||
#include "tlog.h"
|
||||
#include "tmsg.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
int32_t azBegin();
|
||||
void azEnd();
|
||||
int32_t azCheckCfg();
|
||||
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
|
||||
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
|
||||
void azDeleteObjectsByPrefix(const char *prefix);
|
||||
|
||||
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp);
|
||||
int32_t azGetObjectsByPrefix(const char *prefix, const char *path);
|
||||
int32_t azGetObjectToFile(const char *object_name, const char *fileName);
|
||||
int32_t azDeleteObjects(const char *object_name[], int nobject);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // _TD_AZURE_H_
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TCS_H_
|
||||
#define _TD_TCS_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tarray.h"
|
||||
#include "tdef.h"
|
||||
#include "tlog.h"
|
||||
#include "tmsg.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
extern int8_t tsS3EnabledCfg;
|
||||
|
||||
extern int32_t tsS3UploadDelaySec;
|
||||
extern int32_t tsS3BlockSize;
|
||||
extern int32_t tsS3BlockCacheSize;
|
||||
extern int32_t tsS3PageCacheSize;
|
||||
|
||||
extern int8_t tsS3StreamEnabled;
|
||||
|
||||
int32_t tcsInit();
|
||||
void tcsUninit();
|
||||
|
||||
int32_t tcsCheckCfg();
|
||||
|
||||
int32_t tcsPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
|
||||
int32_t tcsGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
|
||||
|
||||
void tcsDeleteObjectsByPrefix(const char *prefix);
|
||||
|
||||
int32_t tcsPutObjectFromFile2(const char *file, const char *object, int8_t withcp);
|
||||
int32_t tcsGetObjectsByPrefix(const char *prefix, const char *path);
|
||||
int32_t tcsDeleteObjects(const char *object_name[], int nobject);
|
||||
int32_t tcsGetObjectToFile(const char *object_name, const char *fileName);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // _TD_TCS_H_
|
|
@ -63,6 +63,10 @@ typedef struct SRpcHandleInfo {
|
|||
int8_t forbiddenIp;
|
||||
int8_t notFreeAhandle;
|
||||
int8_t compressed;
|
||||
int64_t seqNum; // msg seq
|
||||
int64_t qId; // queryId Get from client, other req's qId = -1;
|
||||
int32_t refIdMgt;
|
||||
int32_t msgType;
|
||||
} SRpcHandleInfo;
|
||||
|
||||
typedef struct SRpcMsg {
|
||||
|
@ -124,9 +128,13 @@ typedef struct SRpcInit {
|
|||
int32_t connLimitLock;
|
||||
int32_t timeToGetConn;
|
||||
int8_t supportBatch; // 0: no batch, 1. batch
|
||||
int32_t batchSize;
|
||||
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
|
||||
void *parent;
|
||||
int32_t shareConnLimit;
|
||||
int8_t shareConn; // 0: no share, 1. share
|
||||
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
|
||||
int8_t startReadTimer;
|
||||
int64_t readTimeout; // s
|
||||
|
||||
void *parent;
|
||||
} SRpcInit;
|
||||
|
||||
typedef struct {
|
||||
|
@ -144,6 +152,7 @@ typedef struct {
|
|||
SHashObj *args;
|
||||
SRpcBrokenlinkVal brokenVal;
|
||||
void (*freeFunc)(const void *arg);
|
||||
int64_t st;
|
||||
} SRpcCtx;
|
||||
|
||||
int32_t rpcInit();
|
||||
|
|
|
@ -130,14 +130,15 @@ int taosSetAutoDelFile(char *path);
|
|||
bool lastErrorIsFileNotExist();
|
||||
|
||||
#ifdef BUILD_WITH_RAND_ERR
|
||||
#define STUB_RAND_NETWORK_ERR(status) \
|
||||
do { \
|
||||
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
|
||||
uint32_t r = taosRand() % tsRandErrDivisor; \
|
||||
if ((r + 1) <= tsRandErrChance) { \
|
||||
status = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
|
||||
} \
|
||||
} \
|
||||
#define STUB_RAND_NETWORK_ERR(ret) \
|
||||
do { \
|
||||
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
|
||||
uint32_t r = taosRand() % tsRandErrDivisor; \
|
||||
if ((r + 1) <= tsRandErrChance) { \
|
||||
ret = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
|
||||
uError("random network error: %s, %s", tstrerror(ret), __func__); \
|
||||
} \
|
||||
} \
|
||||
while (0)
|
||||
#else
|
||||
#define STUB_RAND_NETWORK_ERR(status)
|
||||
|
|
|
@ -137,6 +137,7 @@ int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
|
|||
int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
|
||||
int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
|
||||
int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
|
||||
int32_t taosSetSockOpt2(int32_t fd);
|
||||
int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
|
||||
int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
|
||||
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
|
||||
|
@ -159,7 +160,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, st
|
|||
int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
|
||||
|
||||
int32_t taosBlockSIGPIPE();
|
||||
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t* ip);
|
||||
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip);
|
||||
int32_t taosGetFqdn(char *);
|
||||
void tinet_ntoa(char *ipstr, uint32_t ip);
|
||||
uint32_t ip2uint(const char *const ip_addr);
|
||||
|
|
|
@ -94,9 +94,9 @@ int32_t taosGetErrSize();
|
|||
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
|
||||
#define TSDB_CODE_RPC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0026)
|
||||
#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027)
|
||||
|
||||
|
||||
|
||||
#define TSDB_CODE_RPC_ASYNC_IN_PROCESS TAOS_DEF_ERROR_CODE(0, 0x0028)
|
||||
#define TSDB_CODE_RPC_NO_STATE TAOS_DEF_ERROR_CODE(0, 0x0029)
|
||||
#define TSDB_CODE_RPC_STATE_DROPED TAOS_DEF_ERROR_CODE(0, 0x002A)
|
||||
|
||||
//common & util
|
||||
#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //
|
||||
|
|
|
@ -57,6 +57,7 @@ extern int32_t rpcDebugFlag;
|
|||
extern int32_t qDebugFlag;
|
||||
extern int32_t stDebugFlag;
|
||||
extern int32_t wDebugFlag;
|
||||
extern int32_t azDebugFlag;
|
||||
extern int32_t sDebugFlag;
|
||||
extern int32_t tsdbDebugFlag;
|
||||
extern int32_t tqDebugFlag;
|
||||
|
|
|
@ -370,7 +370,10 @@ int32_t openTransporter(const char *user, const char *auth, int32_t numOfThread,
|
|||
connLimitNum = TMAX(connLimitNum, 10);
|
||||
connLimitNum = TMIN(connLimitNum, 1000);
|
||||
rpcInit.connLimitNum = connLimitNum;
|
||||
rpcInit.shareConnLimit = tsShareConnLimit;
|
||||
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
|
||||
rpcInit.startReadTimer = 1;
|
||||
rpcInit.readTimeout = tsReadTimeout;
|
||||
|
||||
int32_t code = taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
|
|
|
@ -410,7 +410,6 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
|
|||
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
|
||||
SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
|
||||
|
||||
// int64_t transporterId = 0;
|
||||
int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, NULL, pSendMsg);
|
||||
if (code) {
|
||||
doRequestCallback(pRequest, code);
|
||||
|
@ -1921,19 +1920,19 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
|
|||
return NULL;
|
||||
}
|
||||
|
||||
//TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
|
||||
// const char* db, int dbLen, uint16_t port) {
|
||||
// char ipStr[TSDB_EP_LEN] = {0};
|
||||
// char dbStr[TSDB_DB_NAME_LEN] = {0};
|
||||
// char userStr[TSDB_USER_LEN] = {0};
|
||||
// char passStr[TSDB_PASSWORD_LEN] = {0};
|
||||
// TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
|
||||
// const char* db, int dbLen, uint16_t port) {
|
||||
// char ipStr[TSDB_EP_LEN] = {0};
|
||||
// char dbStr[TSDB_DB_NAME_LEN] = {0};
|
||||
// char userStr[TSDB_USER_LEN] = {0};
|
||||
// char passStr[TSDB_PASSWORD_LEN] = {0};
|
||||
//
|
||||
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
|
||||
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
|
||||
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
|
||||
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
|
||||
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
|
||||
//}
|
||||
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
|
||||
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
|
||||
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
|
||||
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
|
||||
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
|
||||
// }
|
||||
|
||||
void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
|
||||
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
|
||||
|
@ -2301,7 +2300,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
|
|||
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%.9lf", jsonVd);
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
|
||||
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
|
||||
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s",
|
||||
(*((char*)jsonInnerData) == 1) ? "true" : "false");
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else {
|
||||
tscError("doConvertJson error: invalid type:%d", jsonInnerType);
|
||||
|
@ -2570,6 +2570,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
|
|||
connLimitNum = TMIN(connLimitNum, 500);
|
||||
rpcInit.connLimitNum = connLimitNum;
|
||||
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
|
||||
rpcInit.readTimeout = tsReadTimeout;
|
||||
if (TSDB_CODE_SUCCESS != taosVersionStrToInt(version, &(rpcInit.compatibilityVer))) {
|
||||
tscError("faild to convert taos version from str to int, errcode:%s", terrstr());
|
||||
goto _OVER;
|
||||
|
|
|
@ -89,20 +89,8 @@ static void s3DumpCfgByEp(int8_t epIndex) {
|
|||
|
||||
int32_t s3CheckCfg() {
|
||||
int32_t code = 0, lino = 0;
|
||||
int8_t i = 0;
|
||||
|
||||
if (!tsS3Enabled) {
|
||||
(void)fprintf(stderr, "s3 not configured.\n");
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
code = s3Begin();
|
||||
if (code != 0) {
|
||||
(void)fprintf(stderr, "failed to initialize s3.\n");
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
for (; i < tsS3EpNum; i++) {
|
||||
for (int8_t i = 0; i < tsS3EpNum; i++) {
|
||||
(void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum);
|
||||
s3DumpCfgByEp(i);
|
||||
|
||||
|
@ -192,7 +180,7 @@ int32_t s3CheckCfg() {
|
|||
(void)fprintf(stdout, "=================================================================\n");
|
||||
}
|
||||
|
||||
s3End();
|
||||
// s3End();
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
@ -1529,6 +1517,8 @@ void s3EvictCache(const char *path, long object_size) {}
|
|||
#include "cos_http_io.h"
|
||||
#include "cos_log.h"
|
||||
|
||||
int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); }
|
||||
|
||||
int32_t s3Init() {
|
||||
if (cos_http_io_initialize(NULL, 0) != COSE_OK) {
|
||||
return -1;
|
||||
|
@ -1967,6 +1957,10 @@ long s3Size(const char *object_name) {
|
|||
#else
|
||||
|
||||
int32_t s3Init() { return 0; }
|
||||
int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); }
|
||||
|
||||
void s3End() {}
|
||||
int32_t s3CheckCfg() { return 0; }
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
|
||||
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
|
||||
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }
|
||||
|
|
|
@ -160,7 +160,11 @@ int32_t startRsync() {
|
|||
code = system(cmd);
|
||||
if (code != 0) {
|
||||
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
if (errno == 0) {
|
||||
return 0;
|
||||
} else {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
} else {
|
||||
uInfo("[rsync] cmd:%s start server successful", cmd);
|
||||
}
|
||||
|
@ -358,4 +362,4 @@ int32_t deleteRsync(const char* id) {
|
|||
|
||||
uDebug("[rsync] delete data:%s successful", id);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -56,6 +56,8 @@ int32_t tsShellActivityTimer = 3; // second
|
|||
// queue & threads
|
||||
int32_t tsNumOfRpcThreads = 1;
|
||||
int32_t tsNumOfRpcSessions = 30000;
|
||||
int32_t tsShareConnLimit = 8;
|
||||
int32_t tsReadTimeout = 900;
|
||||
int32_t tsTimeToGetAvailableConn = 500000;
|
||||
int32_t tsKeepAliveIdle = 60;
|
||||
|
||||
|
@ -179,12 +181,12 @@ int32_t tsRedirectFactor = 2;
|
|||
int32_t tsRedirectMaxPeriod = 1000;
|
||||
int32_t tsMaxRetryWaitTime = 10000;
|
||||
bool tsUseAdapter = false;
|
||||
int32_t tsMetaCacheMaxSize = -1; // MB
|
||||
int32_t tsSlowLogThreshold = 10; // seconds
|
||||
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
|
||||
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
|
||||
int32_t tsMetaCacheMaxSize = -1; // MB
|
||||
int32_t tsSlowLogThreshold = 10; // seconds
|
||||
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
|
||||
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
|
||||
int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY;
|
||||
char* tsSlowLogScopeString = "query";
|
||||
char *tsSlowLogScopeString = "query";
|
||||
int32_t tsSlowLogMaxLen = 4096;
|
||||
int32_t tsTimeSeriesThreshold = 50;
|
||||
bool tsMultiResultFunctionStarReturnTags = false;
|
||||
|
@ -306,6 +308,7 @@ char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"};
|
|||
int8_t tsS3Enabled = false;
|
||||
int8_t tsS3EnabledCfg = false;
|
||||
int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false};
|
||||
int8_t tsS3Ablob = false;
|
||||
int8_t tsS3StreamEnabled = false;
|
||||
|
||||
int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true};
|
||||
|
@ -322,7 +325,6 @@ int32_t tsMaxTsmaNum = 3;
|
|||
int32_t tsMaxTsmaCalcDelay = 600;
|
||||
int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d
|
||||
|
||||
|
||||
#define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \
|
||||
if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \
|
||||
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \
|
||||
|
@ -361,7 +363,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, name);
|
||||
|
||||
char *strDup = NULL;
|
||||
if ((strDup = taosStrdup(pItem->str))== NULL){
|
||||
if ((strDup = taosStrdup(pItem->str)) == NULL) {
|
||||
code = terrno;
|
||||
goto _exit;
|
||||
}
|
||||
|
@ -435,6 +437,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
|
|||
}
|
||||
tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL);
|
||||
tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL);
|
||||
tsS3Ablob = (strstr(tsS3Endpoint[i], ".blob.core.windows.net") != NULL);
|
||||
}
|
||||
|
||||
if (tsS3BucketName[0] != '<') {
|
||||
|
@ -450,7 +453,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
|
|||
TAOS_RETURN(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
struct SConfig *taosGetCfg() { return tsCfg; }
|
||||
struct SConfig *taosGetCfg() {
|
||||
return tsCfg;
|
||||
}
|
||||
|
||||
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
|
||||
char *apolloUrl) {
|
||||
|
@ -539,6 +544,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "azDebugFlag", azDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
|
||||
|
@ -572,7 +578,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(
|
||||
cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT));
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
|
||||
TAOS_CHECK_RETURN(
|
||||
cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
|
||||
|
@ -600,16 +607,23 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(
|
||||
cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
|
||||
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
|
||||
TAOS_CHECK_RETURN(
|
||||
cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
|
||||
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
|
||||
|
||||
tsNumOfRpcThreads = tsNumOfCores / 2;
|
||||
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
|
||||
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE));
|
||||
|
||||
tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000);
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
|
||||
|
||||
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_NONE));
|
||||
|
||||
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE));
|
||||
|
||||
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000);
|
||||
TAOS_CHECK_RETURN(
|
||||
cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
|
||||
|
@ -865,8 +879,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
|
||||
pItem = cfgGetItem(pCfg, "numOfRpcThreads");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsNumOfRpcThreads = numOfCores / 2;
|
||||
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
|
||||
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
|
||||
pItem->i32 = tsNumOfRpcThreads;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
@ -878,6 +891,20 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(pCfg, "shareConnLimit");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
|
||||
pItem->i32 = tsShareConnLimit;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(pCfg, "readTimeout");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
|
||||
pItem->i32 = tsReadTimeout;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(pCfg, "timeToGetAvailableConn");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
|
||||
|
@ -1039,6 +1066,9 @@ static int32_t taosSetServerLogCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "wDebugFlag");
|
||||
wDebugFlag = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "azDebugFlag");
|
||||
azDebugFlag = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "sDebugFlag");
|
||||
sDebugFlag = pItem->i32;
|
||||
|
||||
|
@ -1083,9 +1113,9 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope) {
|
|||
|
||||
int32_t slowScope = 0;
|
||||
|
||||
char* scope = NULL;
|
||||
char *tmp = NULL;
|
||||
while((scope = strsep(&pScopeStr, "|")) != NULL){
|
||||
char *scope = NULL;
|
||||
char *tmp = NULL;
|
||||
while ((scope = strsep(&pScopeStr, "|")) != NULL) {
|
||||
taosMemoryFreeClear(tmp);
|
||||
tmp = taosStrdup(scope);
|
||||
if (tmp == NULL) {
|
||||
|
@ -1142,13 +1172,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "firstEp");
|
||||
SEp firstEp = {0};
|
||||
SEp firstEp = {0};
|
||||
TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &firstEp));
|
||||
(void)snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "firstEp", tsFirst, pItem->stype, true));
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "secondEp");
|
||||
SEp secondEp = {0};
|
||||
SEp secondEp = {0};
|
||||
TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &secondEp));
|
||||
(void)snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
|
||||
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, true));
|
||||
|
@ -1252,6 +1282,12 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
|
||||
tsNumOfRpcSessions = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
|
||||
tsShareConnLimit = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
|
||||
tsReadTimeout = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
|
||||
tsTimeToGetAvailableConn = pItem->i32;
|
||||
|
||||
|
@ -1353,6 +1389,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
|
||||
tsNumOfRpcSessions = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
|
||||
tsShareConnLimit = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
|
||||
tsReadTimeout = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
|
||||
tsTimeToGetAvailableConn = pItem->i32;
|
||||
|
||||
|
@ -1651,8 +1693,8 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag);
|
|||
|
||||
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
|
||||
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SConfig *pCfg = NULL;
|
||||
|
||||
if (tsCfg == NULL) {
|
||||
|
@ -1724,7 +1766,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *
|
|||
|
||||
TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit);
|
||||
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit);
|
||||
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit);
|
||||
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit);
|
||||
|
||||
if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
|
||||
(void)printf("failed to load cfg since %s\n", tstrerror(code));
|
||||
|
@ -1753,7 +1795,7 @@ _exit:
|
|||
|
||||
static int32_t taosCheckGlobalCfg() {
|
||||
uint32_t ipv4 = 0;
|
||||
int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4);
|
||||
int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4);
|
||||
if (code) {
|
||||
uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code));
|
||||
TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR);
|
||||
|
@ -1858,7 +1900,7 @@ typedef struct {
|
|||
|
||||
static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) {
|
||||
int32_t code = TSDB_CODE_CFG_NOT_FOUND;
|
||||
char *name = pItem->name;
|
||||
char *name = pItem->name;
|
||||
for (int32_t d = 0; d < optionSize; ++d) {
|
||||
const char *optName = pOptions[d].optionName;
|
||||
if (strcasecmp(name, optName) != 0) continue;
|
||||
|
@ -1951,13 +1993,14 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
|
|||
|
||||
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function
|
||||
static OptionNameAndVar debugOptions[] = {
|
||||
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
|
||||
{"wDebugFlag", &wDebugFlag}, {"sDebugFlag", &sDebugFlag}, {"tsdbDebugFlag", &tsdbDebugFlag},
|
||||
{"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"udfDebugFlag", &udfDebugFlag},
|
||||
{"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag},
|
||||
{"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag},
|
||||
{"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag},
|
||||
{"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug},
|
||||
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
|
||||
{"wDebugFlag", &wDebugFlag}, {"azDebugFlag", &azDebugFlag}, {"sDebugFlag", &sDebugFlag},
|
||||
{"tsdbDebugFlag", &tsdbDebugFlag}, {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag},
|
||||
{"udfDebugFlag", &udfDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag},
|
||||
{"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
|
||||
{"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
|
||||
{"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag},
|
||||
{"tqClientDebug", &tqClientDebug},
|
||||
};
|
||||
|
||||
static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},
|
||||
|
@ -2053,8 +2096,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
|
|||
}
|
||||
case 'f': {
|
||||
if (strcasecmp("fqdn", name) == 0) {
|
||||
SConfigItem* pFqdnItem = cfgGetItem(pCfg, "fqdn");
|
||||
SConfigItem* pServerPortItem = cfgGetItem(pCfg, "serverPort");
|
||||
SConfigItem *pFqdnItem = cfgGetItem(pCfg, "fqdn");
|
||||
SConfigItem *pServerPortItem = cfgGetItem(pCfg, "serverPort");
|
||||
SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
|
||||
if (pFqdnItem == NULL || pServerPortItem == NULL || pFirstEpItem == NULL) {
|
||||
uError("failed to get fqdn or serverPort or firstEp from cfg");
|
||||
|
@ -2069,7 +2112,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
|
|||
char defaultFirstEp[TSDB_EP_LEN] = {0};
|
||||
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
|
||||
|
||||
SEp firstEp = {0};
|
||||
SEp firstEp = {0};
|
||||
TAOS_CHECK_GOTO(
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino,
|
||||
_out);
|
||||
|
@ -2109,8 +2152,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
|
|||
}
|
||||
case 'l': {
|
||||
if (strcasecmp("locale", name) == 0) {
|
||||
SConfigItem* pLocaleItem = cfgGetItem(pCfg, "locale");
|
||||
SConfigItem* pCharsetItem = cfgGetItem(pCfg, "charset");
|
||||
SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale");
|
||||
SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset");
|
||||
if (pLocaleItem == NULL || pCharsetItem == NULL) {
|
||||
uError("failed to get locale or charset from cfg");
|
||||
code = TSDB_CODE_CFG_NOT_FOUND;
|
||||
|
@ -2183,7 +2226,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
|
|||
char defaultFirstEp[TSDB_EP_LEN] = {0};
|
||||
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
|
||||
|
||||
SEp firstEp = {0};
|
||||
SEp firstEp = {0};
|
||||
TAOS_CHECK_GOTO(
|
||||
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino,
|
||||
_out);
|
||||
|
@ -2314,7 +2357,7 @@ int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg,
|
|||
// NOTE: set all command does not change the tmrDebugFlag
|
||||
static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
|
||||
if (flag < 0) TAOS_RETURN(TSDB_CODE_INVALID_PARA);
|
||||
if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore
|
||||
if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore
|
||||
|
||||
SArray *noNeedToSetVars = NULL;
|
||||
SConfigItem *pItem = NULL;
|
||||
|
@ -2335,6 +2378,7 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
|
|||
taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&azDebugFlag, "azDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars);
|
||||
taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars);
|
||||
|
|
|
@ -24,6 +24,7 @@
|
|||
#include "jemalloc/jemalloc.h"
|
||||
#endif
|
||||
#include "dmUtil.h"
|
||||
#include "tcs.h"
|
||||
|
||||
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
|
||||
#include "cus_name.h"
|
||||
|
@ -182,7 +183,6 @@ static void dmSetSignalHandle() {
|
|||
}
|
||||
#endif
|
||||
}
|
||||
extern bool generateNewMeta;
|
||||
|
||||
extern bool generateNewMeta;
|
||||
|
||||
|
@ -331,10 +331,9 @@ static int32_t dmCheckS3() {
|
|||
int32_t code = 0;
|
||||
SConfig *pCfg = taosGetCfg();
|
||||
cfgDumpCfgS3(pCfg, 0, true);
|
||||
#if defined(USE_S3)
|
||||
extern int32_t s3CheckCfg();
|
||||
|
||||
code = s3CheckCfg();
|
||||
#if defined(USE_S3)
|
||||
code = tcsCheckCfg();
|
||||
#endif
|
||||
return code;
|
||||
}
|
||||
|
|
|
@ -74,7 +74,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
|
|||
SRpcMsg rpcMsg = {.pCont = pHead,
|
||||
.contLen = contLen,
|
||||
.msgType = TDMT_MND_RETRIEVE_IP_WHITE,
|
||||
.info.ahandle = (void *)0x9527,
|
||||
.info.ahandle = 0,
|
||||
.info.notFreeAhandle = 1,
|
||||
.info.refId = 0,
|
||||
.info.noResp = 0,
|
||||
|
@ -249,7 +249,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
SRpcMsg rpcMsg = {.pCont = pHead,
|
||||
.contLen = contLen,
|
||||
.msgType = TDMT_MND_STATUS,
|
||||
.info.ahandle = (void *)0x9527,
|
||||
.info.ahandle = 0,
|
||||
.info.notFreeAhandle = 1,
|
||||
.info.refId = 0,
|
||||
.info.noResp = 0,
|
||||
|
@ -322,7 +322,7 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
|
|||
SRpcMsg rpcMsg = {.pCont = pHead,
|
||||
.contLen = contLen,
|
||||
.msgType = TDMT_MND_NOTIFY,
|
||||
.info.ahandle = (void *)0x9527,
|
||||
.info.ahandle = 0,
|
||||
.info.notFreeAhandle = 1,
|
||||
.info.refId = 0,
|
||||
.info.noResp = 1,
|
||||
|
|
|
@ -47,7 +47,6 @@ static void *dmStatusThreadFp(void *param) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
extern SMonVloadInfo tsVinfo;
|
||||
static void *dmStatusInfoThreadFp(void *param) {
|
||||
SDnodeMgmt *pMgmt = param;
|
||||
int64_t lastTime = taosGetTimestampMs();
|
||||
|
@ -73,19 +72,6 @@ static void *dmStatusInfoThreadFp(void *param) {
|
|||
}
|
||||
}
|
||||
}
|
||||
dDebug("begin to lock status info when thread exit");
|
||||
if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) {
|
||||
dError("failed to lock status info lock");
|
||||
return NULL;
|
||||
}
|
||||
if (tsVinfo.pVloads != NULL) {
|
||||
taosArrayDestroy(tsVinfo.pVloads);
|
||||
tsVinfo.pVloads = NULL;
|
||||
}
|
||||
if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) {
|
||||
dError("failed to unlock status info lock");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
aux_source_directory(src IMPLEMENT_SRC)
|
||||
add_library(dnode STATIC ${IMPLEMENT_SRC})
|
||||
target_link_libraries(
|
||||
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
|
||||
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw tcs
|
||||
)
|
||||
|
||||
IF (TD_ENTERPRISE)
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
#include "libs/function/tudf.h"
|
||||
#include "tgrant.h"
|
||||
#include "tcompare.h"
|
||||
#include "tcs.h"
|
||||
#include "tanal.h"
|
||||
// clang-format on
|
||||
|
||||
|
@ -98,9 +99,9 @@ static bool dmDataSpaceAvailable() {
|
|||
static int32_t dmCheckDiskSpace() {
|
||||
// availability
|
||||
int32_t code = 0;
|
||||
code = osUpdate();
|
||||
if(code != 0) {
|
||||
code = 0; // ignore the error, just log it
|
||||
code = osUpdate();
|
||||
if (code != 0) {
|
||||
code = 0; // ignore the error, just log it
|
||||
dError("failed to update os info since %s", tstrerror(code));
|
||||
}
|
||||
if (!dmDataSpaceAvailable()) {
|
||||
|
@ -163,13 +164,6 @@ static int32_t dmCheckDataDirVersionWrapper() {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
#if defined(USE_S3)
|
||||
|
||||
extern int32_t s3Begin();
|
||||
extern void s3End();
|
||||
extern int8_t tsS3Enabled;
|
||||
|
||||
#endif
|
||||
|
||||
int32_t dmInit() {
|
||||
dInfo("start to init dnode env");
|
||||
|
@ -187,7 +181,7 @@ int32_t dmInit() {
|
|||
if ((code = dmInitDnode(dmInstance())) != 0) return code;
|
||||
if ((code = InitRegexCache() != 0)) return code;
|
||||
#if defined(USE_S3)
|
||||
if ((code = s3Begin()) != 0) return code;
|
||||
if ((code = tcsInit()) != 0) return code;
|
||||
#endif
|
||||
|
||||
dInfo("dnode env is initialized");
|
||||
|
@ -221,7 +215,7 @@ void dmCleanup() {
|
|||
DestroyRegexCache();
|
||||
|
||||
#if defined(USE_S3)
|
||||
s3End();
|
||||
tcsUninit();
|
||||
#endif
|
||||
|
||||
dInfo("dnode env is cleaned up");
|
||||
|
|
|
@ -219,6 +219,7 @@ int32_t dmInitVars(SDnode *pDnode) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
extern SMonVloadInfo tsVinfo;
|
||||
void dmClearVars(SDnode *pDnode) {
|
||||
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
|
||||
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
|
||||
|
@ -254,6 +255,25 @@ void dmClearVars(SDnode *pDnode) {
|
|||
(void)taosThreadRwlockUnlock(&pData->lock);
|
||||
|
||||
(void)taosThreadRwlockDestroy(&pData->lock);
|
||||
|
||||
dDebug("begin to lock status info when thread exit");
|
||||
if (taosThreadMutexLock(&pData->statusInfolock) != 0) {
|
||||
dError("failed to lock status info lock");
|
||||
return;
|
||||
}
|
||||
if (tsVinfo.pVloads != NULL) {
|
||||
taosArrayDestroy(tsVinfo.pVloads);
|
||||
tsVinfo.pVloads = NULL;
|
||||
}
|
||||
if (taosThreadMutexUnlock(&pData->statusInfolock) != 0) {
|
||||
dError("failed to unlock status info lock");
|
||||
return;
|
||||
}
|
||||
if (taosThreadMutexDestroy(&pData->statusInfolock) != 0) {
|
||||
dError("failed to destroy status info lock");
|
||||
}
|
||||
memset(&pData->statusInfolock, 0, sizeof(pData->statusInfolock));
|
||||
|
||||
(void)taosThreadMutexDestroy(&pDnode->mutex);
|
||||
memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
|
||||
}
|
||||
|
|
|
@ -16,8 +16,8 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "dmMgmt.h"
|
||||
#include "qworker.h"
|
||||
#include "tversion.h"
|
||||
#include "tanal.h"
|
||||
#include "tversion.h"
|
||||
|
||||
static inline void dmSendRsp(SRpcMsg *pMsg) {
|
||||
if (rpcSendResponse(pMsg) != 0) {
|
||||
|
@ -267,7 +267,7 @@ _OVER:
|
|||
}
|
||||
|
||||
if (IsReq(pRpc)) {
|
||||
SRpcMsg rsp = {.code = code, .info = pRpc->info};
|
||||
SRpcMsg rsp = {.code = code, .info = pRpc->info, .msgType = pRpc->msgType + 1};
|
||||
if (code == TSDB_CODE_MNODE_NOT_FOUND) {
|
||||
dmBuildMnodeRedirectRsp(pDnode, &rsp);
|
||||
}
|
||||
|
@ -411,16 +411,19 @@ int32_t dmInitClient(SDnode *pDnode) {
|
|||
|
||||
rpcInit.noDelayFp = rpcNoDelayMsg;
|
||||
|
||||
int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3) / 2;
|
||||
int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3);
|
||||
connLimitNum = TMAX(connLimitNum, 10);
|
||||
connLimitNum = TMIN(connLimitNum, 500);
|
||||
|
||||
rpcInit.connLimitNum = connLimitNum;
|
||||
rpcInit.connLimitLock = 1;
|
||||
rpcInit.supportBatch = 1;
|
||||
rpcInit.batchSize = 8 * 1024;
|
||||
rpcInit.shareConnLimit = tsShareConnLimit * 2;
|
||||
rpcInit.shareConn = 1;
|
||||
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
|
||||
rpcInit.notWaitAvaliableConn = 0;
|
||||
rpcInit.startReadTimer = 1;
|
||||
rpcInit.readTimeout = tsReadTimeout;
|
||||
|
||||
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
|
||||
dError("failed to convert version string:%s to int", version);
|
||||
|
@ -466,8 +469,10 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
|
|||
rpcInit.connLimitNum = connLimitNum;
|
||||
rpcInit.connLimitLock = 1;
|
||||
rpcInit.supportBatch = 1;
|
||||
rpcInit.batchSize = 8 * 1024;
|
||||
rpcInit.shareConnLimit = tsShareConnLimit * 2;
|
||||
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
|
||||
rpcInit.startReadTimer = 0;
|
||||
rpcInit.readTimeout = 0;
|
||||
|
||||
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
|
||||
dError("failed to convert version string:%s to int", version);
|
||||
|
@ -514,8 +519,11 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
|
|||
rpcInit.connLimitNum = connLimitNum;
|
||||
rpcInit.connLimitLock = 1;
|
||||
rpcInit.supportBatch = 1;
|
||||
rpcInit.batchSize = 8 * 1024;
|
||||
rpcInit.shareConnLimit = tsShareConnLimit * 8;
|
||||
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
|
||||
rpcInit.startReadTimer = 1;
|
||||
rpcInit.readTimeout = tsReadTimeout;
|
||||
|
||||
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
|
||||
dError("failed to convert version string:%s to int", version);
|
||||
}
|
||||
|
@ -569,6 +577,7 @@ int32_t dmInitServer(SDnode *pDnode) {
|
|||
rpcInit.idleTime = tsShellActivityTimer * 1000;
|
||||
rpcInit.parent = pDnode;
|
||||
rpcInit.compressSize = tsCompressMsgSize;
|
||||
rpcInit.shareConnLimit = tsShareConnLimit * 16;
|
||||
|
||||
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
|
||||
dError("failed to convert version string:%s to int", version);
|
||||
|
|
|
@ -237,7 +237,7 @@ static void mndPullupGrant(SMnode *pMnode) {
|
|||
.pCont = pReq,
|
||||
.contLen = contLen,
|
||||
.info.notFreeAhandle = 1,
|
||||
.info.ahandle = (void *)0x9527};
|
||||
.info.ahandle = 0};
|
||||
// TODO check return value
|
||||
if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) {
|
||||
mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__);
|
||||
|
|
|
@ -30,9 +30,9 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
|
|||
(void)qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg);
|
||||
}
|
||||
|
||||
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
|
||||
int32_t code = -1;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo *pInfo) {
|
||||
int32_t code = -1;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
|
||||
SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb, .pWorkerCb = pInfo->workerCb};
|
||||
|
||||
|
@ -67,26 +67,25 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void* p) {
|
||||
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void *p) {
|
||||
if (NULL == p) {
|
||||
return;
|
||||
}
|
||||
|
||||
SBatchRspMsg* pRsp = (SBatchRspMsg*)p;
|
||||
SBatchRspMsg *pRsp = (SBatchRspMsg *)p;
|
||||
rpcFreeCont(pRsp->msg);
|
||||
}
|
||||
|
||||
int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
|
||||
int32_t code = 0;
|
||||
int32_t rspSize = 0;
|
||||
SBatchReq batchReq = {0};
|
||||
SBatchMsg req = {0};
|
||||
int32_t code = 0;
|
||||
int32_t rspSize = 0;
|
||||
SBatchReq batchReq = {0};
|
||||
SBatchMsg req = {0};
|
||||
SBatchRspMsg rsp = {0};
|
||||
SBatchRsp batchRsp = {0};
|
||||
SRpcMsg reqMsg = *pMsg;
|
||||
void *pRsp = NULL;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
SBatchRsp batchRsp = {0};
|
||||
SRpcMsg reqMsg = *pMsg;
|
||||
void *pRsp = NULL;
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
|
||||
if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
@ -94,7 +93,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
|
|||
goto _exit;
|
||||
}
|
||||
|
||||
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
|
||||
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
|
||||
if (msgNum >= MAX_META_MSG_IN_BATCH) {
|
||||
code = TSDB_CODE_INVALID_MSG;
|
||||
mError("too many msgs %d in mnode batch meta req", msgNum);
|
||||
|
@ -108,7 +107,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < msgNum; ++i) {
|
||||
SBatchMsg* req = taosArrayGet(batchReq.pMsgs, i);
|
||||
SBatchMsg *req = taosArrayGet(batchReq.pMsgs, i);
|
||||
|
||||
reqMsg.msgType = req->msgType;
|
||||
reqMsg.pCont = req->msg;
|
||||
|
|
|
@ -4,7 +4,7 @@ aux_source_directory(. MNODE_ARBGROUP_TEST_SRC)
|
|||
add_executable(arbgroupTest ${MNODE_ARBGROUP_TEST_SRC})
|
||||
target_link_libraries(
|
||||
arbgroupTest
|
||||
PRIVATE dnode nodes planner gtest qcom
|
||||
PRIVATE dnode nodes planner gtest qcom tcs
|
||||
)
|
||||
|
||||
add_test(
|
||||
|
|
|
@ -119,6 +119,7 @@ if (${BUILD_CONTRIB})
|
|||
vnode
|
||||
PUBLIC "inc"
|
||||
PUBLIC "src/inc"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
|
||||
|
@ -129,6 +130,7 @@ else()
|
|||
vnode
|
||||
PUBLIC "inc"
|
||||
PUBLIC "src/inc"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
|
||||
|
@ -164,6 +166,7 @@ target_link_libraries(
|
|||
PUBLIC tdb
|
||||
PUBLIC audit
|
||||
PUBLIC crypt
|
||||
PUBLIC tcs
|
||||
|
||||
# PUBLIC bdb
|
||||
# PUBLIC scalar
|
||||
|
|
|
@ -12,8 +12,8 @@
|
|||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include "cos.h"
|
||||
#include "functionMgt.h"
|
||||
#include "tcs.h"
|
||||
#include "tsdb.h"
|
||||
#include "tsdbDataFileRW.h"
|
||||
#include "tsdbIter.h"
|
||||
|
@ -1258,7 +1258,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
|
|||
}
|
||||
|
||||
if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) {
|
||||
SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID};
|
||||
SLastCol lastColTmp = {
|
||||
.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID};
|
||||
if ((code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, &lastColTmp)) != TSDB_CODE_SUCCESS) {
|
||||
tsdbError("tsdb/cache: vgId:%d, put rocks failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino,
|
||||
tstrerror(code));
|
||||
|
@ -1705,8 +1706,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA
|
|||
if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) {
|
||||
code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0);
|
||||
if (code) {
|
||||
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
|
||||
tstrerror(code));
|
||||
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code));
|
||||
taosMemoryFreeClear(pToFree);
|
||||
TAOS_CHECK_EXIT(code);
|
||||
}
|
||||
|
@ -3520,7 +3520,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
|
|||
|
||||
int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
|
||||
|
||||
TAOS_CHECK_RETURN(s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock));
|
||||
TAOS_CHECK_RETURN(tcsGetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock));
|
||||
|
||||
tsdbTrace("block:%p load from s3", *ppBlock);
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@
|
|||
*/
|
||||
|
||||
#include "tsdbFile2.h"
|
||||
#include "cos.h"
|
||||
#include "tcs.h"
|
||||
#include "vnd.h"
|
||||
|
||||
// to_json
|
||||
|
@ -318,7 +318,7 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
|
|||
}
|
||||
*(dot + 1) = 0;
|
||||
|
||||
s3DeleteObjectsByPrefix(object_name_prefix);
|
||||
tcsDeleteObjectsByPrefix(object_name_prefix);
|
||||
|
||||
// remove local last chunk file
|
||||
dot = strrchr(lc_path, '.');
|
||||
|
|
|
@ -13,8 +13,8 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "cos.h"
|
||||
#include "crypt.h"
|
||||
#include "tcs.h"
|
||||
#include "tsdb.h"
|
||||
#include "tsdbDef.h"
|
||||
#include "vnd.h"
|
||||
|
@ -391,7 +391,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
|
|||
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
|
||||
|
||||
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
|
||||
code = tcsGetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
memcpy(buf + n, pBlock, nRead);
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "cos.h"
|
||||
#include "tcs.h"
|
||||
#include "tsdb.h"
|
||||
#include "tsdbFS2.h"
|
||||
#include "vnd.h"
|
||||
|
@ -426,35 +426,6 @@ static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3Kee
|
|||
}
|
||||
}
|
||||
|
||||
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
TdFilePtr fdFrom = NULL;
|
||||
// TdFilePtr fdTo = NULL;
|
||||
|
||||
tsdbTFileName(rtner->tsdb, to, fname);
|
||||
|
||||
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) {
|
||||
TAOS_CHECK_GOTO(terrno, &lino, _exit);
|
||||
}
|
||||
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
TAOS_CHECK_GOTO(s3PutObjectFromFile2(from->fname, object_name, 1), &lino, _exit);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
|
||||
tstrerror(code));
|
||||
}
|
||||
if (taosCloseFile(&fdFrom) != 0) {
|
||||
tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
@ -519,7 +490,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
|
|||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
|
||||
int64_t c_offset = chunksize * (cn - fobj->f->lcn);
|
||||
|
||||
TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
|
||||
}
|
||||
|
||||
// copy last chunk
|
||||
|
@ -618,7 +589,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
|
|||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
|
||||
int64_t c_offset = chunksize * (cn - 1);
|
||||
|
||||
TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
|
||||
TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
|
||||
}
|
||||
|
||||
// copy last chunk
|
||||
|
@ -741,8 +712,6 @@ _exit:
|
|||
int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
|
||||
int32_t code = 0;
|
||||
|
||||
extern int8_t tsS3EnabledCfg;
|
||||
|
||||
int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE);
|
||||
if (expired && tsS3Enabled) {
|
||||
tsdbWarn("s3 grant expired: %d", expired);
|
||||
|
|
|
@ -13,8 +13,8 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "cos.h"
|
||||
#include "sync.h"
|
||||
#include "tcs.h"
|
||||
#include "tsdb.h"
|
||||
#include "vnd.h"
|
||||
|
||||
|
@ -327,7 +327,7 @@ void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) {
|
|||
if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) {
|
||||
char vnode_prefix[TSDB_FILENAME_LEN];
|
||||
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId);
|
||||
s3DeleteObjectsByPrefix(vnode_prefix);
|
||||
tcsDeleteObjectsByPrefix(vnode_prefix);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -22,4 +22,6 @@ add_subdirectory(stream)
|
|||
add_subdirectory(planner)
|
||||
add_subdirectory(qworker)
|
||||
add_subdirectory(geometry)
|
||||
add_subdirectory(command)
|
||||
add_subdirectory(command)
|
||||
add_subdirectory(azure)
|
||||
add_subdirectory(tcs)
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
#if(${TD_LINUX})
|
||||
aux_source_directory(src AZ_SRC)
|
||||
|
||||
add_library(az STATIC ${AZ_SRC})
|
||||
|
||||
if(${BUILD_S3})
|
||||
add_definitions(-DUSE_S3)
|
||||
target_link_libraries(
|
||||
az
|
||||
PUBLIC _azure_sdk
|
||||
PUBLIC crypt
|
||||
)
|
||||
endif()
|
||||
|
||||
target_include_directories(
|
||||
az
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/azure"
|
||||
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
az
|
||||
PUBLIC cjson
|
||||
PUBLIC os
|
||||
PUBLIC util
|
||||
PUBLIC common
|
||||
)
|
||||
|
||||
if(${BUILD_TEST})
|
||||
add_subdirectory(test)
|
||||
endif(${BUILD_TEST})
|
||||
#endif(${TD_LINUX})
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_AZ_INT_H_
|
||||
#define _TD_AZ_INT_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tarray.h"
|
||||
#include "tdef.h"
|
||||
#include "tlog.h"
|
||||
#include "tmsg.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// clang-format off
|
||||
#define azFatal(...) { if (azDebugFlag & DEBUG_FATAL) { taosPrintLog("AZR FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
|
||||
#define azError(...) { if (azDebugFlag & DEBUG_ERROR) { taosPrintLog("AZR ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
|
||||
#define azWarn(...) { if (azDebugFlag & DEBUG_WARN) { taosPrintLog("AZR WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
|
||||
#define azInfo(...) { if (azDebugFlag & DEBUG_INFO) { taosPrintLog("AZR ", DEBUG_INFO, 255, __VA_ARGS__); }}
|
||||
#define azDebug(...) { if (azDebugFlag & DEBUG_DEBUG) { taosPrintLog("AZR ", DEBUG_DEBUG, azDebugFlag, __VA_ARGS__); }}
|
||||
#define azTrace(...) { if (azDebugFlag & DEBUG_TRACE) { taosPrintLog("AZR ", DEBUG_TRACE, azDebugFlag, __VA_ARGS__); }}
|
||||
// clang-format on
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // _TD_AZ_INT_H_
|
|
@ -0,0 +1,191 @@
|
|||
#pragma once
|
||||
|
||||
#include "azure/storage/blobs/blob_options.hpp"
|
||||
|
||||
#include <azure/core/io/body_stream.hpp>
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
|
||||
namespace Azure {
|
||||
namespace Storage {
|
||||
namespace Blobs {
|
||||
namespace _detail {
|
||||
enum class AvroDatumType {
|
||||
String,
|
||||
Bytes,
|
||||
Int,
|
||||
Long,
|
||||
Float,
|
||||
Double,
|
||||
Bool,
|
||||
Null,
|
||||
Record,
|
||||
Enum,
|
||||
Array,
|
||||
Map,
|
||||
Union,
|
||||
Fixed,
|
||||
};
|
||||
|
||||
class AvroStreamReader final {
|
||||
public:
|
||||
// position of a vector that lives through vector resizing
|
||||
struct ReaderPos final {
|
||||
const std::vector<uint8_t>* BufferPtr = nullptr;
|
||||
size_t Offset = 0;
|
||||
};
|
||||
explicit AvroStreamReader(Core::IO::BodyStream& stream) : m_stream(&stream), m_pos{&m_streambuffer, 0} {}
|
||||
AvroStreamReader(const AvroStreamReader&) = delete;
|
||||
AvroStreamReader& operator=(const AvroStreamReader&) = delete;
|
||||
|
||||
int64_t ParseInt(const Core::Context& context);
|
||||
void Advance(size_t n, const Core::Context& context);
|
||||
// Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes
|
||||
// available in m_streambuffer;
|
||||
size_t Preload(size_t n, const Core::Context& context);
|
||||
size_t TryPreload(size_t n, const Core::Context& context);
|
||||
// discards data that's before m_pos
|
||||
void Discard();
|
||||
|
||||
private:
|
||||
size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; }
|
||||
|
||||
private:
|
||||
Core::IO::BodyStream* m_stream;
|
||||
std::vector<uint8_t> m_streambuffer;
|
||||
ReaderPos m_pos;
|
||||
|
||||
friend class AvroDatum;
|
||||
};
|
||||
|
||||
class AvroSchema final {
|
||||
public:
|
||||
static const AvroSchema StringSchema;
|
||||
static const AvroSchema BytesSchema;
|
||||
static const AvroSchema IntSchema;
|
||||
static const AvroSchema LongSchema;
|
||||
static const AvroSchema FloatSchema;
|
||||
static const AvroSchema DoubleSchema;
|
||||
static const AvroSchema BoolSchema;
|
||||
static const AvroSchema NullSchema;
|
||||
static AvroSchema RecordSchema(std::string name, const std::vector<std::pair<std::string, AvroSchema>>& fieldsSchema);
|
||||
static AvroSchema ArraySchema(AvroSchema elementSchema);
|
||||
static AvroSchema MapSchema(AvroSchema elementSchema);
|
||||
static AvroSchema UnionSchema(std::vector<AvroSchema> schemas);
|
||||
static AvroSchema FixedSchema(std::string name, int64_t size);
|
||||
|
||||
const std::string& Name() const { return m_name; }
|
||||
AvroDatumType Type() const { return m_type; }
|
||||
const std::vector<std::string>& FieldNames() const { return m_status->m_keys; }
|
||||
AvroSchema ItemSchema() const { return m_status->m_schemas[0]; }
|
||||
const std::vector<AvroSchema>& FieldSchemas() const { return m_status->m_schemas; }
|
||||
size_t Size() const { return static_cast<size_t>(m_status->m_size); }
|
||||
|
||||
private:
|
||||
explicit AvroSchema(AvroDatumType type) : m_type(type) {}
|
||||
|
||||
private:
|
||||
AvroDatumType m_type;
|
||||
std::string m_name;
|
||||
|
||||
struct SharedStatus {
|
||||
std::vector<std::string> m_keys;
|
||||
std::vector<AvroSchema> m_schemas;
|
||||
int64_t m_size = 0;
|
||||
};
|
||||
std::shared_ptr<SharedStatus> m_status;
|
||||
};
|
||||
|
||||
class AvroDatum final {
|
||||
public:
|
||||
AvroDatum() : m_schema(AvroSchema::NullSchema) {}
|
||||
explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {}
|
||||
|
||||
void Fill(AvroStreamReader& reader, const Core::Context& context);
|
||||
void Fill(AvroStreamReader::ReaderPos& data);
|
||||
|
||||
const AvroSchema& Schema() const { return m_schema; }
|
||||
|
||||
template <class T>
|
||||
T Value() const;
|
||||
struct StringView {
|
||||
const uint8_t* Data = nullptr;
|
||||
size_t Length = 0;
|
||||
};
|
||||
|
||||
private:
|
||||
AvroSchema m_schema;
|
||||
AvroStreamReader::ReaderPos m_data;
|
||||
};
|
||||
|
||||
using AvroMap = std::map<std::string, AvroDatum>;
|
||||
|
||||
class AvroRecord final {
|
||||
public:
|
||||
bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); }
|
||||
const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); }
|
||||
AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); }
|
||||
const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); }
|
||||
AvroDatum& FieldAt(size_t i) { return m_values.at(i); }
|
||||
|
||||
private:
|
||||
size_t FindField(const std::string& key) const {
|
||||
auto i = find(m_keys->begin(), m_keys->end(), key);
|
||||
return i - m_keys->begin();
|
||||
}
|
||||
const std::vector<std::string>* m_keys = nullptr;
|
||||
std::vector<AvroDatum> m_values;
|
||||
|
||||
friend class AvroDatum;
|
||||
};
|
||||
|
||||
class AvroObjectContainerReader final {
|
||||
public:
|
||||
explicit AvroObjectContainerReader(Core::IO::BodyStream& stream);
|
||||
|
||||
bool End() const { return m_eof; }
|
||||
// Calling Next() will invalidates the previous AvroDatum returned by this function and all
|
||||
// AvroDatums propagated from there.
|
||||
AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); }
|
||||
|
||||
private:
|
||||
AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context);
|
||||
|
||||
private:
|
||||
std::unique_ptr<AvroStreamReader> m_reader;
|
||||
std::unique_ptr<AvroSchema> m_objectSchema;
|
||||
std::string m_syncMarker;
|
||||
int64_t m_remainingObjectInCurrentBlock = 0;
|
||||
bool m_eof = false;
|
||||
};
|
||||
|
||||
class AvroStreamParser final : public Core::IO::BodyStream {
|
||||
public:
|
||||
explicit AvroStreamParser(std::unique_ptr<Azure::Core::IO::BodyStream> inner,
|
||||
std::function<void(int64_t, int64_t)> progressCallback,
|
||||
std::function<void(BlobQueryError)> errorCallback)
|
||||
: m_inner(std::move(inner)),
|
||||
m_parser(*m_inner),
|
||||
m_progressCallback(std::move(progressCallback)),
|
||||
m_errorCallback(std::move(errorCallback)) {}
|
||||
|
||||
int64_t Length() const override { return -1; }
|
||||
void Rewind() override { this->m_inner->Rewind(); }
|
||||
|
||||
private:
|
||||
size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<Azure::Core::IO::BodyStream> m_inner;
|
||||
AvroObjectContainerReader m_parser;
|
||||
std::function<void(int64_t, int64_t)> m_progressCallback;
|
||||
std::function<void(BlobQueryError)> m_errorCallback;
|
||||
AvroDatum::StringView m_parserBuffer;
|
||||
};
|
||||
|
||||
} // namespace _detail
|
||||
} // namespace Blobs
|
||||
} // namespace Storage
|
||||
} // namespace Azure
|
|
@ -0,0 +1,260 @@
|
|||
#pragma once
|
||||
|
||||
#include "azure/storage/blobs/blob_client.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace Azure {
|
||||
namespace Storage {
|
||||
namespace Files {
|
||||
namespace DataLake {
|
||||
class FileClient;
|
||||
}
|
||||
} // namespace Files
|
||||
} // namespace Storage
|
||||
} // namespace Azure
|
||||
|
||||
namespace Azure {
|
||||
namespace Storage {
|
||||
namespace Blobs {
|
||||
|
||||
/**
|
||||
* @brief The TDBlockBlobClient allows you to manipulate Azure Storage block blobs.
|
||||
*
|
||||
* Block blobs let you upload large blobs efficiently. Block blobs are comprised of blocks, each
|
||||
* of which is identified by a block ID. You create or modify a block blob by writing a set of
|
||||
* blocks and committing them by their block IDs. Each block can be a different size.
|
||||
*
|
||||
* When you upload a block to a blob in your storage account, it is associated with the specified
|
||||
* block blob, but it does not become part of the blob until you commit a list of blocks that
|
||||
* includes the new block's ID. New blocks remain in an uncommitted state until they are
|
||||
* specifically committed or discarded. Writing a block does not update the last modified time of
|
||||
* an existing blob.
|
||||
*/
|
||||
class TDBlockBlobClient final : public BlobClient {
|
||||
public:
|
||||
/**
|
||||
* @brief Initialize a new instance of TDBlockBlobClient.
|
||||
*
|
||||
* @param connectionString A connection string includes the authentication information required
|
||||
* for your application to access data in an Azure Storage account at runtime.
|
||||
* @param blobContainerName The name of the container containing this blob.
|
||||
* @param blobName The name of this blob.
|
||||
* @param options Optional client options that define the transport pipeline policies for
|
||||
* authentication, retries, etc., that are applied to every request.
|
||||
* @return A new TDBlockBlobClient instance.
|
||||
*/
|
||||
static TDBlockBlobClient CreateFromConnectionString(const std::string& connectionString,
|
||||
const std::string& blobContainerName, const std::string& blobName,
|
||||
const BlobClientOptions& options = BlobClientOptions());
|
||||
|
||||
/**
|
||||
* @brief Initialize a new instance of TDBlockBlobClient.
|
||||
*
|
||||
* @param blobUrl A URL
|
||||
* referencing the blob that includes the name of the account, the name of the container, and
|
||||
* the name of the blob.
|
||||
* @param credential The shared key credential used to sign
|
||||
* requests.
|
||||
* @param options Optional client options that define the transport pipeline
|
||||
* policies for authentication, retries, etc., that are applied to every request.
|
||||
*/
|
||||
explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<StorageSharedKeyCredential> credential,
|
||||
const BlobClientOptions& options = BlobClientOptions());
|
||||
|
||||
/**
|
||||
* @brief Initialize a new instance of TDBlockBlobClient.
|
||||
*
|
||||
* @param blobUrl A URL
|
||||
* referencing the blob that includes the name of the account, the name of the container, and
|
||||
* the name of the blob.
|
||||
* @param credential The token credential used to sign requests.
|
||||
* @param options Optional client options that define the transport pipeline policies for
|
||||
* authentication, retries, etc., that are applied to every request.
|
||||
*/
|
||||
explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<Core::Credentials::TokenCredential> credential,
|
||||
const BlobClientOptions& options = BlobClientOptions());
|
||||
|
||||
/**
|
||||
* @brief Initialize a new instance of TDBlockBlobClient.
|
||||
*
|
||||
* @param blobUrl A URL
|
||||
* referencing the blob that includes the name of the account, the name of the container, and
|
||||
* the name of the blob, and possibly also a SAS token.
|
||||
* @param options Optional client
|
||||
* options that define the transport pipeline policies for authentication, retries, etc., that
|
||||
* are applied to every request.
|
||||
*/
|
||||
explicit TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options = BlobClientOptions());
|
||||
|
||||
/**
|
||||
* @brief Initializes a new instance of the TDBlockBlobClient class with an identical URL
|
||||
* source but the specified snapshot timestamp.
|
||||
*
|
||||
* @param snapshot The snapshot
|
||||
* identifier.
|
||||
* @return A new TDBlockBlobClient instance.
|
||||
* @remarks Pass empty string to remove the snapshot returning the base blob.
|
||||
*/
|
||||
TDBlockBlobClient WithSnapshot(const std::string& snapshot) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a clone of this instance that references a version ID rather than the base
|
||||
* blob.
|
||||
*
|
||||
* @param versionId The version ID returning a URL to the base blob.
|
||||
* @return A new TDBlockBlobClient instance.
|
||||
* @remarks Pass empty string to remove the version ID returning the base blob.
|
||||
*/
|
||||
TDBlockBlobClient WithVersionId(const std::string& versionId) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
|
||||
* an existing block blob overwrites any existing metadata on the blob.
|
||||
*
|
||||
* @param content A BodyStream containing the content to upload.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A UploadBlockBlobResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::UploadBlockBlobResult> Upload(
|
||||
Azure::Core::IO::BodyStream& content, const UploadBlockBlobOptions& options = UploadBlockBlobOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
|
||||
* an existing block blob overwrites any existing metadata on the blob.
|
||||
*
|
||||
* @param buffer A memory buffer containing the content to upload.
|
||||
* @param bufferSize Size of the memory buffer.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A UploadBlockBlobFromResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
|
||||
const uint8_t* buffer, size_t bufferSize,
|
||||
const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
|
||||
* an existing block blob overwrites any existing metadata on the blob.
|
||||
*
|
||||
* @param fileName A file containing the content to upload.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A UploadBlockBlobFromResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
|
||||
const std::string& fileName, const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
|
||||
const std::string& fileName, int64_t offset, int64_t size,
|
||||
const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new Block Blob where the contents of the blob are read from a given URL.
|
||||
*
|
||||
* @param sourceUri Specifies the URL of the source blob.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A UploadBlockBlobFromUriResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::UploadBlockBlobFromUriResult> UploadFromUri(
|
||||
const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options = UploadBlockBlobFromUriOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new block as part of a block blob's staging area to be eventually
|
||||
* committed via the CommitBlockList operation.
|
||||
*
|
||||
* @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the
|
||||
* string must be less than or equal to 64 bytes in size.
|
||||
* @param content A BodyStream containing the content to upload.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A StageBlockResult describing the state of the updated block.
|
||||
*/
|
||||
Azure::Response<Models::StageBlockResult> StageBlock(
|
||||
const std::string& blockId, Azure::Core::IO::BodyStream& content,
|
||||
const StageBlockOptions& options = StageBlockOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Creates a new block to be committed as part of a blob where the contents are read from
|
||||
* the sourceUri.
|
||||
*
|
||||
* @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the
|
||||
* string must be less than or equal to 64 bytes in size.
|
||||
* @param sourceUri Specifies the uri of the source
|
||||
* blob. The value may be a uri of up to 2 KB in length that specifies a blob. The source blob
|
||||
* must either be public or must be authenticated via a shared access signature. If the source
|
||||
* blob is public, no authentication is required to perform the operation.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A StageBlockFromUriResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::StageBlockFromUriResult> StageBlockFromUri(
|
||||
const std::string& blockId, const std::string& sourceUri,
|
||||
const StageBlockFromUriOptions& options = StageBlockFromUriOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Writes a blob by specifying the list of block IDs that make up the blob. In order to
|
||||
* be written as part of a blob, a block must have been successfully written to the server in a
|
||||
* prior StageBlock operation. You can call CommitBlockList to update a blob by uploading only
|
||||
* those blocks that have changed, then committing the new and existing blocks together. You can
|
||||
* do this by specifying whether to commit a block from the committed block list or from the
|
||||
* uncommitted block list, or to commit the most recently uploaded version of the block,
|
||||
* whichever list it may belong to.
|
||||
*
|
||||
* @param blockIds Base64 encoded block IDs to indicate that make up the blob.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A CommitBlobBlockListResult describing the state of the updated block blob.
|
||||
*/
|
||||
Azure::Response<Models::CommitBlockListResult> CommitBlockList(
|
||||
const std::vector<std::string>& blockIds, const CommitBlockListOptions& options = CommitBlockListOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Retrieves the list of blocks that have been uploaded as part of a block blob. There
|
||||
* are two block lists maintained for a blob. The Committed Block list has blocks that have been
|
||||
* successfully committed to a given blob with CommitBlockList. The Uncommitted Block list has
|
||||
* blocks that have been uploaded for a blob using StageBlock, but that have not yet been
|
||||
* committed.
|
||||
*
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A GetBlobBlockListResult describing requested block list.
|
||||
*/
|
||||
Azure::Response<Models::GetBlockListResult> GetBlockList(
|
||||
const GetBlockListOptions& options = GetBlockListOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
/**
|
||||
* @brief Returns the result of a query against the blob.
|
||||
*
|
||||
* @param querySqlExpression The query expression in SQL.
|
||||
* @param options Optional parameters to execute this function.
|
||||
* @param context Context for cancelling long running operations.
|
||||
* @return A QueryBlobResult describing the query result.
|
||||
*/
|
||||
Azure::Response<Models::QueryBlobResult> Query(const std::string& querySqlExpression,
|
||||
const QueryBlobOptions& options = QueryBlobOptions(),
|
||||
const Azure::Core::Context& context = Azure::Core::Context()) const;
|
||||
|
||||
explicit TDBlockBlobClient(BlobClient blobClient);
|
||||
|
||||
private:
|
||||
friend class BlobClient;
|
||||
friend class Files::DataLake::DataLakeFileClient;
|
||||
};
|
||||
|
||||
} // namespace Blobs
|
||||
} // namespace Storage
|
||||
} // namespace Azure
|
|
@ -0,0 +1,534 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#define ALLOW_FORBID_FUNC
|
||||
|
||||
#include "az.h"
|
||||
#include "azInt.h"
|
||||
|
||||
#include "os.h"
|
||||
#include "taoserror.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
#if defined(USE_S3)
|
||||
|
||||
#include <azure/core.hpp>
|
||||
#include <azure/storage/blobs.hpp>
|
||||
#include "td_block_blob_client.hpp"
|
||||
|
||||
// Add appropriate using namespace directives
|
||||
using namespace Azure::Storage;
|
||||
using namespace Azure::Storage::Blobs;
|
||||
|
||||
extern char tsS3Hostname[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
|
||||
extern char tsS3BucketName[TSDB_FQDN_LEN];
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
extern int8_t tsS3EpNum;
|
||||
|
||||
int32_t azBegin() { return TSDB_CODE_SUCCESS; }
|
||||
|
||||
void azEnd() {}
|
||||
|
||||
static void checkPrint(const char *fmt, ...) {
|
||||
va_list arg_ptr;
|
||||
va_start(arg_ptr, fmt);
|
||||
(void)vfprintf(stderr, fmt, arg_ptr);
|
||||
va_end(arg_ptr);
|
||||
}
|
||||
|
||||
static void azDumpCfgByEp(int8_t epIndex) {
|
||||
// clang-format off
|
||||
checkPrint(
|
||||
"%-24s %s\n"
|
||||
"%-24s %s\n"
|
||||
"%-24s %s\n"
|
||||
"%-24s %s\n"
|
||||
"%-24s %s\n"
|
||||
"%-24s %s\n",
|
||||
"hostName", tsS3Hostname[epIndex],
|
||||
"bucketName", tsS3BucketName,
|
||||
"protocol", "https only",
|
||||
"uristyle", "path only",
|
||||
"accessKey", tsS3AccessKeyId[epIndex],
|
||||
"accessKeySecret", tsS3AccessKeySecret[epIndex]);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
static int32_t azListBucket(char const *bucketname) {
|
||||
int32_t code = 0;
|
||||
const std::string delimiter = "/";
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
accountURL = "https://" + accountURL;
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = bucketname;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
Azure::Storage::Blobs::ListBlobsOptions options;
|
||||
options.Prefix = "s3";
|
||||
|
||||
checkPrint("objects:\n");
|
||||
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
|
||||
for (const auto &blob : pageResult.Blobs) {
|
||||
checkPrint("%s\n", blob.Name.c_str());
|
||||
}
|
||||
}
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
|
||||
e.ReasonPhrase.c_str());
|
||||
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
int32_t azCheckCfg() {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
azDumpCfgByEp(0);
|
||||
|
||||
// test put
|
||||
char testdata[17] = "0123456789abcdef";
|
||||
const char *objectname[] = {"s3test.txt"};
|
||||
char path[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]);
|
||||
}
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
int c_offset = 10;
|
||||
int c_len = 6;
|
||||
char buf[7] = {0};
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
|
||||
if (!fp) {
|
||||
checkPrint("failed to open test file: %s.\n", path);
|
||||
TAOS_CHECK_GOTO(terrno, &lino, _next);
|
||||
}
|
||||
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
|
||||
checkPrint("failed to write test file: %s.\n", path);
|
||||
TAOS_CHECK_GOTO(terrno, &lino, _next);
|
||||
}
|
||||
if (taosFsyncFile(fp) < 0) {
|
||||
checkPrint("failed to fsync test file: %s.\n", path);
|
||||
TAOS_CHECK_GOTO(terrno, &lino, _next);
|
||||
}
|
||||
(void)taosCloseFile(&fp);
|
||||
|
||||
checkPrint("\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
|
||||
code = azPutObjectFromFileOffset(path, objectname[0], 0, 16);
|
||||
if (code != 0) {
|
||||
checkPrint("put object %s : failed.\n", objectname[0]);
|
||||
TAOS_CHECK_GOTO(code, &lino, _next);
|
||||
}
|
||||
checkPrint("put object %s: success.\n\n", objectname[0]);
|
||||
|
||||
// list buckets
|
||||
checkPrint("start to list bucket %s by prefix s3.\n", tsS3BucketName);
|
||||
code = azListBucket(tsS3BucketName);
|
||||
if (code != 0) {
|
||||
checkPrint("listing bucket %s : failed.\n", tsS3BucketName);
|
||||
TAOS_CHECK_GOTO(code, &lino, _next);
|
||||
}
|
||||
checkPrint("listing bucket %s: success.\n\n", tsS3BucketName);
|
||||
|
||||
// test range get
|
||||
checkPrint("start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
|
||||
code = azGetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
|
||||
if (code != 0) {
|
||||
checkPrint("get object %s : failed.\n", objectname[0]);
|
||||
TAOS_CHECK_GOTO(code, &lino, _next);
|
||||
}
|
||||
|
||||
(void)memcpy(buf, pBlock, c_len);
|
||||
taosMemoryFree(pBlock);
|
||||
checkPrint("object content: %s\n", buf);
|
||||
checkPrint("get object %s: success.\n\n", objectname[0]);
|
||||
|
||||
// delete test object
|
||||
checkPrint("start to delete object: %s.\n", objectname[0]);
|
||||
// code = azDeleteObjectsByPrefix(objectname[0]);
|
||||
azDeleteObjectsByPrefix(objectname[0]);
|
||||
/*
|
||||
if (code != 0) {
|
||||
(void)fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
|
||||
TAOS_CHECK_GOTO(code, &lino, _next);
|
||||
}
|
||||
*/
|
||||
checkPrint("delete object %s: success.\n\n", objectname[0]);
|
||||
|
||||
_next:
|
||||
if (fp) {
|
||||
(void)taosCloseFile(&fp);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
checkPrint("s3 check failed, code: %d, line: %d.\n", code, lino);
|
||||
}
|
||||
|
||||
checkPrint("=================================================================\n");
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static int32_t azPutObjectFromFileOffsetImpl(const char *file, const char *object_name, int64_t offset, int64_t size) {
|
||||
int32_t code = 0;
|
||||
|
||||
std::string endpointUrl = tsS3Hostname[0];
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
|
||||
accountURL = "https://" + accountURL;
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = tsS3BucketName;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
std::string blobName = "blob.txt";
|
||||
uint8_t blobContent[] = "Hello Azure!";
|
||||
// Create the block blob client
|
||||
// BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
|
||||
// TDBlockBlobClient blobClient(containerClient.GetBlobClient(blobName));
|
||||
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
|
||||
|
||||
blobClient.UploadFrom(file, offset, size);
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s: Status Code: %d, Reason Phrase: %s", __func__, static_cast<int>(e.StatusCode), e.ReasonPhrase.c_str());
|
||||
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
|
||||
int32_t code = 0;
|
||||
|
||||
try {
|
||||
code = azPutObjectFromFileOffsetImpl(file, object_name, offset, size);
|
||||
} catch (const std::exception &e) {
|
||||
azError("%s: Reason Phrase: %s", __func__, e.what());
|
||||
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t size, bool check,
|
||||
uint8_t **ppBlock) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
uint8_t *buf = NULL;
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
accountURL = "https://" + accountURL;
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = tsS3BucketName;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
|
||||
|
||||
Blobs::DownloadBlobToOptions options;
|
||||
options.Range = Azure::Core::Http::HttpRange();
|
||||
options.Range.Value().Offset = offset;
|
||||
options.Range.Value().Length = size;
|
||||
|
||||
buf = (uint8_t *)taosMemoryCalloc(1, size);
|
||||
if (!buf) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
auto res = blobClient.DownloadTo(buf, size, options);
|
||||
if (check && res.Value.ContentRange.Length.Value() != size) {
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
*ppBlock = buf;
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
|
||||
e.ReasonPhrase.c_str());
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
|
||||
if (buf) {
|
||||
taosMemoryFree(buf);
|
||||
}
|
||||
*ppBlock = NULL;
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static int32_t azGetObjectBlockRetry(const char *object_name, int64_t offset, int64_t size, bool check,
|
||||
uint8_t **ppBlock) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
// May use an exponential backoff policy for retries with 503
|
||||
int retryCount = 0;
|
||||
static int maxRetryCount = 5;
|
||||
static int minRetryInterval = 1000; // ms
|
||||
static int maxRetryInterval = 3000; // ms
|
||||
|
||||
_retry:
|
||||
code = azGetObjectBlockImpl(object_name, offset, size, check, ppBlock);
|
||||
if (TSDB_CODE_SUCCESS != code && retryCount++ < maxRetryCount) {
|
||||
taosMsleep(taosRand() % (maxRetryInterval - minRetryInterval + 1) + minRetryInterval);
|
||||
uInfo("%s: 0x%x(%s) and retry get object", __func__, code, tstrerror(code));
|
||||
goto _retry;
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
try {
|
||||
code = azGetObjectBlockRetry(object_name, offset, size, check, ppBlock);
|
||||
} catch (const std::exception &e) {
|
||||
azError("%s: Reason Phrase: %s", __func__, e.what());
|
||||
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
static void azDeleteObjectsByPrefixImpl(const char *prefix) {
|
||||
const std::string delimiter = "/";
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
accountURL = "https://" + accountURL;
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = tsS3BucketName;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
Azure::Storage::Blobs::ListBlobsOptions options;
|
||||
options.Prefix = prefix;
|
||||
|
||||
std::set<std::string> listBlobs;
|
||||
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
|
||||
for (const auto &blob : pageResult.Blobs) {
|
||||
listBlobs.insert(blob.Name);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto blobName : listBlobs) {
|
||||
auto blobClient = containerClient.GetAppendBlobClient(blobName);
|
||||
blobClient.Delete();
|
||||
}
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
|
||||
e.ReasonPhrase.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void azDeleteObjectsByPrefix(const char *prefix) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
try {
|
||||
azDeleteObjectsByPrefixImpl(prefix);
|
||||
} catch (const std::exception &e) {
|
||||
azError("%s: Reason Phrase: %s", __func__, e.what());
|
||||
}
|
||||
}
|
||||
|
||||
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) {
|
||||
int32_t code = 0, lino = 0;
|
||||
uint64_t contentLength = 0;
|
||||
|
||||
if (taosStatFile(file, (int64_t *)&contentLength, NULL, NULL) < 0) {
|
||||
azError("ERROR: %s Failed to stat file %s: ", __func__, file);
|
||||
TAOS_RETURN(terrno);
|
||||
}
|
||||
|
||||
code = azPutObjectFromFileOffset(file, object, 0, contentLength);
|
||||
if (code != 0) {
|
||||
azError("ERROR: %s Failed to put file %s: ", __func__, file);
|
||||
TAOS_CHECK_GOTO(code, &lino, _exit);
|
||||
}
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
azError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t azGetObjectToFile(const char *object_name, const char *fileName) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
accountURL = "https://" + accountURL;
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = tsS3BucketName;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
|
||||
|
||||
auto res = blobClient.DownloadTo(fileName);
|
||||
if (res.Value.ContentRange.Length.Value() <= 0) {
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
|
||||
e.ReasonPhrase.c_str());
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
int32_t azGetObjectsByPrefix(const char *prefix, const char *path) {
|
||||
const std::string delimiter = "/";
|
||||
std::string accountName = tsS3AccessKeyId[0];
|
||||
std::string accountKey = tsS3AccessKeySecret[0];
|
||||
std::string accountURL = tsS3Hostname[0];
|
||||
accountURL = "https://" + accountURL;
|
||||
|
||||
try {
|
||||
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
|
||||
|
||||
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
|
||||
|
||||
std::string containerName = tsS3BucketName;
|
||||
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
|
||||
|
||||
Azure::Storage::Blobs::ListBlobsOptions options;
|
||||
options.Prefix = prefix;
|
||||
|
||||
std::set<std::string> listBlobs;
|
||||
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
|
||||
for (const auto &blob : pageResult.Blobs) {
|
||||
listBlobs.insert(blob.Name);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto blobName : listBlobs) {
|
||||
const char *tmp = strchr(blobName.c_str(), '/');
|
||||
tmp = (tmp == NULL) ? blobName.c_str() : tmp + 1;
|
||||
char fileName[PATH_MAX] = {0};
|
||||
if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) {
|
||||
(void)snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp);
|
||||
} else {
|
||||
(void)snprintf(fileName, PATH_MAX, "%s%s", path, tmp);
|
||||
}
|
||||
if (azGetObjectToFile(blobName.c_str(), fileName)) {
|
||||
TAOS_RETURN(TSDB_CODE_FAILED);
|
||||
}
|
||||
}
|
||||
} catch (const Azure::Core::RequestFailedException &e) {
|
||||
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
|
||||
e.ReasonPhrase.c_str());
|
||||
TAOS_RETURN(TSDB_CODE_FAILED);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t azDeleteObjects(const char *object_name[], int nobject) {
|
||||
for (int i = 0; i < nobject; ++i) {
|
||||
azDeleteObjectsByPrefix(object_name[i]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int32_t azBegin() { return TSDB_CODE_SUCCESS; }
|
||||
|
||||
void azEnd() {}
|
||||
|
||||
int32_t azCheckCfg() { return TSDB_CODE_SUCCESS; }
|
||||
|
||||
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void azDeleteObjectsByPrefix(const char *prefix) {}
|
||||
|
||||
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
|
||||
|
||||
int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; }
|
||||
|
||||
int32_t azGetObjectToFile(const char *object_name, const char *fileName) { return 0; }
|
||||
|
||||
int32_t azDeleteObjects(const char *object_name[], int nobject) { return 0; }
|
||||
|
||||
#endif
|
|
@ -0,0 +1,531 @@
|
|||
#if defined(USE_S3)
|
||||
#include <td_avro_parser.h>
|
||||
|
||||
#include <azure/core/azure_assert.hpp>
|
||||
#include <azure/core/internal/json/json.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
|
||||
namespace Azure {
|
||||
namespace Storage {
|
||||
namespace Blobs {
|
||||
namespace _detail {
|
||||
|
||||
namespace {
|
||||
int64_t parseInt(AvroStreamReader::ReaderPos& data) {
|
||||
uint64_t r = 0;
|
||||
int nb = 0;
|
||||
while (true) {
|
||||
uint8_t c = (*data.BufferPtr)[data.Offset++];
|
||||
r = r | ((static_cast<uint64_t>(c) & 0x7f) << (nb * 7));
|
||||
if (c & 0x80) {
|
||||
++nb;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return static_cast<int64_t>(r >> 1) ^ -static_cast<int64_t>(r & 0x01);
|
||||
}
|
||||
|
||||
AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) {
|
||||
const static std::map<std::string, AvroSchema> BuiltinNameSchemaMap = {
|
||||
{"string", AvroSchema::StringSchema}, {"bytes", AvroSchema::BytesSchema}, {"int", AvroSchema::IntSchema},
|
||||
{"long", AvroSchema::LongSchema}, {"float", AvroSchema::FloatSchema}, {"double", AvroSchema::DoubleSchema},
|
||||
{"boolean", AvroSchema::BoolSchema}, {"null", AvroSchema::NullSchema}, {"string", AvroSchema::StringSchema},
|
||||
};
|
||||
std::map<std::string, AvroSchema> nameSchemaMap = BuiltinNameSchemaMap;
|
||||
|
||||
std::function<AvroSchema(const Core::Json::_internal::json& obj)> parseSchemaFromJsonObject;
|
||||
parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema {
|
||||
if (obj.is_string()) {
|
||||
auto typeName = obj.get<std::string>();
|
||||
return nameSchemaMap.find(typeName)->second;
|
||||
} else if (obj.is_array()) {
|
||||
std::vector<AvroSchema> unionSchemas;
|
||||
for (const auto& s : obj) {
|
||||
unionSchemas.push_back(parseSchemaFromJsonObject(s));
|
||||
}
|
||||
return AvroSchema::UnionSchema(std::move(unionSchemas));
|
||||
} else if (obj.is_object()) {
|
||||
if (obj.count("namespace") != 0) {
|
||||
throw std::runtime_error("Namespace isn't supported yet in Avro schema.");
|
||||
}
|
||||
if (obj.count("aliases") != 0) {
|
||||
throw std::runtime_error("Alias isn't supported yet in Avro schema.");
|
||||
}
|
||||
auto typeName = obj["type"].get<std::string>();
|
||||
auto i = nameSchemaMap.find(typeName);
|
||||
if (i != nameSchemaMap.end()) {
|
||||
return i->second;
|
||||
}
|
||||
if (typeName == "record") {
|
||||
std::vector<std::pair<std::string, AvroSchema>> fieldsSchema;
|
||||
for (const auto& field : obj["fields"]) {
|
||||
fieldsSchema.push_back(
|
||||
std::make_pair(field["name"].get<std::string>(), parseSchemaFromJsonObject(field["type"])));
|
||||
}
|
||||
|
||||
const std::string recordName = obj["name"].get<std::string>();
|
||||
auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema));
|
||||
nameSchemaMap.insert(std::make_pair(recordName, recordSchema));
|
||||
return recordSchema;
|
||||
} else if (typeName == "enum") {
|
||||
throw std::runtime_error("Enum type isn't supported yet in Avro schema.");
|
||||
} else if (typeName == "array") {
|
||||
return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"]));
|
||||
} else if (typeName == "map") {
|
||||
return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"]));
|
||||
} else if (typeName == "fixed") {
|
||||
const std::string fixedName = obj["name"].get<std::string>();
|
||||
auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get<int64_t>());
|
||||
nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema));
|
||||
return fixedSchema;
|
||||
} else {
|
||||
throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema.");
|
||||
}
|
||||
}
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
};
|
||||
|
||||
auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end());
|
||||
return parseSchemaFromJsonObject(jsonRoot);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int64_t AvroStreamReader::ParseInt(const Core::Context& context) {
|
||||
uint64_t r = 0;
|
||||
int nb = 0;
|
||||
while (true) {
|
||||
Preload(1, context);
|
||||
uint8_t c = m_streambuffer[m_pos.Offset++];
|
||||
|
||||
r = r | ((static_cast<uint64_t>(c) & 0x7f) << (nb * 7));
|
||||
if (c & 0x80) {
|
||||
++nb;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return static_cast<int64_t>(r >> 1) ^ -static_cast<int64_t>(r & 0x01);
|
||||
}
|
||||
|
||||
void AvroStreamReader::Advance(size_t n, const Core::Context& context) {
|
||||
Preload(n, context);
|
||||
m_pos.Offset += n;
|
||||
}
|
||||
|
||||
size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) {
|
||||
size_t oldAvailable = AvailableBytes();
|
||||
while (true) {
|
||||
size_t newAvailable = TryPreload(n, context);
|
||||
if (newAvailable >= n) {
|
||||
return newAvailable;
|
||||
}
|
||||
if (oldAvailable == newAvailable) {
|
||||
throw std::runtime_error("Unexpected EOF of Avro stream.");
|
||||
}
|
||||
oldAvailable = newAvailable;
|
||||
}
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
|
||||
size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) {
|
||||
size_t availableBytes = AvailableBytes();
|
||||
if (availableBytes >= n) {
|
||||
return availableBytes;
|
||||
}
|
||||
const size_t MinRead = 4096;
|
||||
size_t tryReadSize = (std::max)(n, MinRead);
|
||||
size_t currSize = m_streambuffer.size();
|
||||
m_streambuffer.resize(m_streambuffer.size() + tryReadSize);
|
||||
size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context);
|
||||
m_streambuffer.resize(currSize + actualReadSize);
|
||||
return AvailableBytes();
|
||||
}
|
||||
|
||||
void AvroStreamReader::Discard() {
|
||||
constexpr size_t MinimumReleaseMemory = 128 * 1024;
|
||||
if (m_pos.Offset < MinimumReleaseMemory) {
|
||||
return;
|
||||
}
|
||||
const size_t availableBytes = AvailableBytes();
|
||||
std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes);
|
||||
m_streambuffer.resize(availableBytes);
|
||||
m_pos.Offset = 0;
|
||||
}
|
||||
|
||||
const AvroSchema AvroSchema::StringSchema(AvroDatumType::String);
|
||||
const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes);
|
||||
const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int);
|
||||
const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long);
|
||||
const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float);
|
||||
const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double);
|
||||
const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool);
|
||||
const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null);
|
||||
|
||||
AvroSchema AvroSchema::RecordSchema(std::string name,
|
||||
const std::vector<std::pair<std::string, AvroSchema>>& fieldsSchema) {
|
||||
AvroSchema recordSchema(AvroDatumType::Record);
|
||||
recordSchema.m_name = std::move(name);
|
||||
recordSchema.m_status = std::make_shared<SharedStatus>();
|
||||
for (auto& i : fieldsSchema) {
|
||||
recordSchema.m_status->m_keys.push_back(i.first);
|
||||
recordSchema.m_status->m_schemas.push_back(i.second);
|
||||
}
|
||||
return recordSchema;
|
||||
}
|
||||
|
||||
AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) {
|
||||
AvroSchema arraySchema(AvroDatumType::Array);
|
||||
arraySchema.m_status = std::make_shared<SharedStatus>();
|
||||
arraySchema.m_status->m_schemas.push_back(std::move(elementSchema));
|
||||
return arraySchema;
|
||||
}
|
||||
|
||||
AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) {
|
||||
AvroSchema mapSchema(AvroDatumType::Map);
|
||||
mapSchema.m_status = std::make_shared<SharedStatus>();
|
||||
mapSchema.m_status->m_schemas.push_back(std::move(elementSchema));
|
||||
return mapSchema;
|
||||
}
|
||||
|
||||
AvroSchema AvroSchema::UnionSchema(std::vector<AvroSchema> schemas) {
|
||||
AvroSchema unionSchema(AvroDatumType::Union);
|
||||
unionSchema.m_status = std::make_shared<SharedStatus>();
|
||||
unionSchema.m_status->m_schemas = std::move(schemas);
|
||||
return unionSchema;
|
||||
}
|
||||
|
||||
AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) {
|
||||
AvroSchema fixedSchema(AvroDatumType::Fixed);
|
||||
fixedSchema.m_name = std::move(name);
|
||||
fixedSchema.m_status = std::make_shared<SharedStatus>();
|
||||
fixedSchema.m_status->m_size = size;
|
||||
return fixedSchema;
|
||||
}
|
||||
|
||||
void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) {
|
||||
m_data = reader.m_pos;
|
||||
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
|
||||
int64_t stringSize = reader.ParseInt(context);
|
||||
reader.Advance(static_cast<size_t>(stringSize), context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long ||
|
||||
m_schema.Type() == AvroDatumType::Enum) {
|
||||
reader.ParseInt(context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Float) {
|
||||
reader.Advance(4, context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Double) {
|
||||
reader.Advance(8, context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Bool) {
|
||||
reader.Advance(1, context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Null) {
|
||||
reader.Advance(0, context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Record) {
|
||||
for (const auto& s : m_schema.FieldSchemas()) {
|
||||
AvroDatum(s).Fill(reader, context);
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Array) {
|
||||
while (true) {
|
||||
int64_t numElementsInBlock = reader.ParseInt(context);
|
||||
if (numElementsInBlock == 0) {
|
||||
break;
|
||||
} else if (numElementsInBlock < 0) {
|
||||
int64_t blockSize = reader.ParseInt(context);
|
||||
reader.Advance(static_cast<size_t>(blockSize), context);
|
||||
} else {
|
||||
for (auto i = 0; i < numElementsInBlock; ++i) {
|
||||
AvroDatum(m_schema.ItemSchema()).Fill(reader, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Map) {
|
||||
while (true) {
|
||||
int64_t numElementsInBlock = reader.ParseInt(context);
|
||||
if (numElementsInBlock == 0) {
|
||||
break;
|
||||
} else if (numElementsInBlock < 0) {
|
||||
int64_t blockSize = reader.ParseInt(context);
|
||||
reader.Advance(static_cast<size_t>(blockSize), context);
|
||||
} else {
|
||||
for (int64_t i = 0; i < numElementsInBlock; ++i) {
|
||||
AvroDatum(AvroSchema::StringSchema).Fill(reader, context);
|
||||
AvroDatum(m_schema.ItemSchema()).Fill(reader, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Union) {
|
||||
int64_t i = reader.ParseInt(context);
|
||||
AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]).Fill(reader, context);
|
||||
} else if (m_schema.Type() == AvroDatumType::Fixed) {
|
||||
reader.Advance(m_schema.Size(), context);
|
||||
} else {
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
}
|
||||
|
||||
void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) {
|
||||
m_data = data;
|
||||
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
|
||||
int64_t stringSize = parseInt(data);
|
||||
data.Offset += static_cast<size_t>(stringSize);
|
||||
} else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long ||
|
||||
m_schema.Type() == AvroDatumType::Enum) {
|
||||
parseInt(data);
|
||||
} else if (m_schema.Type() == AvroDatumType::Float) {
|
||||
data.Offset += 4;
|
||||
} else if (m_schema.Type() == AvroDatumType::Double) {
|
||||
data.Offset += 8;
|
||||
} else if (m_schema.Type() == AvroDatumType::Bool) {
|
||||
data.Offset += 1;
|
||||
} else if (m_schema.Type() == AvroDatumType::Null) {
|
||||
data.Offset += 0;
|
||||
} else if (m_schema.Type() == AvroDatumType::Record) {
|
||||
for (const auto& s : m_schema.FieldSchemas()) {
|
||||
AvroDatum(s).Fill(data);
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Array) {
|
||||
while (true) {
|
||||
int64_t numElementsInBlock = parseInt(data);
|
||||
if (numElementsInBlock == 0) {
|
||||
break;
|
||||
} else if (numElementsInBlock < 0) {
|
||||
int64_t blockSize = parseInt(data);
|
||||
data.Offset += static_cast<size_t>(blockSize);
|
||||
} else {
|
||||
for (auto i = 0; i < numElementsInBlock; ++i) {
|
||||
AvroDatum(m_schema.ItemSchema()).Fill(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Map) {
|
||||
while (true) {
|
||||
int64_t numElementsInBlock = parseInt(data);
|
||||
if (numElementsInBlock == 0) {
|
||||
break;
|
||||
} else if (numElementsInBlock < 0) {
|
||||
int64_t blockSize = parseInt(data);
|
||||
data.Offset += static_cast<size_t>(blockSize);
|
||||
} else {
|
||||
for (int64_t i = 0; i < numElementsInBlock; ++i) {
|
||||
AvroDatum(AvroSchema::StringSchema).Fill(data);
|
||||
AvroDatum(m_schema.ItemSchema()).Fill(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (m_schema.Type() == AvroDatumType::Union) {
|
||||
int64_t i = parseInt(data);
|
||||
AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]).Fill(data);
|
||||
} else if (m_schema.Type() == AvroDatumType::Fixed) {
|
||||
data.Offset += m_schema.Size();
|
||||
} else {
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
AvroDatum::StringView AvroDatum::Value() const {
|
||||
auto data = m_data;
|
||||
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
|
||||
const int64_t length = parseInt(data);
|
||||
const uint8_t* start = &(*data.BufferPtr)[data.Offset];
|
||||
StringView ret{start, static_cast<size_t>(length)};
|
||||
data.Offset += static_cast<size_t>(length);
|
||||
return ret;
|
||||
}
|
||||
if (m_schema.Type() == AvroDatumType::Fixed) {
|
||||
const size_t fixedSize = m_schema.Size();
|
||||
const uint8_t* start = &(*data.BufferPtr)[data.Offset];
|
||||
StringView ret{start, fixedSize};
|
||||
data.Offset += fixedSize;
|
||||
return ret;
|
||||
}
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
|
||||
template <>
|
||||
std::string AvroDatum::Value() const {
|
||||
auto stringView = Value<StringView>();
|
||||
return std::string(stringView.Data, stringView.Data + stringView.Length);
|
||||
}
|
||||
|
||||
template <>
|
||||
std::vector<uint8_t> AvroDatum::Value() const {
|
||||
auto stringView = Value<StringView>();
|
||||
return std::vector<uint8_t>(stringView.Data, stringView.Data + stringView.Length);
|
||||
}
|
||||
|
||||
template <>
|
||||
int64_t AvroDatum::Value() const {
|
||||
auto data = m_data;
|
||||
return parseInt(data);
|
||||
}
|
||||
|
||||
template <>
|
||||
int32_t AvroDatum::Value() const {
|
||||
return static_cast<int32_t>(Value<int64_t>());
|
||||
}
|
||||
|
||||
template <>
|
||||
bool AvroDatum::Value() const {
|
||||
return Value<int64_t>();
|
||||
}
|
||||
|
||||
template <>
|
||||
std::nullptr_t AvroDatum::Value() const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <>
|
||||
AvroRecord AvroDatum::Value() const {
|
||||
auto data = m_data;
|
||||
|
||||
AvroRecord r;
|
||||
r.m_keys = &m_schema.FieldNames();
|
||||
for (const auto& schema : m_schema.FieldSchemas()) {
|
||||
auto datum = AvroDatum(schema);
|
||||
datum.Fill(data);
|
||||
r.m_values.push_back(std::move(datum));
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
template <>
|
||||
AvroMap AvroDatum::Value() const {
|
||||
auto data = m_data;
|
||||
|
||||
AvroMap m;
|
||||
while (true) {
|
||||
int64_t numElementsInBlock = parseInt(data);
|
||||
if (numElementsInBlock == 0) {
|
||||
break;
|
||||
}
|
||||
if (numElementsInBlock < 0) {
|
||||
numElementsInBlock = -numElementsInBlock;
|
||||
parseInt(data);
|
||||
}
|
||||
for (int64_t i = 0; i < numElementsInBlock; ++i) {
|
||||
auto keyDatum = AvroDatum(AvroSchema::StringSchema);
|
||||
keyDatum.Fill(data);
|
||||
auto valueDatum = AvroDatum(m_schema.ItemSchema());
|
||||
valueDatum.Fill(data);
|
||||
m[keyDatum.Value<std::string>()] = valueDatum;
|
||||
}
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
template <>
|
||||
AvroDatum AvroDatum::Value() const {
|
||||
auto data = m_data;
|
||||
if (m_schema.Type() == AvroDatumType::Union) {
|
||||
int64_t i = parseInt(data);
|
||||
auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]);
|
||||
datum.Fill(data);
|
||||
return datum;
|
||||
}
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
|
||||
AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream)
|
||||
: m_reader(std::make_unique<AvroStreamReader>(stream)) {}
|
||||
|
||||
AvroDatum AvroObjectContainerReader::NextImpl(const AvroSchema* schema, const Core::Context& context) {
|
||||
AZURE_ASSERT_FALSE(m_eof);
|
||||
static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16);
|
||||
if (!schema) {
|
||||
static AvroSchema FileHeaderSchema = []() {
|
||||
std::vector<std::pair<std::string, AvroSchema>> fieldsSchema;
|
||||
fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4)));
|
||||
fieldsSchema.push_back(std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema)));
|
||||
fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema));
|
||||
return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema));
|
||||
}();
|
||||
auto fileHeaderDatum = AvroDatum(FileHeaderSchema);
|
||||
fileHeaderDatum.Fill(*m_reader, context);
|
||||
auto fileHeader = fileHeaderDatum.Value<AvroRecord>();
|
||||
if (fileHeader.Field("magic").Value<std::string>() != "Obj\01") {
|
||||
throw std::runtime_error("Invalid Avro object container magic.");
|
||||
}
|
||||
AvroMap meta = fileHeader.Field("meta").Value<AvroMap>();
|
||||
std::string objectSchemaJson = meta["avro.schema"].Value<std::string>();
|
||||
std::string codec = "null";
|
||||
if (meta.count("avro.codec") != 0) {
|
||||
codec = meta["avro.codec"].Value<std::string>();
|
||||
}
|
||||
if (codec != "null") {
|
||||
throw std::runtime_error("Unsupported Avro codec: " + codec);
|
||||
}
|
||||
m_syncMarker = fileHeader.Field("sync").Value<std::string>();
|
||||
m_objectSchema = std::make_unique<AvroSchema>(ParseSchemaFromJsonString(objectSchemaJson));
|
||||
schema = m_objectSchema.get();
|
||||
}
|
||||
|
||||
if (m_remainingObjectInCurrentBlock == 0) {
|
||||
m_reader->Discard();
|
||||
m_remainingObjectInCurrentBlock = m_reader->ParseInt(context);
|
||||
int64_t ObjectsSize = m_reader->ParseInt(context);
|
||||
m_reader->Preload(static_cast<size_t>(ObjectsSize), context);
|
||||
}
|
||||
|
||||
auto objectDatum = AvroDatum(*m_objectSchema);
|
||||
objectDatum.Fill(*m_reader, context);
|
||||
if (--m_remainingObjectInCurrentBlock == 0) {
|
||||
auto markerDatum = AvroDatum(SyncMarkerSchema);
|
||||
markerDatum.Fill(*m_reader, context);
|
||||
auto marker = markerDatum.Value<std::string>();
|
||||
if (marker != m_syncMarker) {
|
||||
throw std::runtime_error("Sync marker doesn't match.");
|
||||
}
|
||||
m_eof = m_reader->TryPreload(1, context) == 0;
|
||||
}
|
||||
return objectDatum;
|
||||
}
|
||||
|
||||
size_t AvroStreamParser::OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) {
|
||||
if (m_parserBuffer.Length != 0) {
|
||||
size_t bytesToCopy = (std::min)(m_parserBuffer.Length, count);
|
||||
std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy);
|
||||
m_parserBuffer.Data += bytesToCopy;
|
||||
m_parserBuffer.Length -= bytesToCopy;
|
||||
return bytesToCopy;
|
||||
}
|
||||
while (!m_parser.End()) {
|
||||
auto datum = m_parser.Next(context);
|
||||
if (datum.Schema().Type() == AvroDatumType::Union) {
|
||||
datum = datum.Value<AvroDatum>();
|
||||
}
|
||||
if (datum.Schema().Type() != AvroDatumType::Record) {
|
||||
continue;
|
||||
}
|
||||
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") {
|
||||
auto record = datum.Value<AvroRecord>();
|
||||
auto dataDatum = record.Field("data");
|
||||
m_parserBuffer = dataDatum.Value<AvroDatum::StringView>();
|
||||
return OnRead(buffer, count, context);
|
||||
}
|
||||
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" && m_progressCallback) {
|
||||
auto record = datum.Value<AvroRecord>();
|
||||
auto bytesScanned = record.Field("bytesScanned").Value<int64_t>();
|
||||
auto totalBytes = record.Field("totalBytes").Value<int64_t>();
|
||||
m_progressCallback(bytesScanned, totalBytes);
|
||||
}
|
||||
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" && m_errorCallback) {
|
||||
auto record = datum.Value<AvroRecord>();
|
||||
BlobQueryError e;
|
||||
e.Name = record.Field("name").Value<std::string>();
|
||||
e.Description = record.Field("description").Value<std::string>();
|
||||
e.IsFatal = record.Field("fatal").Value<bool>();
|
||||
e.Position = record.Field("position").Value<int64_t>();
|
||||
m_errorCallback(std::move(e));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
} // namespace _detail
|
||||
} // namespace Blobs
|
||||
} // namespace Storage
|
||||
} // namespace Azure
|
||||
|
||||
#endif
|
|
@ -0,0 +1,625 @@
|
|||
#if defined(USE_S3)
|
||||
|
||||
#include "td_block_blob_client.hpp"
|
||||
|
||||
#include <azure/core/platform.hpp>
|
||||
|
||||
#if defined(AZ_PLATFORM_WINDOWS)
|
||||
#if !defined(WIN32_LEAN_AND_MEAN)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#if !defined(NOMINMAX)
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include <td_avro_parser.h>
|
||||
|
||||
#include <azure/core/io/body_stream.hpp>
|
||||
#include <azure/storage/common/crypt.hpp>
|
||||
#include <azure/storage/common/internal/concurrent_transfer.hpp>
|
||||
#include <azure/storage/common/internal/constants.hpp>
|
||||
#include <azure/storage/common/internal/file_io.hpp>
|
||||
#include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
|
||||
#include <azure/storage/common/storage_common.hpp>
|
||||
#include <azure/storage/common/storage_exception.hpp>
|
||||
|
||||
namespace Azure {
|
||||
namespace Storage {
|
||||
namespace Blobs {
|
||||
|
||||
TDBlockBlobClient TDBlockBlobClient::CreateFromConnectionString(const std::string& connectionString,
|
||||
const std::string& blobContainerName,
|
||||
const std::string& blobName,
|
||||
const BlobClientOptions& options) {
|
||||
TDBlockBlobClient newClient(
|
||||
BlobClient::CreateFromConnectionString(connectionString, blobContainerName, blobName, options));
|
||||
return newClient;
|
||||
}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<StorageSharedKeyCredential> credential,
|
||||
const BlobClientOptions& options)
|
||||
: BlobClient(blobUrl, std::move(credential), options) {}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl,
|
||||
std::shared_ptr<Core::Credentials::TokenCredential> credential,
|
||||
const BlobClientOptions& options)
|
||||
: BlobClient(blobUrl, std::move(credential), options) {}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options)
|
||||
: BlobClient(blobUrl, options) {}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {}
|
||||
|
||||
TDBlockBlobClient TDBlockBlobClient::WithSnapshot(const std::string& snapshot) const {
|
||||
TDBlockBlobClient newClient(*this);
|
||||
if (snapshot.empty()) {
|
||||
newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQuerySnapshot);
|
||||
} else {
|
||||
newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQuerySnapshot,
|
||||
_internal::UrlEncodeQueryParameter(snapshot));
|
||||
}
|
||||
return newClient;
|
||||
}
|
||||
|
||||
TDBlockBlobClient TDBlockBlobClient::WithVersionId(const std::string& versionId) const {
|
||||
TDBlockBlobClient newClient(*this);
|
||||
if (versionId.empty()) {
|
||||
newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQueryVersionId);
|
||||
} else {
|
||||
newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQueryVersionId,
|
||||
_internal::UrlEncodeQueryParameter(versionId));
|
||||
}
|
||||
return newClient;
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobResult> TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content,
|
||||
const UploadBlockBlobOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions;
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
|
||||
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
|
||||
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
|
||||
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
|
||||
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
|
||||
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
|
||||
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
|
||||
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
|
||||
protocolLayerOptions.Tier = options.AccessTier;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
if (options.ImmutabilityPolicy.HasValue()) {
|
||||
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
|
||||
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
|
||||
}
|
||||
protocolLayerOptions.LegalHold = options.HasLegalHold;
|
||||
|
||||
return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
|
||||
const uint8_t* buffer, size_t bufferSize, const UploadBlockBlobFromOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxBlockNumber = 50000;
|
||||
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
|
||||
|
||||
if (static_cast<uint64_t>(options.TransferOptions.SingleUploadThreshold) > (std::numeric_limits<size_t>::max)()) {
|
||||
throw Azure::Core::RequestFailedException("Single upload threshold is too big");
|
||||
}
|
||||
if (bufferSize <= static_cast<size_t>(options.TransferOptions.SingleUploadThreshold)) {
|
||||
Azure::Core::IO::MemoryBodyStream contentStream(buffer, bufferSize);
|
||||
UploadBlockBlobOptions uploadBlockBlobOptions;
|
||||
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
|
||||
uploadBlockBlobOptions.Metadata = options.Metadata;
|
||||
uploadBlockBlobOptions.Tags = options.Tags;
|
||||
uploadBlockBlobOptions.AccessTier = options.AccessTier;
|
||||
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
|
||||
return Upload(contentStream, uploadBlockBlobOptions, context);
|
||||
}
|
||||
|
||||
int64_t chunkSize;
|
||||
if (options.TransferOptions.ChunkSize.HasValue()) {
|
||||
chunkSize = options.TransferOptions.ChunkSize.Value();
|
||||
} else {
|
||||
int64_t minChunkSize = (bufferSize + MaxBlockNumber - 1) / MaxBlockNumber;
|
||||
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
|
||||
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
|
||||
}
|
||||
if (chunkSize > MaxStageBlockSize) {
|
||||
throw Azure::Core::RequestFailedException("Block size is too big.");
|
||||
}
|
||||
|
||||
std::vector<std::string> blockIds;
|
||||
auto getBlockId = [](int64_t id) {
|
||||
constexpr size_t BlockIdLength = 64;
|
||||
std::string blockId = std::to_string(id);
|
||||
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
|
||||
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
|
||||
};
|
||||
|
||||
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
|
||||
Azure::Core::IO::MemoryBodyStream contentStream(buffer + offset, static_cast<size_t>(length));
|
||||
StageBlockOptions chunkOptions;
|
||||
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
|
||||
if (chunkId == numChunks - 1) {
|
||||
blockIds.resize(static_cast<size_t>(numChunks));
|
||||
}
|
||||
};
|
||||
|
||||
_internal::ConcurrentTransfer(0, bufferSize, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc);
|
||||
|
||||
for (size_t i = 0; i < blockIds.size(); ++i) {
|
||||
blockIds[i] = getBlockId(static_cast<int64_t>(i));
|
||||
}
|
||||
CommitBlockListOptions commitBlockListOptions;
|
||||
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
|
||||
commitBlockListOptions.Metadata = options.Metadata;
|
||||
commitBlockListOptions.Tags = options.Tags;
|
||||
commitBlockListOptions.AccessTier = options.AccessTier;
|
||||
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
|
||||
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
|
||||
|
||||
Models::UploadBlockBlobFromResult ret;
|
||||
ret.ETag = std::move(commitBlockListResponse.Value.ETag);
|
||||
ret.LastModified = std::move(commitBlockListResponse.Value.LastModified);
|
||||
ret.VersionId = std::move(commitBlockListResponse.Value.VersionId);
|
||||
ret.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
|
||||
ret.EncryptionKeySha256 = std::move(commitBlockListResponse.Value.EncryptionKeySha256);
|
||||
ret.EncryptionScope = std::move(commitBlockListResponse.Value.EncryptionScope);
|
||||
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(ret),
|
||||
std::move(commitBlockListResponse.RawResponse));
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
|
||||
const std::string& fileName, const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const {
|
||||
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxBlockNumber = 50000;
|
||||
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
|
||||
|
||||
{
|
||||
Azure::Core::IO::FileBodyStream contentStream(fileName);
|
||||
|
||||
if (contentStream.Length() <= options.TransferOptions.SingleUploadThreshold) {
|
||||
UploadBlockBlobOptions uploadBlockBlobOptions;
|
||||
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
|
||||
uploadBlockBlobOptions.Metadata = options.Metadata;
|
||||
uploadBlockBlobOptions.Tags = options.Tags;
|
||||
uploadBlockBlobOptions.AccessTier = options.AccessTier;
|
||||
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
|
||||
return Upload(contentStream, uploadBlockBlobOptions, context);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> blockIds;
|
||||
auto getBlockId = [](int64_t id) {
|
||||
constexpr size_t BlockIdLength = 64;
|
||||
std::string blockId = std::to_string(id);
|
||||
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
|
||||
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
|
||||
};
|
||||
|
||||
_internal::FileReader fileReader(fileName);
|
||||
|
||||
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
|
||||
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length);
|
||||
StageBlockOptions chunkOptions;
|
||||
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
|
||||
if (chunkId == numChunks - 1) {
|
||||
blockIds.resize(static_cast<size_t>(numChunks));
|
||||
}
|
||||
};
|
||||
|
||||
int64_t chunkSize;
|
||||
if (options.TransferOptions.ChunkSize.HasValue()) {
|
||||
chunkSize = options.TransferOptions.ChunkSize.Value();
|
||||
} else {
|
||||
int64_t minChunkSize = (fileReader.GetFileSize() + MaxBlockNumber - 1) / MaxBlockNumber;
|
||||
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
|
||||
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
|
||||
}
|
||||
if (chunkSize > MaxStageBlockSize) {
|
||||
throw Azure::Core::RequestFailedException("Block size is too big.");
|
||||
}
|
||||
|
||||
_internal::ConcurrentTransfer(0, fileReader.GetFileSize(), chunkSize, options.TransferOptions.Concurrency,
|
||||
uploadBlockFunc);
|
||||
|
||||
for (size_t i = 0; i < blockIds.size(); ++i) {
|
||||
blockIds[i] = getBlockId(static_cast<int64_t>(i));
|
||||
}
|
||||
CommitBlockListOptions commitBlockListOptions;
|
||||
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
|
||||
commitBlockListOptions.Metadata = options.Metadata;
|
||||
commitBlockListOptions.Tags = options.Tags;
|
||||
commitBlockListOptions.AccessTier = options.AccessTier;
|
||||
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
|
||||
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
|
||||
|
||||
Models::UploadBlockBlobFromResult result;
|
||||
result.ETag = commitBlockListResponse.Value.ETag;
|
||||
result.LastModified = commitBlockListResponse.Value.LastModified;
|
||||
result.VersionId = commitBlockListResponse.Value.VersionId;
|
||||
result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
|
||||
result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256;
|
||||
result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope;
|
||||
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
|
||||
std::move(commitBlockListResponse.RawResponse));
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
|
||||
const std::string& fileName, int64_t offset, int64_t size, const UploadBlockBlobFromOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_internal::FileReader fileReader(fileName);
|
||||
|
||||
{
|
||||
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, size);
|
||||
|
||||
if (size <= options.TransferOptions.SingleUploadThreshold) {
|
||||
UploadBlockBlobOptions uploadBlockBlobOptions;
|
||||
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
|
||||
uploadBlockBlobOptions.Metadata = options.Metadata;
|
||||
uploadBlockBlobOptions.Tags = options.Tags;
|
||||
uploadBlockBlobOptions.AccessTier = options.AccessTier;
|
||||
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
|
||||
return Upload(contentStream, uploadBlockBlobOptions, context);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> blockIds;
|
||||
auto getBlockId = [](int64_t id) {
|
||||
constexpr size_t BlockIdLength = 64;
|
||||
std::string blockId = std::to_string(id);
|
||||
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
|
||||
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
|
||||
};
|
||||
|
||||
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
|
||||
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length);
|
||||
StageBlockOptions chunkOptions;
|
||||
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
|
||||
if (chunkId == numChunks - 1) {
|
||||
blockIds.resize(static_cast<size_t>(numChunks));
|
||||
}
|
||||
};
|
||||
|
||||
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
|
||||
constexpr int64_t MaxBlockNumber = 50000;
|
||||
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
|
||||
|
||||
int64_t chunkSize;
|
||||
if (options.TransferOptions.ChunkSize.HasValue()) {
|
||||
chunkSize = options.TransferOptions.ChunkSize.Value();
|
||||
} else {
|
||||
int64_t minChunkSize = (size + MaxBlockNumber - 1) / MaxBlockNumber;
|
||||
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
|
||||
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
|
||||
}
|
||||
if (chunkSize > MaxStageBlockSize) {
|
||||
throw Azure::Core::RequestFailedException("Block size is too big.");
|
||||
}
|
||||
|
||||
_internal::ConcurrentTransfer(offset, size, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc);
|
||||
|
||||
for (size_t i = 0; i < blockIds.size(); ++i) {
|
||||
blockIds[i] = getBlockId(static_cast<int64_t>(i));
|
||||
}
|
||||
CommitBlockListOptions commitBlockListOptions;
|
||||
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
|
||||
commitBlockListOptions.Metadata = options.Metadata;
|
||||
commitBlockListOptions.Tags = options.Tags;
|
||||
commitBlockListOptions.AccessTier = options.AccessTier;
|
||||
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
|
||||
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
|
||||
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
|
||||
|
||||
Models::UploadBlockBlobFromResult result;
|
||||
result.ETag = commitBlockListResponse.Value.ETag;
|
||||
result.LastModified = commitBlockListResponse.Value.LastModified;
|
||||
result.VersionId = commitBlockListResponse.Value.VersionId;
|
||||
result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
|
||||
result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256;
|
||||
result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope;
|
||||
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
|
||||
std::move(commitBlockListResponse.RawResponse));
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromUriResult> TDBlockBlobClient::UploadFromUri(
|
||||
const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::UploadBlockBlobFromUriOptions protocolLayerOptions;
|
||||
protocolLayerOptions.CopySource = sourceUri;
|
||||
protocolLayerOptions.CopySourceBlobProperties = options.CopySourceBlobProperties;
|
||||
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
|
||||
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
|
||||
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
|
||||
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
|
||||
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
|
||||
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
|
||||
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
|
||||
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
|
||||
protocolLayerOptions.Tier = options.AccessTier;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch;
|
||||
protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.SourceIfTags = options.SourceAccessConditions.TagConditions;
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
protocolLayerOptions.CopySourceTags = options.CopySourceTagsMode;
|
||||
if (!options.SourceAuthorization.empty()) {
|
||||
protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization;
|
||||
}
|
||||
|
||||
return _detail::BlockBlobClient::UploadFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::StageBlockResult> TDBlockBlobClient::StageBlock(const std::string& blockId,
|
||||
Azure::Core::IO::BodyStream& content,
|
||||
const StageBlockOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::StageBlockBlobBlockOptions protocolLayerOptions;
|
||||
protocolLayerOptions.BlockId = blockId;
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
return _detail::BlockBlobClient::StageBlock(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::StageBlockFromUriResult> TDBlockBlobClient::StageBlockFromUri(
|
||||
const std::string& blockId, const std::string& sourceUri, const StageBlockFromUriOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::StageBlockBlobBlockFromUriOptions protocolLayerOptions;
|
||||
protocolLayerOptions.BlockId = blockId;
|
||||
protocolLayerOptions.SourceUrl = sourceUri;
|
||||
if (options.SourceRange.HasValue()) {
|
||||
std::string rangeStr = "bytes=" + std::to_string(options.SourceRange.Value().Offset) + "-";
|
||||
if (options.SourceRange.Value().Length.HasValue()) {
|
||||
rangeStr += std::to_string(options.SourceRange.Value().Offset + options.SourceRange.Value().Length.Value() - 1);
|
||||
}
|
||||
protocolLayerOptions.SourceRange = rangeStr;
|
||||
}
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch;
|
||||
protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
if (!options.SourceAuthorization.empty()) {
|
||||
protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization;
|
||||
}
|
||||
|
||||
return _detail::BlockBlobClient::StageBlockFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::CommitBlockListResult> TDBlockBlobClient::CommitBlockList(
|
||||
const std::vector<std::string>& blockIds, const CommitBlockListOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::CommitBlockBlobBlockListOptions protocolLayerOptions;
|
||||
protocolLayerOptions.Blocks.Latest = blockIds;
|
||||
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
|
||||
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
|
||||
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
|
||||
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
|
||||
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
|
||||
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
|
||||
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
|
||||
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
|
||||
protocolLayerOptions.Tier = options.AccessTier;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
if (options.ImmutabilityPolicy.HasValue()) {
|
||||
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
|
||||
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
|
||||
}
|
||||
protocolLayerOptions.LegalHold = options.HasLegalHold;
|
||||
|
||||
return _detail::BlockBlobClient::CommitBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::GetBlockListResult> TDBlockBlobClient::GetBlockList(const GetBlockListOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::GetBlockBlobBlockListOptions protocolLayerOptions;
|
||||
protocolLayerOptions.ListType = options.ListType;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
return _detail::BlockBlobClient::GetBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions,
|
||||
_internal::WithReplicaStatus(context));
|
||||
}
|
||||
/*
|
||||
Azure::Response<Models::QueryBlobResult> TDBlockBlobClient::Query(const std::string& querySqlExpression,
|
||||
const QueryBlobOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlobClient::QueryBlobOptions protocolLayerOptions;
|
||||
protocolLayerOptions.QueryRequest.QueryType = Models::_detail::QueryRequestQueryType::SQL;
|
||||
protocolLayerOptions.QueryRequest.Expression = querySqlExpression;
|
||||
if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) {
|
||||
Models::_detail::DelimitedTextConfiguration c;
|
||||
c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator;
|
||||
c.ColumnSeparator = options.InputTextConfiguration.m_columnSeparator;
|
||||
c.FieldQuote = options.InputTextConfiguration.m_quotationCharacter;
|
||||
c.EscapeChar = options.InputTextConfiguration.m_escapeCharacter;
|
||||
c.HeadersPresent = options.InputTextConfiguration.m_hasHeaders;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.InputTextConfiguration.m_format;
|
||||
q.Format.DelimitedTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
|
||||
} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) {
|
||||
Models::_detail::JsonTextConfiguration c;
|
||||
c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.InputTextConfiguration.m_format;
|
||||
q.Format.JsonTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
|
||||
} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) {
|
||||
Models::_detail::ParquetConfiguration c;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.InputTextConfiguration.m_format;
|
||||
q.Format.ParquetTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
|
||||
} else if (options.InputTextConfiguration.m_format.ToString().empty()) {
|
||||
} else {
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) {
|
||||
Models::_detail::DelimitedTextConfiguration c;
|
||||
c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator;
|
||||
c.ColumnSeparator = options.OutputTextConfiguration.m_columnSeparator;
|
||||
c.FieldQuote = options.OutputTextConfiguration.m_quotationCharacter;
|
||||
c.EscapeChar = options.OutputTextConfiguration.m_escapeCharacter;
|
||||
c.HeadersPresent = options.OutputTextConfiguration.m_hasHeaders;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.OutputTextConfiguration.m_format;
|
||||
q.Format.DelimitedTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
|
||||
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) {
|
||||
Models::_detail::JsonTextConfiguration c;
|
||||
c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.OutputTextConfiguration.m_format;
|
||||
q.Format.JsonTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
|
||||
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) {
|
||||
Models::_detail::ParquetConfiguration c;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.OutputTextConfiguration.m_format;
|
||||
q.Format.ParquetTextConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
|
||||
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Arrow) {
|
||||
Models::_detail::ArrowConfiguration c;
|
||||
c.Schema = options.OutputTextConfiguration.m_schema;
|
||||
Models::_detail::QuerySerialization q;
|
||||
q.Format.Type = options.OutputTextConfiguration.m_format;
|
||||
q.Format.ArrowConfiguration = std::move(c);
|
||||
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
|
||||
} else if (options.InputTextConfiguration.m_format.ToString().empty()) {
|
||||
} else {
|
||||
AZURE_UNREACHABLE_CODE();
|
||||
}
|
||||
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
auto response =
|
||||
_detail::BlobClient::Query(*m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context));
|
||||
|
||||
const auto statusCode = response.RawResponse->GetStatusCode();
|
||||
const auto reasonPhrase = response.RawResponse->GetReasonPhrase();
|
||||
const auto requestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderRequestId) != 0
|
||||
? response.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId)
|
||||
: std::string();
|
||||
|
||||
const auto clientRequestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderClientRequestId) != 0
|
||||
? response.RawResponse->GetHeaders().at(_internal::HttpHeaderClientRequestId)
|
||||
: std::string();
|
||||
|
||||
auto defaultErrorHandler = [statusCode, reasonPhrase, requestId, clientRequestId](BlobQueryError e) {
|
||||
if (e.IsFatal) {
|
||||
StorageException exception("Fatal " + e.Name + " at " + std::to_string(e.Position));
|
||||
exception.StatusCode = statusCode;
|
||||
exception.ReasonPhrase = reasonPhrase;
|
||||
exception.RequestId = requestId;
|
||||
exception.ClientRequestId = clientRequestId;
|
||||
exception.ErrorCode = e.Name;
|
||||
exception.Message = e.Description;
|
||||
|
||||
throw exception;
|
||||
}
|
||||
};
|
||||
|
||||
response.Value.BodyStream =
|
||||
std::make_unique<_detail::AvroStreamParser>(std::move(response.Value.BodyStream), options.ProgressHandler,
|
||||
options.ErrorHandler ? options.ErrorHandler : defaultErrorHandler);
|
||||
return response;
|
||||
}
|
||||
*/
|
||||
} // namespace Blobs
|
||||
} // namespace Storage
|
||||
} // namespace Azure
|
||||
|
||||
#endif
|
|
@ -0,0 +1,22 @@
|
|||
if (TD_LINUX)
|
||||
|
||||
aux_source_directory(. AZ_TEST_SRC)
|
||||
|
||||
add_executable(azTest ${AZ_TEST_SRC})
|
||||
target_include_directories(azTest
|
||||
PUBLIC
|
||||
"${TD_SOURCE_DIR}/include/libs/azure"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
|
||||
)
|
||||
|
||||
target_link_libraries(azTest
|
||||
az
|
||||
gtest_main
|
||||
)
|
||||
enable_testing()
|
||||
add_test(
|
||||
NAME az_test
|
||||
COMMAND azTest
|
||||
)
|
||||
|
||||
endif(TD_LINUX)
|
|
@ -0,0 +1,201 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <queue>
|
||||
|
||||
#include "az.h"
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
|
||||
int32_t azInitEnv() {
|
||||
int32_t code = 0;
|
||||
|
||||
extern int8_t tsS3EpNum;
|
||||
|
||||
extern char tsS3Hostname[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
|
||||
extern char tsS3BucketName[TSDB_FQDN_LEN];
|
||||
|
||||
/* TCS parameter format
|
||||
tsS3Hostname[0] = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
tsS3AccessKeyId[0] = "<access-key-id/account-name>";
|
||||
tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
|
||||
tsS3BucketName = "<bucket/container-name>";
|
||||
*/
|
||||
|
||||
const char *hostname = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
const char *accessKeyId = "<access-key-id/account-name>";
|
||||
const char *accessKeySecret = "<access-key-secret/account-key>";
|
||||
const char *bucketName = "<bucket/container-name>";
|
||||
|
||||
if (hostname[0] != '<') {
|
||||
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
|
||||
} else {
|
||||
const char *accountId = getenv("ablob_account_id");
|
||||
if (!accountId) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *accountSecret = getenv("ablob_account_secret");
|
||||
if (!accountSecret) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *containerName = getenv("ablob_container");
|
||||
if (!containerName) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
TAOS_STRCPY(&tsS3Hostname[0][0], accountId);
|
||||
TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net");
|
||||
TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId);
|
||||
TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret);
|
||||
TAOS_STRCPY(tsS3BucketName, containerName);
|
||||
}
|
||||
|
||||
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
|
||||
|
||||
tsS3Enabled = true;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
// TEST(AzTest, DISABLED_InterfaceTest) {
|
||||
TEST(AzTest, InterfaceTest) {
|
||||
int code = 0;
|
||||
bool check = false;
|
||||
bool withcp = false;
|
||||
|
||||
code = azInitEnv();
|
||||
if (code) {
|
||||
std::cout << "ablob env init failed with: " << code << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
GTEST_ASSERT_EQ(tsS3Enabled, 1);
|
||||
|
||||
code = azBegin();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azCheckCfg();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
const int size = 4096;
|
||||
char data[size] = {0};
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
const char object_name[] = "azut.bin";
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azPutObjectFromFileOffset(path, object_name, 0, size);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
code = azGetObjectBlock(object_name, 0, size, check, &pBlock);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
|
||||
}
|
||||
|
||||
taosMemoryFree(pBlock);
|
||||
|
||||
code = azGetObjectToFile(object_name, path_download);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
azDeleteObjectsByPrefix(object_name);
|
||||
// list object to check
|
||||
|
||||
code = azPutObjectFromFile2(path, object_name, withcp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azGetObjectsByPrefix(object_name, tsTempDir);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
const char *object_name_arr[] = {object_name};
|
||||
code = azDeleteObjects(object_name_arr, 1);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
azEnd();
|
||||
}
|
|
@ -226,7 +226,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
|
|||
concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS != pOperator->pTaskInfo->code) {
|
||||
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
|
||||
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(pOperator->pTaskInfo->code));
|
||||
T_LONG_JMP(pTaskInfo->env, pOperator->pTaskInfo->code);
|
||||
}
|
||||
if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
|
||||
|
@ -530,6 +530,16 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
|
||||
int32_t index = pWrapper->sourceIndex;
|
||||
SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index);
|
||||
|
||||
int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, index);
|
||||
if (pRpcHandle != NULL) {
|
||||
int32_t ret = asyncFreeConnById(pExchangeInfo->pTransporter, *pRpcHandle);
|
||||
if (ret != 0) {
|
||||
qDebug("failed to free rpc handle, code:%s, %p", tstrerror(ret), pExchangeInfo);
|
||||
}
|
||||
*pRpcHandle = -1;
|
||||
}
|
||||
|
||||
if (!pSourceDataInfo) {
|
||||
return terrno;
|
||||
}
|
||||
|
|
|
@ -3293,8 +3293,44 @@ static int32_t rewriteIsTrue(SNode* pSrc, SNode** pIsTrue) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
extern int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX];
|
||||
static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) {
|
||||
if (commonType->type < TSDB_DATA_TYPE_NULL || commonType->type >= TSDB_DATA_TYPE_MAX ||
|
||||
newType->type < TSDB_DATA_TYPE_NULL || newType->type >= TSDB_DATA_TYPE_MAX) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
int8_t type1 = commonType->type;
|
||||
int8_t type2 = newType->type;
|
||||
int8_t resultType;
|
||||
if (type1 < type2) {
|
||||
resultType = gDisplyTypes[type1][type2];
|
||||
} else {
|
||||
resultType = gDisplyTypes[type2][type1];
|
||||
}
|
||||
if (resultType == -1) {
|
||||
return TSDB_CODE_SCALAR_CONVERT_ERROR;
|
||||
}
|
||||
if (commonType->type == newType->type) {
|
||||
commonType->bytes = TMAX(commonType->bytes, newType->bytes);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
if (resultType == commonType->type){
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
if(resultType == newType->type) {
|
||||
*commonType = *newType;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]);
|
||||
if(resultType == TSDB_DATA_TYPE_VARCHAR && (IS_FLOAT_TYPE(commonType->type) || IS_FLOAT_TYPE(newType->type))) {
|
||||
commonType->bytes += TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE];
|
||||
}
|
||||
commonType->type = resultType;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseWhen) {
|
||||
bool first = true;
|
||||
bool allNullThen = true;
|
||||
SNode* pNode = NULL;
|
||||
FOREACH(pNode, pCaseWhen->pWhenThenList) {
|
||||
|
@ -3313,10 +3349,18 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW
|
|||
continue;
|
||||
}
|
||||
allNullThen = false;
|
||||
if (first || dataTypeComp(&pCaseWhen->node.resType, &pThenExpr->resType) < 0) {
|
||||
pCaseWhen->node.resType = pThenExpr->resType;
|
||||
pCxt->errCode = selectCommonType(&pCaseWhen->node.resType, &pThenExpr->resType);
|
||||
if(TSDB_CODE_SUCCESS != pCxt->errCode){
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
SExprNode* pElseExpr = (SExprNode*)pCaseWhen->pElse;
|
||||
if (NULL != pElseExpr) {
|
||||
pCxt->errCode = selectCommonType(&pCaseWhen->node.resType, &pElseExpr->resType);
|
||||
if(TSDB_CODE_SUCCESS != pCxt->errCode) {
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
first = false;
|
||||
}
|
||||
|
||||
if (allNullThen) {
|
||||
|
|
|
@ -1030,6 +1030,31 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
|
|||
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
|
||||
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0};
|
||||
|
||||
int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
|
||||
/*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIME NCHA UTINY USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
|
||||
/*NULL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, -1, -1, 8,
|
||||
/*BOOL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 5, 10, 11, 12, 13, 14, 8, -1, -1, -1, -1, 8,
|
||||
/*TINY*/ 0, 0, 2, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*SMAL*/ 0, 0, 0, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*INT */ 0, 0, 0, 0, 4, 5, 8, 8, 8, 5, 10, 4, 4, 5, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*BIGI*/ 0, 0, 0, 0, 0, 5, 8, 8, 8, 5, 10, 5, 5, 5, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*FLOA*/ 0, 0, 0, 0, 0, 0, 6, 7, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 7, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 5, 5, 5, 8, 8, -1, -1, -1, -1, 8,
|
||||
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10, 10, -1, -1, -1, -1, 10,
|
||||
/*UTINY*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 13, 14, 8, -1, -1, -1, -1, 8,
|
||||
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 8, -1, -1, -1, -1, 8,
|
||||
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 8, -1, -1, -1, -1, 8,
|
||||
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, -1, -1, -1, -1, 8,
|
||||
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -1, -1, -1, -1, 8,
|
||||
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -1, -1, -1, -1,
|
||||
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1,
|
||||
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1,
|
||||
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,
|
||||
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20
|
||||
};
|
||||
|
||||
int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
|
||||
if (type1 == type2) {
|
||||
return 0;
|
||||
|
|
|
@ -504,6 +504,10 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
|
|||
code);
|
||||
// called if drop task rsp received code
|
||||
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
|
||||
|
||||
if (pMsg->handle == NULL) {
|
||||
qError("sch handle is NULL, may be already released and mem lea");
|
||||
}
|
||||
if (pMsg) {
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
|
|
|
@ -3,6 +3,7 @@ add_library(stream STATIC ${STREAM_SRC})
|
|||
target_include_directories(
|
||||
stream
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
|
||||
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
|
||||
)
|
||||
|
||||
|
@ -11,7 +12,7 @@ if(${BUILD_WITH_ROCKSDB})
|
|||
if (${BUILD_CONTRIB})
|
||||
target_link_libraries(
|
||||
stream
|
||||
PUBLIC rocksdb tdb
|
||||
PUBLIC rocksdb tdb tcs
|
||||
PRIVATE os util transport qcom executor wal index
|
||||
)
|
||||
target_include_directories(
|
||||
|
@ -30,13 +31,13 @@ if(${BUILD_WITH_ROCKSDB})
|
|||
)
|
||||
target_link_libraries(
|
||||
stream
|
||||
PUBLIC rocksdb tdb
|
||||
PUBLIC rocksdb tdb tcs
|
||||
PRIVATE os util transport qcom executor wal index
|
||||
)
|
||||
else()
|
||||
target_link_libraries(
|
||||
stream
|
||||
PUBLIC rocksdb tdb
|
||||
PUBLIC rocksdb tdb tcs
|
||||
PRIVATE os util transport qcom executor wal index
|
||||
)
|
||||
target_include_directories(
|
||||
|
@ -58,4 +59,3 @@ endif(${BUILD_WITH_ROCKSDB})
|
|||
if(${BUILD_TEST})
|
||||
ADD_SUBDIRECTORY(test)
|
||||
endif(${BUILD_TEST})
|
||||
|
||||
|
|
|
@ -5079,8 +5079,8 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
|
|||
}
|
||||
|
||||
char content[256] = {0};
|
||||
nBytes = tsnprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest, p->curChkpId,
|
||||
"processVer", processId);
|
||||
nBytes = tsnprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest,
|
||||
p->curChkpId, "processVer", processId);
|
||||
if (nBytes <= 0 || nBytes >= sizeof(content)) {
|
||||
code = TSDB_CODE_OUT_OF_RANGE;
|
||||
stError("chkp failed to format meta file: %s, reason: invalid msg", dstDir);
|
||||
|
|
|
@ -13,10 +13,10 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "cos.h"
|
||||
#include "rsync.h"
|
||||
#include "streamBackendRocksdb.h"
|
||||
#include "streamInt.h"
|
||||
#include "tcs.h"
|
||||
|
||||
static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName);
|
||||
static int32_t deleteCheckpointFile(const char* id, const char* name);
|
||||
|
@ -343,7 +343,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
|
|||
// And if we don't start a new timer, and the lost of checkpoint-trigger message may cause the whole checkpoint
|
||||
// procedure to be stucked.
|
||||
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
|
||||
int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1);
|
||||
int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1);
|
||||
if (old == 0) {
|
||||
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
|
||||
stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
|
||||
|
@ -352,7 +352,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
|
|||
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
|
||||
"trigger-recv-monitor");
|
||||
pTmrInfo->launchChkptId = pActiveInfo->activeId;
|
||||
} else { // already launched, do nothing
|
||||
} else { // already launched, do nothing
|
||||
stError("s-task:%s previous checkpoint-trigger monitor tmr is set, not start new one", pTask->id.idStr);
|
||||
}
|
||||
}
|
||||
|
@ -373,10 +373,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
|
|||
|
||||
if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
|
||||
code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
|
||||
code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
|
||||
} else { // only one task exists, no need to dispatch downstream info
|
||||
code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId,
|
||||
-1);
|
||||
code =
|
||||
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1);
|
||||
streamFreeQitem((SStreamQueueItem*)pBlock);
|
||||
}
|
||||
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
|
||||
|
@ -399,8 +399,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
|
|||
if (taskLevel == TASK_LEVEL__SINK) {
|
||||
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num);
|
||||
streamFreeQitem((SStreamQueueItem*)pBlock);
|
||||
code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
|
||||
} else { // source & agg tasks need to forward the checkpoint msg downwards
|
||||
code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
|
||||
} else { // source & agg tasks need to forward the checkpoint msg downwards
|
||||
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
|
||||
code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock);
|
||||
if (code) {
|
||||
|
@ -445,7 +445,7 @@ static int32_t processCheckpointReadyHelp(SActiveCheckpointInfo* pInfo, int32_t
|
|||
.transId = pInfo->transId,
|
||||
.streamId = streamId,
|
||||
.downstreamNodeId = downstreamNodeId};
|
||||
void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
|
||||
void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
|
||||
if (p == NULL) {
|
||||
stError("s-task:%s failed to set checkpoint ready recv msg, code:%s", id, tstrerror(terrno));
|
||||
return terrno;
|
||||
|
@ -560,8 +560,8 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
|
|||
}
|
||||
streamMutexUnlock(&pInfo->lock);
|
||||
|
||||
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%"PRId64", current checkpointId:%"PRId64,
|
||||
pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
|
||||
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64,
|
||||
pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
|
||||
}
|
||||
|
||||
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
|
||||
|
@ -575,8 +575,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
|
|||
|
||||
if (pReq->checkpointId <= pInfo->checkpointId) {
|
||||
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " Ver:%" PRId64
|
||||
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64
|
||||
" transId:%d ignored",
|
||||
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64 " transId:%d ignored",
|
||||
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
|
||||
pReq->transId);
|
||||
streamMutexUnlock(&pTask->lock);
|
||||
|
@ -623,7 +622,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
|
|||
}
|
||||
|
||||
bool valid = (pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer &&
|
||||
pInfo->processedVer <= pReq->checkpointVer);
|
||||
pInfo->processedVer <= pReq->checkpointVer);
|
||||
|
||||
if (!valid) {
|
||||
stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64
|
||||
|
@ -908,7 +907,7 @@ static int32_t doChkptStatusCheck(SStreamTask* pTask) {
|
|||
if (pTmrInfo->launchChkptId != pActiveInfo->activeId) {
|
||||
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
|
||||
stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64
|
||||
", quit, ref:%d",
|
||||
", quit, ref:%d",
|
||||
id, vgId, pTmrInfo->launchChkptId, ref);
|
||||
return -1;
|
||||
}
|
||||
|
@ -1005,7 +1004,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
|
|||
int32_t numOfNotSend = 0;
|
||||
|
||||
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
|
||||
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
|
||||
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
|
||||
|
||||
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
|
||||
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
|
||||
|
@ -1023,7 +1022,8 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
|
|||
}
|
||||
|
||||
if (++pTmrInfo->activeCounter < 50) {
|
||||
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor");
|
||||
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
|
||||
"trigger-recv-monitor");
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -1201,8 +1201,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
|
|||
STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher;
|
||||
|
||||
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId};
|
||||
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
|
||||
if (px == NULL) { // pause the stream task, if memory not enough
|
||||
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
|
||||
if (px == NULL) { // pause the stream task, if memory not enough
|
||||
code = terrno;
|
||||
}
|
||||
} else {
|
||||
|
@ -1213,8 +1213,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
|
|||
}
|
||||
|
||||
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId};
|
||||
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
|
||||
if (px == NULL) { // pause the stream task, if memory not enough
|
||||
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
|
||||
if (px == NULL) { // pause the stream task, if memory not enough
|
||||
code = terrno;
|
||||
break;
|
||||
}
|
||||
|
@ -1288,11 +1288,11 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
|
|||
static int32_t uploadCheckpointToS3(const char* id, const char* path) {
|
||||
int32_t code = 0;
|
||||
int32_t nBytes = 0;
|
||||
|
||||
/*
|
||||
if (s3Init() != 0) {
|
||||
return TSDB_CODE_THIRDPARTY_ERROR;
|
||||
}
|
||||
|
||||
*/
|
||||
TdDirPtr pDir = taosOpenDir(path);
|
||||
if (pDir == NULL) {
|
||||
return terrno;
|
||||
|
@ -1325,11 +1325,11 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) {
|
|||
break;
|
||||
}
|
||||
|
||||
code = s3PutObjectFromFile2(filename, object, 0);
|
||||
code = tcsPutObjectFromFile2(filename, object, 0);
|
||||
if (code != 0) {
|
||||
stError("[s3] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
|
||||
stError("[tcs] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
|
||||
} else {
|
||||
stDebug("[s3] upload checkpoint:%s", filename);
|
||||
stDebug("[tcs] upload checkpoint:%s", filename);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1355,7 +1355,7 @@ int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char
|
|||
taosMemoryFree(buf);
|
||||
return TSDB_CODE_OUT_OF_RANGE;
|
||||
}
|
||||
int32_t code = s3GetObjectToFile(buf, dstName);
|
||||
int32_t code = tcsGetObjectToFile(buf, dstName);
|
||||
if (code != 0) {
|
||||
taosMemoryFree(buf);
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
|
@ -1418,7 +1418,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path, int64_t che
|
|||
if (strlen(tsSnodeAddress) != 0) {
|
||||
return downloadByRsync(id, path, checkpointId);
|
||||
} else if (tsS3StreamEnabled) {
|
||||
return s3GetObjectsByPrefix(id, path);
|
||||
return tcsGetObjectsByPrefix(id, path);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
@ -1432,7 +1432,7 @@ int32_t deleteCheckpoint(const char* id) {
|
|||
if (strlen(tsSnodeAddress) != 0) {
|
||||
return deleteRsync(id);
|
||||
} else if (tsS3StreamEnabled) {
|
||||
s3DeleteObjectsByPrefix(id);
|
||||
tcsDeleteObjectsByPrefix(id);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
@ -1446,7 +1446,7 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
|
|||
}
|
||||
|
||||
char* tmp = object;
|
||||
int32_t code = s3DeleteObjects((const char**)&tmp, 1);
|
||||
int32_t code = tcsDeleteObjects((const char**)&tmp, 1);
|
||||
if (code != 0) {
|
||||
return TSDB_CODE_THIRDPARTY_ERROR;
|
||||
}
|
||||
|
@ -1488,4 +1488,4 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) {
|
|||
streamMutexUnlock(&pTask->lock);
|
||||
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -101,4 +101,4 @@ IF(NOT TD_DARWIN)
|
|||
NAME backendTest
|
||||
COMMAND backendTest
|
||||
)
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
|
|
@ -0,0 +1,22 @@
|
|||
aux_source_directory(src TCS_SRC)
|
||||
|
||||
add_library(tcs STATIC ${TCS_SRC})
|
||||
target_include_directories(
|
||||
tcs
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
|
||||
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
tcs
|
||||
PUBLIC az
|
||||
PUBLIC common
|
||||
# PUBLIC cjson
|
||||
# PUBLIC os
|
||||
# PUBLIC util
|
||||
# PUBLIC crypt
|
||||
)
|
||||
|
||||
if(${BUILD_TEST})
|
||||
add_subdirectory(test)
|
||||
endif(${BUILD_TEST})
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TCS_INT_H_
|
||||
#define _TD_TCS_INT_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tarray.h"
|
||||
#include "tdef.h"
|
||||
#include "tlog.h"
|
||||
#include "tmsg.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int8_t tsS3Ablob;
|
||||
|
||||
typedef enum {
|
||||
TOS_PROTO_NIL,
|
||||
TOS_PROTO_S3,
|
||||
TOS_PROTO_ABLOB,
|
||||
} STosProto;
|
||||
|
||||
typedef struct {
|
||||
int32_t (*Begin)();
|
||||
void (*End)();
|
||||
int32_t (*CheckCfg)();
|
||||
|
||||
int32_t (*PutObjectFromFileOffset)(const char* file, const char* object_name, int64_t offset, int64_t size);
|
||||
int32_t (*GetObjectBlock)(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock);
|
||||
|
||||
void (*DeleteObjectsByPrefix)(const char* prefix);
|
||||
|
||||
int32_t (*PutObjectFromFile2)(const char* file, const char* object, int8_t withcp);
|
||||
int32_t (*GetObjectsByPrefix)(const char* prefix, const char* path);
|
||||
int32_t (*DeleteObjects)(const char* object_name[], int nobject);
|
||||
int32_t (*GetObjectToFile)(const char* object_name, const char* fileName);
|
||||
} STcs;
|
||||
|
||||
extern STcs tcs;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // _TD_TCS_INT_H_
|
|
@ -0,0 +1,105 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "tcs.h"
|
||||
#include "os.h"
|
||||
#include "taoserror.h"
|
||||
#include "tcsInt.h"
|
||||
#include "tglobal.h"
|
||||
|
||||
#include "az.h"
|
||||
#include "cos.h"
|
||||
|
||||
int32_t tcsInit() {
|
||||
int32_t code = 0;
|
||||
|
||||
STosProto proto = tsS3Ablob ? TOS_PROTO_ABLOB : TOS_PROTO_S3;
|
||||
|
||||
if (TOS_PROTO_S3 == proto) {
|
||||
tcs.Begin = s3Begin;
|
||||
tcs.End = s3End;
|
||||
tcs.CheckCfg = s3CheckCfg;
|
||||
|
||||
tcs.PutObjectFromFileOffset = s3PutObjectFromFileOffset;
|
||||
tcs.GetObjectBlock = s3GetObjectBlock;
|
||||
|
||||
tcs.DeleteObjectsByPrefix = s3DeleteObjectsByPrefix;
|
||||
|
||||
tcs.PutObjectFromFile2 = s3PutObjectFromFile2;
|
||||
tcs.GetObjectsByPrefix = s3GetObjectsByPrefix;
|
||||
tcs.DeleteObjects = s3DeleteObjects;
|
||||
tcs.GetObjectToFile = s3GetObjectToFile;
|
||||
|
||||
} else if (TOS_PROTO_ABLOB == proto) {
|
||||
tcs.Begin = azBegin;
|
||||
tcs.End = azEnd;
|
||||
tcs.CheckCfg = azCheckCfg;
|
||||
|
||||
tcs.PutObjectFromFileOffset = azPutObjectFromFileOffset;
|
||||
tcs.GetObjectBlock = azGetObjectBlock;
|
||||
|
||||
tcs.DeleteObjectsByPrefix = azDeleteObjectsByPrefix;
|
||||
|
||||
tcs.PutObjectFromFile2 = azPutObjectFromFile2;
|
||||
tcs.GetObjectsByPrefix = azGetObjectsByPrefix;
|
||||
tcs.DeleteObjects = azDeleteObjects;
|
||||
tcs.GetObjectToFile = azGetObjectToFile;
|
||||
|
||||
} else {
|
||||
code = TSDB_CODE_INVALID_PARA;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = tcs.Begin();
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
void tcsUninit() { tcs.End(); }
|
||||
|
||||
int32_t tcsCheckCfg() {
|
||||
int32_t code = 0;
|
||||
|
||||
if (!tsS3Enabled) {
|
||||
(void)fprintf(stderr, "tcs not configured.\n");
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
code = tcsInit();
|
||||
if (code != 0) {
|
||||
(void)fprintf(stderr, "failed to initialize tcs.\n");
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
code = tcs.CheckCfg();
|
||||
if (code != 0) {
|
||||
(void)fprintf(stderr, "failed to check tcs.\n");
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
tcsUninit();
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tcsPutObjectFromFileOffset(const char* file, const char* object_name, int64_t offset, int64_t size) {
|
||||
return tcs.PutObjectFromFileOffset(file, object_name, offset, size);
|
||||
}
|
||||
|
||||
int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock) {
|
||||
return tcs.GetObjectBlock(object_name, offset, size, check, ppBlock);
|
||||
}
|
||||
|
||||
void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); }
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "tcs.h"
|
||||
#include "tcsInt.h"
|
||||
|
||||
STcs tcs;
|
||||
|
||||
int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) {
|
||||
return tcs.PutObjectFromFile2(file, object, withcp);
|
||||
}
|
||||
|
||||
int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); }
|
||||
|
||||
int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); }
|
||||
|
||||
int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) {
|
||||
return tcs.GetObjectToFile(object_name, fileName);
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
if (TD_LINUX)
|
||||
|
||||
aux_source_directory(. TCS_TEST_SRC)
|
||||
|
||||
add_executable(tcsTest ${TCS_TEST_SRC})
|
||||
target_include_directories(tcsTest
|
||||
PUBLIC
|
||||
"${TD_SOURCE_DIR}/include/libs/tcs"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
|
||||
)
|
||||
|
||||
target_link_libraries(tcsTest
|
||||
tcs
|
||||
gtest_main
|
||||
)
|
||||
enable_testing()
|
||||
add_test(
|
||||
NAME tcs_test
|
||||
COMMAND tcsTest
|
||||
)
|
||||
|
||||
endif()
|
|
@ -0,0 +1,351 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <queue>
|
||||
|
||||
#include "tcs.h"
|
||||
#include "tcsInt.h"
|
||||
|
||||
int32_t tcsInitEnv(int8_t isBlob) {
|
||||
int32_t code = 0;
|
||||
|
||||
extern char tsS3Hostname[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
|
||||
extern char tsS3BucketName[TSDB_FQDN_LEN];
|
||||
|
||||
/* TCS parameter format
|
||||
tsS3Hostname[0] = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
tsS3AccessKeyId[0] = "<access-key-id/account-name>";
|
||||
tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
|
||||
tsS3BucketName = "<bucket/container-name>";
|
||||
*/
|
||||
|
||||
tsS3Ablob = isBlob;
|
||||
if (isBlob) {
|
||||
const char *hostname = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
const char *accessKeyId = "<access-key-id/account-name>";
|
||||
const char *accessKeySecret = "<access-key-secret/account-key>";
|
||||
const char *bucketName = "<bucket/container-name>";
|
||||
|
||||
if (hostname[0] != '<') {
|
||||
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
|
||||
} else {
|
||||
const char *accountId = getenv("ablob_account_id");
|
||||
if (!accountId) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *accountSecret = getenv("ablob_account_secret");
|
||||
if (!accountSecret) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *containerName = getenv("ablob_container");
|
||||
if (!containerName) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
TAOS_STRCPY(&tsS3Hostname[0][0], accountId);
|
||||
TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net");
|
||||
TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId);
|
||||
TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret);
|
||||
TAOS_STRCPY(tsS3BucketName, containerName);
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
const char *hostname = "endpoint/<account-name>.blob.core.windows.net";
|
||||
const char *accessKeyId = "<access-key-id/account-name>";
|
||||
const char *accessKeySecret = "<access-key-secret/account-key>";
|
||||
const char *bucketName = "<bucket/container-name>";
|
||||
*/
|
||||
|
||||
// const char *hostname = "http://192.168.1.52:9000";
|
||||
// const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel";
|
||||
// const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX";
|
||||
// const char *bucketName = "test-bucket";
|
||||
const char *hostname = "192.168.1.52:9000";
|
||||
const char *accessKeyId = "fGPPyYjzytw05nw44ViA";
|
||||
const char *accessKeySecret = "vK1VcwxgSOykicx6hk8fL1x15uEtyDSFU3w4hTaZ";
|
||||
|
||||
const char *bucketName = "ci-bucket19";
|
||||
|
||||
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
|
||||
|
||||
// setup s3 env
|
||||
extern int8_t tsS3EpNum;
|
||||
extern int8_t tsS3Https[TSDB_MAX_EP_NUM];
|
||||
|
||||
tsS3EpNum = 1;
|
||||
tsS3Https[0] = false;
|
||||
}
|
||||
|
||||
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
|
||||
|
||||
tsS3Enabled = true;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
// TEST(TcsTest, DISABLED_InterfaceTest) {
|
||||
TEST(TcsTest, InterfaceTest) {
|
||||
int code = 0;
|
||||
bool check = false;
|
||||
bool withcp = false;
|
||||
|
||||
code = tcsInitEnv(true);
|
||||
if (code) {
|
||||
std::cout << "ablob env init failed with: " << code << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
GTEST_ASSERT_EQ(tsS3Enabled, 1);
|
||||
GTEST_ASSERT_EQ(tsS3Ablob, 1);
|
||||
|
||||
code = tcsInit();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsCheckCfg();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
const int size = 4096;
|
||||
char data[size] = {0};
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
const char object_name[] = "tcsut.bin";
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsPutObjectFromFileOffset(path, object_name, 0, size);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
|
||||
}
|
||||
|
||||
taosMemoryFree(pBlock);
|
||||
|
||||
code = tcsGetObjectToFile(object_name, path_download);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
tcsDeleteObjectsByPrefix(object_name);
|
||||
// list object to check
|
||||
|
||||
code = tcsPutObjectFromFile2(path, object_name, withcp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsGetObjectsByPrefix(object_name, tsTempDir);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
const char *object_name_arr[] = {object_name};
|
||||
code = tcsDeleteObjects(object_name_arr, 1);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
tcsUninit();
|
||||
}
|
||||
|
||||
// TEST(TcsTest, DISABLED_InterfaceNonBlobTest) {
|
||||
TEST(TcsTest, InterfaceNonBlobTest) {
|
||||
int code = 0;
|
||||
bool check = false;
|
||||
bool withcp = false;
|
||||
|
||||
code = tcsInitEnv(false);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
GTEST_ASSERT_EQ(tsS3Enabled, 1);
|
||||
GTEST_ASSERT_EQ(tsS3Ablob, 0);
|
||||
|
||||
code = tcsInit();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsCheckCfg();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
const int size = 4096;
|
||||
char data[size] = {0};
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
const char object_name[] = "tcsut.bin";
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsPutObjectFromFileOffset(path, object_name, 0, size);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
|
||||
}
|
||||
|
||||
taosMemoryFree(pBlock);
|
||||
|
||||
code = tcsGetObjectToFile(object_name, path_download);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
tcsDeleteObjectsByPrefix(object_name);
|
||||
// list object to check
|
||||
|
||||
code = tcsPutObjectFromFile2(path, object_name, withcp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = tcsGetObjectsByPrefix(object_name, tsTempDir);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
const char *object_name_arr[] = {object_name};
|
||||
code = tcsDeleteObjects(object_name_arr, 1);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
tcsUninit();
|
||||
}
|
|
@ -114,6 +114,7 @@ typedef struct SExHandle {
|
|||
void* handle;
|
||||
int64_t refId;
|
||||
void* pThrd;
|
||||
int8_t pThrdIdx;
|
||||
queue q;
|
||||
int8_t inited;
|
||||
SRWLatch latch;
|
||||
|
@ -135,57 +136,33 @@ typedef struct SCvtAddr {
|
|||
bool cvt;
|
||||
} SCvtAddr;
|
||||
|
||||
typedef struct {
|
||||
SEpSet epSet; // ip list provided by app
|
||||
SEpSet origEpSet;
|
||||
void* ahandle; // handle provided by app
|
||||
tmsg_t msgType; // message type
|
||||
int8_t connType; // connection type cli/srv
|
||||
|
||||
STransCtx appCtx; //
|
||||
STransMsg* pRsp; // for synchronous API
|
||||
tsem_t* pSem; // for synchronous API
|
||||
STransSyncMsg* pSyncMsg; // for syncchronous with timeout API
|
||||
int64_t syncMsgRef;
|
||||
SCvtAddr cvtAddr;
|
||||
|
||||
int32_t retryMinInterval;
|
||||
int32_t retryMaxInterval;
|
||||
int32_t retryStepFactor;
|
||||
int64_t retryMaxTimeout;
|
||||
int64_t retryInitTimestamp;
|
||||
int64_t retryNextInterval;
|
||||
bool retryInit;
|
||||
int32_t retryStep;
|
||||
int8_t epsetRetryCnt;
|
||||
int32_t retryCode;
|
||||
|
||||
void* task;
|
||||
int hThrdIdx;
|
||||
} STransConnCtx;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
|
||||
typedef struct {
|
||||
int8_t inUse;
|
||||
int8_t numOfEps;
|
||||
SEp eps[];
|
||||
} SReqEpSet;
|
||||
|
||||
#define TRANS_VER 2
|
||||
typedef struct {
|
||||
char version : 4; // RPC version
|
||||
char comp : 2; // compression algorithm, 0:no compression 1:lz4
|
||||
char noResp : 2; // noResp bits, 0: resp, 1: resp
|
||||
char persist : 2; // persist handle,0: no persit, 1: persist handle
|
||||
char release : 2;
|
||||
char version : 4; // RPC version
|
||||
char comp : 2; // compression algorithm, 0:no compression 1:lz4
|
||||
char noResp : 2; // noResp bits, 0: resp, 1: resp
|
||||
char withUserInfo : 2; // 0: sent user info or not
|
||||
char secured : 2;
|
||||
char spi : 2;
|
||||
char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset
|
||||
|
||||
uint64_t timestamp;
|
||||
char user[TSDB_UNI_LEN];
|
||||
int32_t compatibilityVer;
|
||||
uint32_t magicNum;
|
||||
STraceId traceId;
|
||||
uint64_t ahandle; // ahandle assigned by client
|
||||
uint32_t code; // del later
|
||||
int64_t qid;
|
||||
uint32_t code; // del later
|
||||
uint32_t msgType;
|
||||
int32_t msgLen;
|
||||
int64_t seqNum;
|
||||
uint8_t content[0]; // message body starts from here
|
||||
} STransMsgHead;
|
||||
|
||||
|
@ -206,6 +183,35 @@ typedef struct {
|
|||
|
||||
#pragma pack(pop)
|
||||
|
||||
int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet);
|
||||
int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet);
|
||||
|
||||
int32_t transValidReqEpset(SReqEpSet* pReqEpSet);
|
||||
|
||||
typedef struct {
|
||||
SReqEpSet* epSet; // ip list provided by app
|
||||
SReqEpSet* origEpSet;
|
||||
void* ahandle; // handle provided by app
|
||||
tmsg_t msgType; // message type
|
||||
|
||||
STransCtx userCtx; //
|
||||
STransMsg* pRsp; // for synchronous API
|
||||
tsem_t* pSem; // for synchronous API
|
||||
STransSyncMsg* pSyncMsg; // for syncchronous with timeout API
|
||||
int64_t syncMsgRef;
|
||||
SCvtAddr* pCvtAddr;
|
||||
|
||||
int64_t retryInitTimestamp;
|
||||
int64_t retryNextInterval;
|
||||
int64_t retryMaxTimeout;
|
||||
int32_t retryMinInterval;
|
||||
int32_t retryMaxInterval;
|
||||
int32_t retryStepFactor;
|
||||
int32_t retryStep;
|
||||
int32_t retryCode;
|
||||
int8_t retryInit;
|
||||
int8_t epsetRetryCnt;
|
||||
} SReqCtx;
|
||||
typedef enum { Normal, Quit, Release, Register, Update, FreeById } STransMsgType;
|
||||
typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus;
|
||||
|
||||
|
@ -272,24 +278,24 @@ bool transAsyncPoolIsEmpty(SAsyncPool* pool);
|
|||
} \
|
||||
} while (0)
|
||||
|
||||
#define ASYNC_CHECK_HANDLE(exh1, id) \
|
||||
do { \
|
||||
if (id > 0) { \
|
||||
SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), id); \
|
||||
if (exh2 == NULL || id != exh2->refId) { \
|
||||
tDebug("ref:%" PRId64 " already released", id); \
|
||||
code = terrno; \
|
||||
goto _return1; \
|
||||
} \
|
||||
} else { \
|
||||
tDebug("invalid handle to release"); \
|
||||
goto _return2; \
|
||||
} \
|
||||
#define ASYNC_CHECK_HANDLE(idMgt, id, exh1) \
|
||||
do { \
|
||||
if (id > 0) { \
|
||||
SExHandle* exh2 = transAcquireExHandle(idMgt, id); \
|
||||
if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) { \
|
||||
tError("handle not match, exh1:%p, exh2:%p, refId:%"PRId64"", exh1, exh2, id); \
|
||||
code = TSDB_CODE_INVALID_MSG; \
|
||||
goto _return1; \
|
||||
} \
|
||||
} else { \
|
||||
tError("invalid handle to release"); \
|
||||
goto _return2; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
int32_t transInitBuffer(SConnBuffer* buf);
|
||||
int32_t transClearBuffer(SConnBuffer* buf);
|
||||
int32_t transDestroyBuffer(SConnBuffer* buf);
|
||||
void transDestroyBuffer(SConnBuffer* buf);
|
||||
int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
|
||||
bool transReadComplete(SConnBuffer* connBuf);
|
||||
int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf);
|
||||
|
@ -300,30 +306,31 @@ int32_t transSetConnOption(uv_tcp_t* stream, int keepalive);
|
|||
void transRefSrvHandle(void* handle);
|
||||
void transUnrefSrvHandle(void* handle);
|
||||
|
||||
void transRefCliHandle(void* handle);
|
||||
void transUnrefCliHandle(void* handle);
|
||||
void transRefCliHandle(void* handle);
|
||||
int32_t transUnrefCliHandle(void* handle);
|
||||
int32_t transGetRefCount(void* handle);
|
||||
|
||||
int32_t transReleaseCliHandle(void* handle);
|
||||
int32_t transReleaseSrvHandle(void* handle);
|
||||
|
||||
int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx);
|
||||
int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp);
|
||||
int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated,
|
||||
int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx);
|
||||
int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp);
|
||||
int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated,
|
||||
int32_t timeoutMs);
|
||||
int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId);
|
||||
int32_t transFreeConnById(void* shandle, int64_t transpointId);
|
||||
int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId);
|
||||
int32_t transFreeConnById(void* pInit, int64_t transpointId);
|
||||
|
||||
int32_t transSendResponse(const STransMsg* msg);
|
||||
int32_t transRegisterMsg(const STransMsg* msg);
|
||||
int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn);
|
||||
int32_t transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func);
|
||||
int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn);
|
||||
int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func);
|
||||
|
||||
int32_t transSockInfo2Str(struct sockaddr* sockname, char* dst);
|
||||
void transSockInfo2Str(struct sockaddr* sockname, char* dst);
|
||||
|
||||
int32_t transAllocHandle(int64_t* refId);
|
||||
|
||||
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
|
||||
void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
|
||||
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
|
||||
void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
|
||||
|
||||
void transCloseClient(void* arg);
|
||||
void transCloseServer(void* arg);
|
||||
|
@ -336,33 +343,29 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key);
|
|||
void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType);
|
||||
|
||||
// request list
|
||||
typedef struct STransReq {
|
||||
queue q;
|
||||
uv_write_t wreq;
|
||||
} STransReq;
|
||||
|
||||
void transReqQueueInit(queue* q);
|
||||
void* transReqQueuePush(queue* q);
|
||||
void* transReqQueueRemove(void* arg);
|
||||
void transReqQueueClear(queue* q);
|
||||
typedef struct SWriteReq {
|
||||
queue node; // req queue node
|
||||
void* conn;
|
||||
} SWriteReq;
|
||||
|
||||
// queue sending msgs
|
||||
typedef struct {
|
||||
SArray* q;
|
||||
void (*freeFunc)(const void* arg);
|
||||
queue node;
|
||||
void (*freeFunc)(void* arg);
|
||||
int32_t size;
|
||||
} STransQueue;
|
||||
|
||||
/*
|
||||
* init queue
|
||||
* note: queue'size is small, default 1
|
||||
*/
|
||||
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg));
|
||||
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(void* arg));
|
||||
|
||||
/*
|
||||
* put arg into queue
|
||||
* if queue'size > 1, return false; else return true
|
||||
*/
|
||||
bool transQueuePush(STransQueue* queue, void* arg);
|
||||
void transQueuePush(STransQueue* queue, void* arg);
|
||||
/*
|
||||
* the size of queue
|
||||
*/
|
||||
|
@ -375,10 +378,25 @@ void* transQueuePop(STransQueue* queue);
|
|||
* get ith from queue
|
||||
*/
|
||||
void* transQueueGet(STransQueue* queue, int i);
|
||||
/*
|
||||
* head elm from queue
|
||||
*/
|
||||
|
||||
void* tranQueueHead(STransQueue* q);
|
||||
/*
|
||||
* remove all match elm from queue
|
||||
*/
|
||||
void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size);
|
||||
/*
|
||||
* rm ith from queue
|
||||
*/
|
||||
|
||||
void* transQueueRm(STransQueue* queue, int i);
|
||||
/*
|
||||
* remove el from queue
|
||||
*/
|
||||
|
||||
void transQueueRemove(STransQueue* q, void* e);
|
||||
/*
|
||||
* queue empty or not
|
||||
*/
|
||||
|
@ -418,9 +436,9 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg));
|
|||
SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
|
||||
void transDQCancel(SDelayQueue* queue, SDelayTask* task);
|
||||
|
||||
bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
|
||||
bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b);
|
||||
|
||||
bool transEpSetIsEqual2(SEpSet* a, SEpSet* b);
|
||||
bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b);
|
||||
/*
|
||||
* init global func
|
||||
*/
|
||||
|
@ -432,14 +450,14 @@ void transPrintEpSet(SEpSet* pEpSet);
|
|||
|
||||
void transFreeMsg(void* msg);
|
||||
int32_t transCompressMsg(char* msg, int32_t len);
|
||||
int32_t transDecompressMsg(char** msg, int32_t len);
|
||||
int32_t transDecompressMsg(char** msg, int32_t* len);
|
||||
|
||||
int32_t transOpenRefMgt(int size, void (*func)(void*));
|
||||
void transCloseRefMgt(int32_t refMgt);
|
||||
int64_t transAddExHandle(int32_t refMgt, void* p);
|
||||
int32_t transRemoveExHandle(int32_t refMgt, int64_t refId);
|
||||
void transRemoveExHandle(int32_t refMgt, int64_t refId);
|
||||
void* transAcquireExHandle(int32_t refMgt, int64_t refId);
|
||||
int32_t transReleaseExHandle(int32_t refMgt, int64_t refId);
|
||||
void transReleaseExHandle(int32_t refMgt, int64_t refId);
|
||||
void transDestroyExHandle(void* handle);
|
||||
|
||||
int32_t transGetRefMgt();
|
||||
|
@ -465,6 +483,33 @@ int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf);
|
|||
int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf);
|
||||
int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf);
|
||||
|
||||
enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING };
|
||||
|
||||
#if defined(WINDOWS) || defined(DARWIN)
|
||||
#define BUFFER_LIMIT 1
|
||||
#define STATE_BUFFER_LIMIT 1
|
||||
#else
|
||||
#define BUFFER_LIMIT 4
|
||||
#define STATE_BUFFER_LIMIT 8
|
||||
#endif
|
||||
|
||||
#define HEAP_MISS_HIT_LIMIT 100000
|
||||
#define READ_TIMEOUT 100000
|
||||
|
||||
typedef struct {
|
||||
queue node; // queue for write
|
||||
queue q; // queue for reqs
|
||||
uv_write_t wreq;
|
||||
void* arg;
|
||||
} SWReqsWrapper;
|
||||
|
||||
int32_t initWQ(queue* wq);
|
||||
void destroyWQ(queue* wq);
|
||||
uv_write_t* allocWReqFromWQ(queue* wq, void* arg);
|
||||
|
||||
void freeWReqToWQ(queue* wq, SWReqsWrapper* w);
|
||||
|
||||
int32_t transSetReadOption(uv_handle_t* handle);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -32,8 +32,8 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
|
||||
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
|
||||
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
|
||||
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
|
||||
|
||||
void taosCloseServer(void* arg);
|
||||
void taosCloseClient(void* arg);
|
||||
|
@ -70,12 +70,16 @@ typedef struct {
|
|||
int32_t connLimitNum;
|
||||
int8_t connLimitLock; // 0: no lock. 1. lock
|
||||
int8_t supportBatch; // 0: no batch, 1: support batch
|
||||
int32_t batchSize;
|
||||
int32_t shareConnLimit;
|
||||
int8_t optBatchFetch;
|
||||
int32_t timeToGetConn;
|
||||
int index;
|
||||
void* parent;
|
||||
void* tcphandle; // returned handle from TCP initialization
|
||||
int64_t refId;
|
||||
int8_t shareConn;
|
||||
int8_t startReadTimer;
|
||||
int64_t readTimeout;
|
||||
TdThreadMutex mutex;
|
||||
} SRpcInfo;
|
||||
|
||||
|
|
|
@ -15,13 +15,13 @@
|
|||
|
||||
#include "transComm.h"
|
||||
|
||||
void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = {
|
||||
void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* pInit) = {
|
||||
transInitServer, transInitClient};
|
||||
|
||||
void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient};
|
||||
|
||||
void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle};
|
||||
void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle};
|
||||
void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, NULL};
|
||||
|
||||
int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle};
|
||||
|
||||
|
@ -42,6 +42,8 @@ void* rpcOpen(const SRpcInit* pInit) {
|
|||
if (pRpc == NULL) {
|
||||
TAOS_CHECK_GOTO(terrno, NULL, _end);
|
||||
}
|
||||
|
||||
pRpc->startReadTimer = pInit->startReadTimer;
|
||||
if (pInit->label) {
|
||||
int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label);
|
||||
memcpy(pRpc->label, pInit->label, len);
|
||||
|
@ -77,7 +79,15 @@ void* rpcOpen(const SRpcInit* pInit) {
|
|||
|
||||
pRpc->connLimitLock = pInit->connLimitLock;
|
||||
pRpc->supportBatch = pInit->supportBatch;
|
||||
pRpc->batchSize = pInit->batchSize;
|
||||
pRpc->shareConnLimit = pInit->shareConnLimit;
|
||||
if (pRpc->shareConnLimit <= 0) {
|
||||
pRpc->shareConnLimit = BUFFER_LIMIT;
|
||||
}
|
||||
|
||||
pRpc->readTimeout = pInit->readTimeout;
|
||||
if (pRpc->readTimeout < 0) {
|
||||
pRpc->readTimeout = INT64_MAX;
|
||||
}
|
||||
|
||||
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
|
||||
if (pRpc->numOfThreads <= 0) {
|
||||
|
@ -115,6 +125,8 @@ void* rpcOpen(const SRpcInit* pInit) {
|
|||
int64_t refId = transAddExHandle(transGetInstMgt(), pRpc);
|
||||
void* tmp = transAcquireExHandle(transGetInstMgt(), refId);
|
||||
pRpc->refId = refId;
|
||||
|
||||
pRpc->shareConn = pInit->shareConn;
|
||||
return (void*)refId;
|
||||
_end:
|
||||
taosMemoryFree(pRpc);
|
||||
|
@ -127,9 +139,8 @@ void rpcClose(void* arg) {
|
|||
if (arg == NULL) {
|
||||
return;
|
||||
}
|
||||
TAOS_UNUSED(transRemoveExHandle(transGetInstMgt(), (int64_t)arg));
|
||||
TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)arg));
|
||||
|
||||
transRemoveExHandle(transGetInstMgt(), (int64_t)arg);
|
||||
transReleaseExHandle(transGetInstMgt(), (int64_t)arg);
|
||||
tInfo("end to close rpc");
|
||||
return;
|
||||
}
|
||||
|
@ -175,29 +186,29 @@ void* rpcReallocCont(void* ptr, int64_t contLen) {
|
|||
return st + TRANS_MSG_OVERHEAD;
|
||||
}
|
||||
|
||||
int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
|
||||
return transSendRequest(shandle, pEpSet, pMsg, NULL);
|
||||
int32_t rpcSendRequest(void* pInit, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
|
||||
return transSendRequest(pInit, pEpSet, pMsg, NULL);
|
||||
}
|
||||
int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
|
||||
int32_t rpcSendRequestWithCtx(void* pInit, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
|
||||
if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) {
|
||||
return transSendRequest(shandle, pEpSet, pMsg, pCtx);
|
||||
return transSendRequest(pInit, pEpSet, pMsg, pCtx);
|
||||
} else {
|
||||
return transSendRequestWithId(shandle, pEpSet, pMsg, pRid);
|
||||
return transSendRequestWithId(pInit, pEpSet, pMsg, pRid);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) {
|
||||
return transSendRequestWithId(shandle, pEpSet, pReq, transpointId);
|
||||
int32_t rpcSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) {
|
||||
return transSendRequestWithId(pInit, pEpSet, pReq, transpointId);
|
||||
}
|
||||
|
||||
int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
|
||||
return transSendRecv(shandle, pEpSet, pMsg, pRsp);
|
||||
int32_t rpcSendRecv(void* pInit, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
|
||||
return transSendRecv(pInit, pEpSet, pMsg, pRsp);
|
||||
}
|
||||
int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated,
|
||||
int32_t rpcSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated,
|
||||
int32_t timeoutMs) {
|
||||
return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs);
|
||||
return transSendRecvWithTimeout(pInit, pEpSet, pMsg, pRsp, epUpdated, timeoutMs);
|
||||
}
|
||||
int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); }
|
||||
int32_t rpcFreeConnById(void* pInit, int64_t connId) { return transFreeConnById(pInit, connId); }
|
||||
|
||||
int32_t rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); }
|
||||
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -15,7 +15,7 @@
|
|||
|
||||
#include "transComm.h"
|
||||
|
||||
#define BUFFER_CAP 4096
|
||||
#define BUFFER_CAP 8 * 1024
|
||||
|
||||
static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
|
||||
|
||||
|
@ -59,7 +59,7 @@ int32_t transCompressMsg(char* msg, int32_t len) {
|
|||
taosMemoryFree(buf);
|
||||
return ret;
|
||||
}
|
||||
int32_t transDecompressMsg(char** msg, int32_t len) {
|
||||
int32_t transDecompressMsg(char** msg, int32_t* len) {
|
||||
STransMsgHead* pHead = (STransMsgHead*)(*msg);
|
||||
if (pHead->comp == 0) return 0;
|
||||
|
||||
|
@ -68,16 +68,18 @@ int32_t transDecompressMsg(char** msg, int32_t len) {
|
|||
STransCompMsg* pComp = (STransCompMsg*)pCont;
|
||||
int32_t oriLen = htonl(pComp->contLen);
|
||||
|
||||
char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead));
|
||||
int32_t tlen = *len;
|
||||
char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead));
|
||||
if (buf == NULL) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
STransMsgHead* pNewHead = (STransMsgHead*)buf;
|
||||
int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content,
|
||||
len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
|
||||
tlen - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
|
||||
memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead));
|
||||
|
||||
*len = oriLen + sizeof(STransMsgHead);
|
||||
pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead));
|
||||
|
||||
taosMemoryFree(pHead);
|
||||
|
@ -95,13 +97,12 @@ void transFreeMsg(void* msg) {
|
|||
tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD);
|
||||
taosMemoryFree((char*)msg - sizeof(STransMsgHead));
|
||||
}
|
||||
int transSockInfo2Str(struct sockaddr* sockname, char* dst) {
|
||||
void transSockInfo2Str(struct sockaddr* sockname, char* dst) {
|
||||
struct sockaddr_in addr = *(struct sockaddr_in*)sockname;
|
||||
|
||||
char buf[20] = {0};
|
||||
int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf));
|
||||
sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port));
|
||||
return r;
|
||||
}
|
||||
int32_t transInitBuffer(SConnBuffer* buf) {
|
||||
buf->buf = taosMemoryCalloc(1, BUFFER_CAP);
|
||||
|
@ -116,10 +117,9 @@ int32_t transInitBuffer(SConnBuffer* buf) {
|
|||
buf->invalid = 0;
|
||||
return 0;
|
||||
}
|
||||
int32_t transDestroyBuffer(SConnBuffer* p) {
|
||||
void transDestroyBuffer(SConnBuffer* p) {
|
||||
taosMemoryFree(p->buf);
|
||||
p->buf = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t transClearBuffer(SConnBuffer* buf) {
|
||||
|
@ -184,7 +184,7 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG));
|
||||
tError("failed to reset buffer, total:%d, len:%d since %s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG));
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
return 0;
|
||||
|
@ -281,7 +281,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs
|
|||
async->data = item;
|
||||
err = uv_async_init(loop, async, cb);
|
||||
if (err != 0) {
|
||||
tError("failed to init async, reason:%s", uv_err_name(err));
|
||||
tError("failed to init async since %s", uv_err_name(err));
|
||||
code = TSDB_CODE_THIRDPARTY_ERROR;
|
||||
break;
|
||||
}
|
||||
|
@ -333,14 +333,16 @@ int transAsyncSend(SAsyncPool* pool, queue* q) {
|
|||
SAsyncItem* item = async->data;
|
||||
|
||||
if (taosThreadMutexLock(&item->mtx) != 0) {
|
||||
tError("failed to lock mutex");
|
||||
tError("failed to lock mutex since %s", tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
|
||||
QUEUE_PUSH(&item->qmsg, q);
|
||||
TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx));
|
||||
|
||||
int ret = uv_async_send(async);
|
||||
if (ret != 0) {
|
||||
tError("failed to send async,reason:%s", uv_err_name(ret));
|
||||
tError("failed to send async since %s", uv_err_name(ret));
|
||||
return TSDB_CODE_THIRDPARTY_ERROR;
|
||||
}
|
||||
return 0;
|
||||
|
@ -348,15 +350,17 @@ int transAsyncSend(SAsyncPool* pool, queue* q) {
|
|||
|
||||
void transCtxInit(STransCtx* ctx) {
|
||||
// init transCtx
|
||||
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK);
|
||||
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
|
||||
}
|
||||
void transCtxCleanup(STransCtx* ctx) {
|
||||
if (ctx->args == NULL) {
|
||||
if (ctx == NULL || ctx->args == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
STransCtxVal* iter = taosHashIterate(ctx->args, NULL);
|
||||
while (iter) {
|
||||
int32_t* type = taosHashGetKey(iter, NULL);
|
||||
tDebug("free msg type %s dump func", TMSG_INFO(*type));
|
||||
ctx->freeFunc(iter->val);
|
||||
iter = taosHashIterate(ctx->args, iter);
|
||||
}
|
||||
|
@ -385,7 +389,7 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) {
|
|||
|
||||
int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
|
||||
if (code != 0) {
|
||||
tError("failed to put val to hash, reason:%s", tstrerror(code));
|
||||
tError("failed to put val to hash since %s", tstrerror(code));
|
||||
}
|
||||
iter = taosHashIterate(src->args, iter);
|
||||
}
|
||||
|
@ -415,120 +419,93 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
void transReqQueueInit(queue* q) {
|
||||
// init req queue
|
||||
QUEUE_INIT(q);
|
||||
}
|
||||
void* transReqQueuePush(queue* q) {
|
||||
STransReq* req = taosMemoryCalloc(1, sizeof(STransReq));
|
||||
if (req == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
req->wreq.data = req;
|
||||
QUEUE_PUSH(q, &req->q);
|
||||
return &req->wreq;
|
||||
}
|
||||
void* transReqQueueRemove(void* arg) {
|
||||
void* ret = NULL;
|
||||
uv_write_t* wreq = arg;
|
||||
|
||||
STransReq* req = wreq ? wreq->data : NULL;
|
||||
if (req == NULL) return NULL;
|
||||
QUEUE_REMOVE(&req->q);
|
||||
|
||||
ret = wreq && wreq->handle ? wreq->handle->data : NULL;
|
||||
taosMemoryFree(req);
|
||||
|
||||
return ret;
|
||||
}
|
||||
void transReqQueueClear(queue* q) {
|
||||
while (!QUEUE_IS_EMPTY(q)) {
|
||||
queue* h = QUEUE_HEAD(q);
|
||||
QUEUE_REMOVE(h);
|
||||
STransReq* req = QUEUE_DATA(h, STransReq, q);
|
||||
taosMemoryFree(req);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) {
|
||||
queue->q = taosArrayInit(2, sizeof(void*));
|
||||
if (queue->q == NULL) {
|
||||
return terrno;
|
||||
}
|
||||
queue->freeFunc = (void (*)(const void*))freeFunc;
|
||||
|
||||
int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) {
|
||||
QUEUE_INIT(&wq->node);
|
||||
wq->freeFunc = (void (*)(void*))freeFunc;
|
||||
wq->size = 0;
|
||||
return 0;
|
||||
}
|
||||
bool transQueuePush(STransQueue* queue, void* arg) {
|
||||
if (queue->q == NULL) {
|
||||
return true;
|
||||
}
|
||||
if (taosArrayPush(queue->q, &arg) == NULL) {
|
||||
return false;
|
||||
}
|
||||
if (taosArrayGetSize(queue->q) > 1) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
void transQueuePush(STransQueue* q, void* arg) {
|
||||
queue* node = arg;
|
||||
QUEUE_PUSH(&q->node, node);
|
||||
q->size++;
|
||||
}
|
||||
void* transQueuePop(STransQueue* queue) {
|
||||
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
|
||||
return NULL;
|
||||
}
|
||||
void* ptr = taosArrayGetP(queue->q, 0);
|
||||
taosArrayRemove(queue->q, 0);
|
||||
return ptr;
|
||||
}
|
||||
int32_t transQueueSize(STransQueue* queue) {
|
||||
if (queue->q == NULL) {
|
||||
return 0;
|
||||
}
|
||||
return taosArrayGetSize(queue->q);
|
||||
}
|
||||
void* transQueueGet(STransQueue* queue, int i) {
|
||||
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
|
||||
return NULL;
|
||||
}
|
||||
if (i >= taosArrayGetSize(queue->q)) {
|
||||
return NULL;
|
||||
}
|
||||
void* transQueuePop(STransQueue* q) {
|
||||
if (q->size == 0) return NULL;
|
||||
|
||||
void* ptr = taosArrayGetP(queue->q, i);
|
||||
return ptr;
|
||||
queue* head = QUEUE_HEAD(&q->node);
|
||||
QUEUE_REMOVE(head);
|
||||
q->size--;
|
||||
return head;
|
||||
}
|
||||
int32_t transQueueSize(STransQueue* q) { return q->size; }
|
||||
|
||||
void* transQueueGet(STransQueue* q, int idx) {
|
||||
if (q->size == 0) return NULL;
|
||||
|
||||
while (idx-- > 0) {
|
||||
queue* node = QUEUE_NEXT(&q->node);
|
||||
if (node == &q->node) return NULL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void* transQueueRm(STransQueue* queue, int i) {
|
||||
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
|
||||
return NULL;
|
||||
}
|
||||
if (i >= taosArrayGetSize(queue->q)) {
|
||||
return NULL;
|
||||
}
|
||||
void* ptr = taosArrayGetP(queue->q, i);
|
||||
taosArrayRemove(queue->q, i);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
bool transQueueEmpty(STransQueue* queue) {
|
||||
if (queue->q == NULL) {
|
||||
return true;
|
||||
}
|
||||
return taosArrayGetSize(queue->q) == 0;
|
||||
}
|
||||
void transQueueClear(STransQueue* queue) {
|
||||
if (queue->freeFunc != NULL) {
|
||||
for (int i = 0; i < taosArrayGetSize(queue->q); i++) {
|
||||
void* p = taosArrayGetP(queue->q, i);
|
||||
queue->freeFunc(p);
|
||||
void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) {
|
||||
queue* d = dst;
|
||||
queue* node = QUEUE_NEXT(&q->node);
|
||||
while (node != &q->node) {
|
||||
queue* next = QUEUE_NEXT(node);
|
||||
if (filter && filter(node, arg)) {
|
||||
QUEUE_REMOVE(node);
|
||||
q->size--;
|
||||
QUEUE_PUSH(d, node);
|
||||
if (--size == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
node = next;
|
||||
}
|
||||
taosArrayClear(queue->q);
|
||||
}
|
||||
void transQueueDestroy(STransQueue* queue) {
|
||||
transQueueClear(queue);
|
||||
taosArrayDestroy(queue->q);
|
||||
|
||||
void* tranQueueHead(STransQueue* q) {
|
||||
if (q->size == 0) return NULL;
|
||||
|
||||
queue* head = QUEUE_HEAD(&q->node);
|
||||
return head;
|
||||
}
|
||||
|
||||
void* transQueueRm(STransQueue* q, int i) {
|
||||
// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
|
||||
// return NULL;
|
||||
// }
|
||||
// if (i >= taosArrayGetSize(queue->q)) {
|
||||
// return NULL;
|
||||
// }
|
||||
// void* ptr = taosArrayGetP(queue->q, i);
|
||||
// taosArrayRemove(queue->q, i);
|
||||
// return ptr;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void transQueueRemove(STransQueue* q, void* e) {
|
||||
if (q->size == 0) return;
|
||||
queue* node = e;
|
||||
QUEUE_REMOVE(node);
|
||||
q->size--;
|
||||
}
|
||||
|
||||
bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; }
|
||||
|
||||
void transQueueClear(STransQueue* q) {
|
||||
while (!QUEUE_IS_EMPTY(&q->node)) {
|
||||
queue* h = QUEUE_HEAD(&q->node);
|
||||
QUEUE_REMOVE(h);
|
||||
if (q->freeFunc != NULL) (q->freeFunc)(h);
|
||||
q->size--;
|
||||
}
|
||||
}
|
||||
void transQueueDestroy(STransQueue* q) { transQueueClear(q); }
|
||||
|
||||
static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
|
||||
SDelayTask* arg1 = container_of(a, SDelayTask, node);
|
||||
SDelayTask* arg2 = container_of(b, SDelayTask, node);
|
||||
|
@ -690,7 +667,13 @@ void transPrintEpSet(SEpSet* pEpSet) {
|
|||
len += tsnprintf(buf + len, sizeof(buf) - len, "}");
|
||||
tTrace("%s, inUse:%d", buf, pEpSet->inUse);
|
||||
}
|
||||
bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
|
||||
bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) {
|
||||
if (a == NULL && b == NULL) {
|
||||
return true;
|
||||
} else if (a == NULL || b == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) {
|
||||
return false;
|
||||
}
|
||||
|
@ -701,7 +684,7 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
|
|||
}
|
||||
return true;
|
||||
}
|
||||
bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) {
|
||||
bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) {
|
||||
if (a->numOfEps != b->numOfEps) {
|
||||
return false;
|
||||
}
|
||||
|
@ -757,28 +740,31 @@ int64_t transAddExHandle(int32_t refMgt, void* p) {
|
|||
// acquire extern handle
|
||||
return taosAddRef(refMgt, p);
|
||||
}
|
||||
int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) {
|
||||
void transRemoveExHandle(int32_t refMgt, int64_t refId) {
|
||||
// acquire extern handle
|
||||
return taosRemoveRef(refMgt, refId);
|
||||
int32_t code = taosRemoveRef(refMgt, refId);
|
||||
if (code != 0) {
|
||||
tTrace("failed to remove %" PRId64 " from resetId:%d", refId, refMgt);
|
||||
}
|
||||
}
|
||||
|
||||
void* transAcquireExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle
|
||||
return (void*)taosAcquireRef(refMgt, refId);
|
||||
}
|
||||
|
||||
int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) {
|
||||
void transReleaseExHandle(int32_t refMgt, int64_t refId) {
|
||||
// release extern handle
|
||||
return taosReleaseRef(refMgt, refId);
|
||||
int32_t code = taosReleaseRef(refMgt, refId);
|
||||
if (code != 0) {
|
||||
tTrace("failed to release %" PRId64 " from resetId:%d", refId, refMgt);
|
||||
}
|
||||
}
|
||||
void transDestroyExHandle(void* handle) {
|
||||
if (handle == NULL) {
|
||||
return;
|
||||
}
|
||||
SExHandle* eh = handle;
|
||||
if (!QUEUE_IS_EMPTY(&eh->q)) {
|
||||
tDebug("handle %p mem leak", handle);
|
||||
}
|
||||
tDebug("free exhandle %p", handle);
|
||||
tDebug("trans destroy sid:%" PRId64 ", memory %p", eh->refId, handle);
|
||||
taosMemoryFree(handle);
|
||||
}
|
||||
|
||||
|
@ -841,7 +827,7 @@ int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) {
|
|||
|
||||
int32_t err = uv_inet_ntop(AF_INET, &addr, buf, 32);
|
||||
if (err != 0) {
|
||||
tError("failed to convert ip to string, reason:%s", uv_strerror(err));
|
||||
tError("failed to convert ip to string since %s", uv_strerror(err));
|
||||
return TSDB_CODE_THIRDPARTY_ERROR;
|
||||
}
|
||||
|
||||
|
@ -890,3 +876,113 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) {
|
|||
// STUB_RAND_NETWORK_ERR(status)
|
||||
// return status;
|
||||
// }
|
||||
|
||||
int32_t initWQ(queue* wq) {
|
||||
int32_t code = 0;
|
||||
QUEUE_INIT(wq);
|
||||
for (int i = 0; i < 4; i++) {
|
||||
SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper));
|
||||
if (w == NULL) {
|
||||
TAOS_CHECK_GOTO(terrno, NULL, _exception);
|
||||
}
|
||||
w->wreq.data = w;
|
||||
w->arg = NULL;
|
||||
QUEUE_INIT(&w->node);
|
||||
QUEUE_PUSH(wq, &w->q);
|
||||
}
|
||||
return 0;
|
||||
_exception:
|
||||
destroyWQ(wq);
|
||||
return code;
|
||||
}
|
||||
void destroyWQ(queue* wq) {
|
||||
while (!QUEUE_IS_EMPTY(wq)) {
|
||||
queue* h = QUEUE_HEAD(wq);
|
||||
QUEUE_REMOVE(h);
|
||||
SWReqsWrapper* w = QUEUE_DATA(h, SWReqsWrapper, q);
|
||||
taosMemoryFree(w);
|
||||
}
|
||||
}
|
||||
|
||||
uv_write_t* allocWReqFromWQ(queue* wq, void* arg) {
|
||||
if (!QUEUE_IS_EMPTY(wq)) {
|
||||
queue* node = QUEUE_HEAD(wq);
|
||||
QUEUE_REMOVE(node);
|
||||
SWReqsWrapper* w = QUEUE_DATA(node, SWReqsWrapper, q);
|
||||
w->arg = arg;
|
||||
QUEUE_INIT(&w->node);
|
||||
|
||||
return &w->wreq;
|
||||
} else {
|
||||
SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper));
|
||||
if (w == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
w->wreq.data = w;
|
||||
w->arg = arg;
|
||||
QUEUE_INIT(&w->node);
|
||||
return &w->wreq;
|
||||
}
|
||||
}
|
||||
|
||||
void freeWReqToWQ(queue* wq, SWReqsWrapper* w) {
|
||||
QUEUE_INIT(&w->node);
|
||||
QUEUE_PUSH(wq, &w->q);
|
||||
}
|
||||
|
||||
int32_t transSetReadOption(uv_handle_t* handle) {
|
||||
int32_t code = 0;
|
||||
int32_t fd;
|
||||
int ret = uv_fileno((uv_handle_t*)handle, &fd);
|
||||
if (ret != 0) {
|
||||
tWarn("failed to get fd since %s", uv_err_name(ret));
|
||||
return TSDB_CODE_THIRDPARTY_ERROR;
|
||||
}
|
||||
code = taosSetSockOpt2(fd);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) {
|
||||
if (pEpset == NULL) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
if (pReqEpSet == NULL) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
int32_t size = sizeof(SReqEpSet) + sizeof(SEp) * pEpset->numOfEps;
|
||||
SReqEpSet* pReq = (SReqEpSet*)taosMemoryCalloc(1, size);
|
||||
if (pReq == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
memcpy((char*)pReq, (char*)pEpset, size);
|
||||
// clear previous
|
||||
taosMemoryFree(*pReqEpSet);
|
||||
|
||||
if (transValidReqEpset(pReq) != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(pReq);
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
*pReqEpSet = pReq;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet) {
|
||||
if (pReqEpSet == NULL) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
memcpy((char*)pEpSet, (char*)pReqEpSet, sizeof(SReqEpSet) + sizeof(SEp) * pReqEpSet->numOfEps);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t transValidReqEpset(SReqEpSet* pReqEpSet) {
|
||||
if (pReqEpSet == NULL) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
if (pReqEpSet->numOfEps == 0 || pReqEpSet->numOfEps > TSDB_MAX_EP_NUM || pReqEpSet->inUse >= TSDB_MAX_EP_NUM) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -117,7 +117,7 @@ int main(int argc, char *argv[]) {
|
|||
rpcInit.connType = TAOS_CONN_CLIENT;
|
||||
rpcInit.connLimitNum = 10;
|
||||
rpcInit.connLimitLock = 1;
|
||||
rpcInit.batchSize = 16 * 1024;
|
||||
rpcInit.shareConnLimit = 16 * 1024;
|
||||
rpcInit.supportBatch = 1;
|
||||
|
||||
rpcDebugFlag = 135;
|
||||
|
|
|
@ -395,10 +395,11 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
|
|||
SetFilePointer(h, 0, NULL, FILE_END);
|
||||
}
|
||||
if (h == INVALID_HANDLE_VALUE) {
|
||||
DWORD dwError = GetLastError();
|
||||
DWORD dwError = GetLastError();
|
||||
terrno = TAOS_SYSTEM_WINAPI_ERROR(dwError);
|
||||
// LPVOID lpMsgBuf;
|
||||
// FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf, 0,
|
||||
// FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf,
|
||||
// 0,
|
||||
// NULL);
|
||||
// printf("CreateFile failed with error %d: %s", dwError, (char *)lpMsgBuf);
|
||||
// LocalFree(lpMsgBuf);
|
||||
|
@ -915,7 +916,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
|
|||
}
|
||||
|
||||
struct stat fileStat;
|
||||
int32_t code = fstat(pFile->fd, &fileStat);
|
||||
int32_t code = fstat(pFile->fd, &fileStat);
|
||||
if (-1 == code) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return terrno;
|
||||
|
@ -983,7 +984,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
|
|||
}
|
||||
|
||||
#ifdef _TD_DARWIN_64
|
||||
if(lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) {
|
||||
if (lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
@ -1015,7 +1016,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
|
|||
}
|
||||
return writeLen;
|
||||
|
||||
#else // for linux
|
||||
#else // for linux
|
||||
|
||||
int64_t leftbytes = size;
|
||||
int64_t sentbytes;
|
||||
|
@ -1126,7 +1127,7 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
|
|||
if ((*ppFile)->hFile != NULL) {
|
||||
// FlushFileBuffers((*ppFile)->hFile);
|
||||
if (!CloseHandle((*ppFile)->hFile)) {
|
||||
terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
|
||||
terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
|
||||
code = -1;
|
||||
}
|
||||
(*ppFile)->hFile = NULL;
|
||||
|
@ -1469,7 +1470,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
|
|||
while (!feof(pSrcFile->fp)) {
|
||||
len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
|
||||
if (len > 0) {
|
||||
if(gzwrite(dstFp, data, len) == 0) {
|
||||
if (gzwrite(dstFp, data, len) == 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
ret = terrno;
|
||||
goto cmp_end;
|
||||
|
@ -1564,4 +1565,4 @@ int taosSetAutoDelFile(char *path) {
|
|||
}
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
}
|
|
@ -308,6 +308,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
|
|||
uint32_t r = taosRand() % tsRandErrDivisor;
|
||||
if ((r + 1) <= tsRandErrChance) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
uError("random memory error: %s, %s", tstrerror(terrno), __func__);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue