Merge branch '3.0' into merge/3340

This commit is contained in:
Shengliang Guan 2024-10-24 09:15:18 +08:00
commit 63da0865a2
173 changed files with 21465 additions and 9498 deletions

1
.gitignore vendored
View File

@ -121,6 +121,7 @@ TAGS
contrib/*
!contrib/CMakeLists.txt
!contrib/test
!contrib/azure-cmake
sql
debug*/
.env

View File

@ -0,0 +1,15 @@
# azure
ExternalProject_Add(azure
URL https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-storage-blobs_12.13.0-beta.1.tar.gz
URL_HASH SHA256=3eca486fd60e3522d0a633025ecd652a71515b1e944799b2e8ee31fd590305a9
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
SOURCE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1"
#BUILD_IN_SOURCE TRUE
#BUILD_ALWAYS 1
#UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

View File

@ -220,20 +220,20 @@ ELSE ()
IF ("${SIMD_SUPPORT}" MATCHES "true")
IF (COMPILER_SUPPORT_FMA)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
MESSAGE(STATUS "FMA instructions is ACTIVATED")
ENDIF()
IF (COMPILER_SUPPORT_AVX)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
MESSAGE(STATUS "AVX instructions is ACTIVATED")
ENDIF()
IF (COMPILER_SUPPORT_AVX2)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
MESSAGE(STATUS "AVX2 instructions is ACTIVATED")
ENDIF()
MESSAGE(STATUS "FMA instructions is ACTIVATED")
ENDIF()
IF (COMPILER_SUPPORT_AVX)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
MESSAGE(STATUS "AVX instructions is ACTIVATED")
ENDIF()
IF (COMPILER_SUPPORT_AVX2)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
MESSAGE(STATUS "AVX2 instructions is ACTIVATED")
ENDIF()
IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true")
@ -256,7 +256,8 @@ ELSE ()
IF (${BUILD_SANITIZER})
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
MESSAGE(STATUS "Compile with Address Sanitizer!")
ELSEIF (${BUILD_RELEASE})
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")

View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG main
GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG main
GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

View File

@ -152,6 +152,7 @@ if(${BUILD_WITH_S3})
cat("${TD_SUPPORT_DIR}/xml2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/libs3_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${TD_SUPPORT_DIR}/azure_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
add_definitions(-DUSE_S3)
# cos
@ -614,9 +615,16 @@ if (${BUILD_PCRE2})
add_subdirectory(pcre2 EXCLUDE_FROM_ALL)
endif(${BUILD_PCRE2})
if(${TD_LINUX} AND ${BUILD_WITH_S3})
add_subdirectory(azure-cmake EXCLUDE_FROM_ALL)
endif()
# ================================================================================================
# Build test
# ================================================================================================
MESSAGE("build with dependency tests: ${BUILD_DEPENDENCY_TESTS}")
if(${BUILD_DEPENDENCY_TESTS})
add_subdirectory(test EXCLUDE_FROM_ALL)
endif(${BUILD_DEPENDENCY_TESTS})

View File

@ -0,0 +1,70 @@
# lib_azure_sdk
set(AZURE_DIR "${TD_CONTRIB_DIR}/azure-sdk-for-cpp-azure-storage-blobs_12.13.0-beta.1")
set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk")
file(GLOB AZURE_SDK_SRC
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/credentials/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/cryptography/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/io/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/tracing/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp"
)
file(GLOB AZURE_SDK_UNIFIED_SRC
${AZURE_SDK_SRC}
)
set(AZURE_SDK_INCLUDES
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/inc/"
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/inc/"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/inc/"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/"
)
add_library(_azure_sdk STATIC ${AZURE_SDK_UNIFIED_SRC})
target_compile_definitions(_azure_sdk PRIVATE BUILD_CURL_HTTP_TRANSPORT_ADAPTER)
target_include_directories(
_azure_sdk
PUBLIC "$ENV{HOME}/.cos-local.2/include"
)
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
#find_library(CURL_LIBRARY curl)
#find_library(XML2_LIBRARY xml2)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
#find_library(CoreFoundation_Library CoreFoundation)
#find_library(SystemConfiguration_Library SystemConfiguration)
target_link_libraries(
_azure_sdk
PRIVATE ${CURL_LIBRARY}
PRIVATE ${SSL_LIBRARY}
PRIVATE ${CRYPTO_LIBRARY}
PRIVATE ${XML2_LIBRARY}
#PRIVATE xml2
PRIVATE zlib
# PRIVATE ${CoreFoundation_Library}
# PRIVATE ${SystemConfiguration_Library}
)
# Originally, on Windows azure-core is built with bcrypt and crypt32 by default
if (TARGET OpenSSL::SSL)
target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL)
endif()
# Originally, on Windows azure-core is built with winhttp by default
if (TARGET td_contrib::curl)
target_link_libraries(_azure_sdk PRIVATE td_contrib::curl)
endif()
target_include_directories(_azure_sdk SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES})
add_library(td_contrib::azure_sdk ALIAS _azure_sdk)

View File

@ -28,5 +28,6 @@ if(${BUILD_WITH_TRAFT})
# add_subdirectory(traft)
endif(${BUILD_WITH_TRAFT})
add_subdirectory(azure)
add_subdirectory(tdev)
add_subdirectory(lz4)

View File

@ -0,0 +1,26 @@
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED True)
add_executable (
azure-test
main.cpp
)
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(XML2_LIBRARY xml2 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
#find_library(XML2_LIBRARY xml2)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
#find_library(CoreFoundation_Library CoreFoundation)
#find_library(SystemConfiguration_Library SystemConfiguration)
target_link_libraries(
azure-test
PRIVATE _azure_sdk
PRIVATE ${CURL_LIBRARY}
PRIVATE ${XML2_LIBRARY}
PRIVATE ${SSL_LIBRARY}
PRIVATE ${CRYPTO_LIBRARY}
PRIVATE dl
PRIVATE pthread
)

View File

@ -0,0 +1,99 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
// Include the necessary SDK headers
#include <azure/core.hpp>
#include <azure/storage/blobs.hpp>
// Add appropriate using namespace directives
using namespace Azure::Storage;
using namespace Azure::Storage::Blobs;
// Secrets should be stored & retrieved from secure locations such as Azure::KeyVault. For
// convenience and brevity of samples, the secrets are retrieved from environment variables.
std::string GetEndpointUrl() {
// return std::getenv("AZURE_STORAGE_ACCOUNT_URL");
std::string accountId = getenv("ablob_account_id");
if (accountId.empty()) {
return accountId;
}
return accountId + ".blob.core.windows.net";
}
std::string GetAccountName() {
// return std::getenv("AZURE_STORAGE_ACCOUNT_NAME");
return getenv("ablob_account_id");
}
std::string GetAccountKey() {
// return std::getenv("AZURE_STORAGE_ACCOUNT_KEY");
return getenv("ablob_account_secret");
}
int main() {
std::string endpointUrl = GetEndpointUrl();
std::string accountName = GetAccountName();
std::string accountKey = GetAccountKey();
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
std::string accountURL = "https://fd2d01cd892f844eeaa2273.blob.core.windows.net";
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = "myblobcontainer";
// auto containerClient = blobServiceClient.GetBlobContainerClient("myblobcontainer");
auto containerClient = blobServiceClient.GetBlobContainerClient("td-test");
// Create the container if it does not exist
std::cout << "Creating container: " << containerName << std::endl;
// containerClient.CreateIfNotExists();
std::string blobName = "blob.txt";
uint8_t blobContent[] = "Hello Azure!";
// Create the block blob client
BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
// Upload the blob
std::cout << "Uploading blob: " << blobName << std::endl;
blobClient.UploadFrom(blobContent, sizeof(blobContent));
/*
auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential);
// Create some data to upload into the blob.
std::vector<uint8_t> data = {1, 2, 3, 4};
Azure::Core::IO::MemoryBodyStream stream(data);
Azure::Response<Models::UploadBlockBlobResult> response = blockBlobClient.Upload(stream);
Models::UploadBlockBlobResult model = response.Value;
std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString()
<< std::endl;
*/
} catch (const Azure::Core::RequestFailedException& e) {
std::cout << "Status Code: " << static_cast<int>(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase
<< std::endl;
std::cout << e.what() << std::endl;
return 1;
}
return 0;
}

View File

@ -19,7 +19,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.3.3</version>
<version>3.4.0</version>
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>

View File

@ -18,7 +18,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.3.3</version>
<version>3.4.0</version>
</dependency>
<!-- druid -->
<dependency>

View File

@ -17,7 +17,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.3.3</version>
<version>3.4.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>

View File

@ -67,7 +67,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.3.3</version>
<version>3.4.0</version>
<!-- <scope>system</scope>-->
<!-- <systemPath>${project.basedir}/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar</systemPath>-->
</dependency>

View File

@ -22,7 +22,7 @@
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>3.3.3</version>
<version>3.4.0</version>
</dependency>
<!-- ANCHOR_END: dep-->

View File

@ -17,8 +17,8 @@ public class SchemalessWsTest {
private static final String jsonDemo = "{\"metric\": \"metric_json\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}";
public static void main(String[] args) throws SQLException {
final String url = "jdbc:TAOS-RS://" + host + ":6041?user=root&password=taosdata&batchfetch=true";
try(Connection connection = DriverManager.getConnection(url)){
final String url = "jdbc:TAOS-WS://" + host + ":6041?user=root&password=taosdata";
try (Connection connection = DriverManager.getConnection(url)) {
init(connection);
AbstractConnection conn = connection.unwrap(AbstractConnection.class);

View File

@ -12,9 +12,9 @@ public class WSConnectExample {
public static void main(String[] args) throws Exception {
// use
// String jdbcUrl =
// "jdbc:TAOS-RS://localhost:6041/dbName?user=root&password=taosdata&batchfetch=true";
// "jdbc:TAOS-WS://localhost:6041/dbName?user=root&password=taosdata";
// if you want to connect a specified database named "dbName".
String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata&batchfetch=true";
String jdbcUrl = "jdbc:TAOS-WS://localhost:6041?user=root&password=taosdata";
Properties connProps = new Properties();
connProps.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");

View File

@ -15,7 +15,7 @@ public class WSParameterBindingBasicDemo {
public static void main(String[] args) throws SQLException {
String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true";
String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041";
try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) {
init(conn);
@ -40,7 +40,7 @@ public class WSParameterBindingBasicDemo {
pstmt.setFloat(4, random.nextFloat());
pstmt.addBatch();
}
int [] exeResult = pstmt.executeBatch();
int[] exeResult = pstmt.executeBatch();
// you can check exeResult here
System.out.println("Successfully inserted " + exeResult.length + " rows to power.meters.");
}
@ -60,7 +60,8 @@ public class WSParameterBindingBasicDemo {
try (Statement stmt = conn.createStatement()) {
stmt.execute("CREATE DATABASE IF NOT EXISTS power");
stmt.execute("USE power");
stmt.execute("CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
stmt.execute(
"CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
}
}
}

View File

@ -40,7 +40,7 @@ public class WSParameterBindingFullDemo {
public static void main(String[] args) throws SQLException {
String jdbcUrl = "jdbc:TAOS-RS://" + host + ":6041/?batchfetch=true";
String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041/";
try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) {
@ -51,8 +51,10 @@ public class WSParameterBindingFullDemo {
stmtAll(conn);
} catch (SQLException ex) {
// handle any errors, please refer to the JDBC specifications for detailed exceptions info
System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: " + ex.getMessage());
// handle any errors, please refer to the JDBC specifications for detailed
// exceptions info
System.out.println("Failed to insert data using stmt, ErrCode:" + ex.getErrorCode() + "; ErrMessage: "
+ ex.getMessage());
throw ex;
} catch (Exception ex) {
System.out.println("Failed to insert data using stmt, ErrMessage: " + ex.getMessage());
@ -104,30 +106,29 @@ public class WSParameterBindingFullDemo {
pstmt.setTagBoolean(3, true);
pstmt.setTagString(4, "binary_value");
pstmt.setTagNString(5, "nchar_value");
pstmt.setTagVarbinary(6, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e});
pstmt.setTagGeometry(7, new byte[]{
pstmt.setTagVarbinary(6, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e });
pstmt.setTagGeometry(7, new byte[] {
0x01, 0x01, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x59,
0x40, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x59, 0x40});
0x00, 0x00, 0x00, 0x59, 0x40 });
long current = System.currentTimeMillis();
pstmt.setTimestamp(1, new Timestamp(current));
pstmt.setInt(2, 1);
pstmt.setDouble(3, 1.1);
pstmt.setBoolean(4, true);
pstmt.setString(5, "binary_value");
pstmt.setNString(6, "nchar_value");
pstmt.setVarbinary(7, new byte[]{(byte) 0x98, (byte) 0xf4, 0x6e});
pstmt.setGeometry(8, new byte[]{
pstmt.setVarbinary(7, new byte[] { (byte) 0x98, (byte) 0xf4, 0x6e });
pstmt.setGeometry(8, new byte[] {
0x01, 0x01, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x59,
0x40, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x59, 0x40});
0x00, 0x00, 0x00, 0x59, 0x40 });
pstmt.addBatch();
pstmt.executeBatch();
System.out.println("Successfully inserted rows to example_all_type_stmt.ntb");

View File

@ -3,6 +3,6 @@
```
:::note
对于 Rust 连接器, 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性,那么只有 Websocket 的实现会被编译进来。
对于 Rust 连接器, 连接方式的不同只体现在使用的特性不同。如果启用了 "ws" 特性,那么只有 WebSocket 的实现会被编译进来。
:::

View File

@ -28,7 +28,7 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
1. 通过客户端驱动程序 taosc 直接与服务端程序 taosd 建立连接,这种连接方式下文中简称 “原生连接”。
2. 通过 taosAdapter 组件提供的 REST API 建立与 taosd 的连接,这种连接方式下文中简称 “REST 连接”
3. 通过 taosAdapter 组件提供的 Websocket API 建立与 taosd 的连接,这种连接方式下文中简称 “Websocket 连接”
3. 通过 taosAdapter 组件提供的 WebSocket API 建立与 taosd 的连接,这种连接方式下文中简称 “WebSocket 连接”
![TDengine connection type](connection-type-zh.webp)
@ -38,9 +38,9 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
关键不同点在于:
1. 使用 原生连接,需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本配套。
2. 使用 REST 连接,用户无需安装客户端驱动程序 taosc具有跨平台易用的优势但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 Websocket 连接相比REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。
3. 使用 Websocket 连接,用户也无需安装客户端驱动程序 taosc。
4. 连接云服务实例,必须使用 REST 连接 或 Websocket 连接。
2. 使用 REST 连接,用户无需安装客户端驱动程序 taosc具有跨平台易用的优势但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 WebSocket 连接相比REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。
3. 使用 WebSocket 连接,用户也无需安装客户端驱动程序 taosc。
4. 连接云服务实例,必须使用 REST 连接 或 WebSocket 连接。
**推荐使用 WebSocket 连接**
@ -126,7 +126,7 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速
```bash
pip3 install taos-ws-py
```
:::note 此安装包为 Websocket 连接器
:::note 此安装包为 WebSocket 连接器
- 同时安装 `taospy``taos-ws-py`
```bash
pip3 install taospy[ws]
@ -182,7 +182,7 @@ taos = { version = "*"}
```
:::info
Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 Websocket 连接,如果仅需要建立 Websocket 连接,可设置 `ws` 特性:
Rust 连接器通过不同的特性区分不同的连接方式。默认同时支持原生连接和 WebSocket 连接,如果仅需要建立 WebSocket 连接,可设置 `ws` 特性:
```toml
taos = { version = "*", default-features = false, features = ["ws"] }
@ -201,7 +201,7 @@ taos = { version = "*", default-features = false, features = ["ws"] }
```
npm install @tdengine/websocket
```
:::note Node.js 目前只支持 Websocket 连接
:::note Node.js 目前只支持 WebSocket 连接
- **安装验证**
- 新建安装验证目录,例如:`~/tdengine-test`,下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/nodejsChecker.js)到本地。
- 在命令行中执行以下命令。
@ -271,12 +271,10 @@ dotnet add package TDengine.Connector
<TabItem label="Java" value="java">
Java 连接器建立连接的参数有 URL 和 Properties。
TDengine 的 JDBC URL 规范格式为:
`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
URL 和 Properties 的详细参数说明和如何使用详见 [url 规范](../../reference/connector/java/#url-规范)
**注**REST 连接中增加 `batchfetch` 参数并设置为 true将开启 WebSocket 连接。
</TabItem>
<TabItem label="Python" value="python">
Python 连接器使用 `connect()` 方法来建立连接,下面是连接参数的具体说明:
@ -387,8 +385,8 @@ DSN 的详细说明和如何使用详见 [连接功能](../../reference/connecto
- `reconnectIntervalMs`:重连间隔毫秒时间,默认为 2000。
</TabItem>
<TabItem label="C" value="c">
**Websocket 连接**
C/C++ 语言连接器 Websocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串,其基本结构如下:
**WebSocket 连接**
C/C++ 语言连接器 WebSocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串,其基本结构如下:
```text
<driver>[+<protocol>]://[[<username>:<password>@]<host>:<port>][/<database>][?<p1>=<v1>[&<p2>=<v2>]]
@ -417,8 +415,8 @@ C/C++ 语言连接器原生连接方式使用 `taos_connect()` 函数用于建
</TabItem>
</Tabs>
### Websocket 连接
下面是各语言连接器建立 Websocket 连接代码样例。演示了如何使用 Websocket 连接方式连接到 TDengine 数据库,并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。
### WebSocket 连接
下面是各语言连接器建立 WebSocket 连接代码样例。演示了如何使用 WebSocket 连接方式连接到 TDengine 数据库,并对连接设定一些参数。整个过程主要涉及到数据库连接的建立和异常处理。
<Tabs defaultValue="java" groupId="lang">
<TabItem label="Java" value="java">

View File

@ -33,7 +33,7 @@ REST API直接调用 `taosadapter` 提供的 REST API 接口,进行数据
</TabItem>
<TabItem label="Python" value="python">
```python title="Websocket 连接"
```python title="WebSocket 连接"
{{#include docs/examples/python/create_db_ws.py}}
```
@ -69,7 +69,7 @@ REST API直接调用 `taosadapter` 提供的 REST API 接口,进行数据
</TabItem>
<TabItem label="C" value="c">
```c title="Websocket 连接"
```c title="WebSocket 连接"
{{#include docs/examples/c-ws/create_db_demo.c:create_db_and_table}}
```
@ -114,7 +114,7 @@ NOW 为系统内部函数,默认为客户端所在计算机当前时间。 NOW
</TabItem>
<TabItem label="Python" value="python">
```python title="Websocket 连接"
```python title="WebSocket 连接"
{{#include docs/examples/python/insert_ws.py}}
```
@ -151,7 +151,7 @@ NOW 为系统内部函数,默认为客户端所在计算机当前时间。 NOW
</TabItem>
<TabItem label="C" value="c">
```c title="Websocket 连接"
```c title="WebSocket 连接"
{{#include docs/examples/c-ws/insert_data_demo.c:insert_data}}
```
@ -189,7 +189,7 @@ curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \
</TabItem>
<TabItem label="Python" value="python">
```python title="Websocket 连接"
```python title="WebSocket 连接"
{{#include docs/examples/python/query_ws.py}}
```
@ -230,7 +230,7 @@ rust 连接器还支持使用 **serde** 进行反序列化行为结构体的结
</TabItem>
<TabItem label="C" value="c">
```c title="Websocket 连接"
```c title="WebSocket 连接"
{{#include docs/examples/c-ws/query_data_demo.c:query_data}}
```
@ -273,7 +273,7 @@ reqId 可用于请求链路追踪reqId 就像分布式系统中的 traceId
</TabItem>
<TabItem label="Python" value="python">
```python title="Websocket 连接"
```python title="WebSocket 连接"
{{#include docs/examples/python/reqid_ws.py}}
```
@ -310,7 +310,7 @@ reqId 可用于请求链路追踪reqId 就像分布式系统中的 traceId
</TabItem>
<TabItem label="C" value="c">
```c "Websocket 连接"
```c "WebSocket 连接"
{{#include docs/examples/c-ws/with_reqid_demo.c:with_reqid}}
```

View File

@ -191,7 +191,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c6="passit" 1626006833640000000
:::
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">

View File

@ -23,7 +23,7 @@ import TabItem from "@theme/TabItem";
- 执行批量插入操作,将这些数据行插入到对应的子表中。
3. 最后打印实际插入表中的行数。
## Websocket 连接
## WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
```java

View File

@ -94,7 +94,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
</TabItem>
<TabItem label="C" value="c">
- Websocket 连接: 因为使用 dsn不需要 `td.connect.ip``td.connect.port``td.connect.user` 和 `td.connect.pass` 四个配置项,其余同通用配置项。
- WebSocket 连接: 因为使用 dsn不需要 `td.connect.ip``td.connect.port``td.connect.user` 和 `td.connect.pass` 四个配置项,其余同通用配置项。
- 原生连接: 同通用基础配置项。
</TabItem>
@ -103,8 +103,8 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
</TabItem>
</Tabs>
### Websocket 连接
介绍各语言连接器使用 Websocket 连接方式创建消费者。指定连接的服务器地址,设置自动提交,从最新消息开始消费,指定 `group.id``client.id` 等信息。有的语言的连接器还支持反序列化参数。
### WebSocket 连接
介绍各语言连接器使用 WebSocket 连接方式创建消费者。指定连接的服务器地址,设置自动提交,从最新消息开始消费,指定 `group.id``client.id` 等信息。有的语言的连接器还支持反序列化参数。
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
@ -234,7 +234,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
## 订阅消费数据
消费者订阅主题后,可以开始接收并处理这些主题中的消息。订阅消费数据的示例代码如下:
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
@ -403,7 +403,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
## 指定订阅的 Offset
消费者可以指定从特定 Offset 开始读取分区中的消息,这允许消费者重读消息或跳过已处理的消息。下面展示各语言连接器如何指定订阅的 Offset。
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
@ -549,7 +549,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
**注意**:手工提交消费进度前确保消息正常处理完成,否则处理出错的消息不会被再次消费。自动提交是在本次 `poll` 消息时可能会提交上次消息的消费进度,因此请确保消息处理完毕再进行下一次 `poll` 或消息获取。
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
@ -663,7 +663,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
## 取消订阅和关闭消费
消费者可以取消对主题的订阅,停止接收消息。当消费者不再需要时,应该关闭消费者实例,以释放资源和断开与 TDengine 服务器的连接。
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
@ -766,7 +766,7 @@ Rust 连接器创建消费者的参数为 DSN 可以设置的参数列表请
## 完整示例
### Websocket 连接
### WebSocket 连接
<Tabs defaultValue="java" groupId="lang">
<TabItem value="java" label="Java">
<details>

View File

@ -30,9 +30,8 @@ toc_max_heading_level: 4
目前只有 Java 连接器在 WebSocket 连接模式下支持双活,其配置示例如下
```java
url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
url = "jdbc:TAOS-WS://" + host + ":6041/?user=root&password=taosdata";
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD, "true");
properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_HOST, "192.168.1.11");
properties.setProperty(TSDBDriver.PROPERTY_KEY_SLAVE_CLUSTER_PORT, "6041");
properties.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true");
@ -43,13 +42,13 @@ connection = DriverManager.getConnection(url, properties);
其中的配置属性及含义如下表
| 属性名 | 含义 |
| ----------------- | ------------------ |
| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip默认空 |
| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号,默认空 |
| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用false: 不启用。默认为 false。双活场景下请设置为 true |
| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔,单位毫秒:默认 2000 毫秒,也就是 2 秒;最小值为 0 表示立即重试;最大值不做限制 |
| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数:默认值为 3最小值为 0表示不进行重试最大值不做限制 |
| 属性名 | 含义 |
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
| PROPERTY_KEY_SLAVE_CLUSTER_HOST | 第二节点的主机名或者 ip默认空 |
| PROPERTY_KEY_SLAVE_CLUSTER_PORT | 第二节点的端口号,默认空 |
| PROPERTY_KEY_ENABLE_AUTO_RECONNECT | 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。双活场景下请设置为 true |
| PROPERTY_KEY_RECONNECT_INTERVAL_MS | 重连的时间间隔,单位毫秒:默认 2000 毫秒,也就是 2 秒;最小值为 0 表示立即重试;最大值不做限制 |
| PROPERTY_KEY_RECONNECT_RETRY_COUNT | 每节点最多重试次数:默认值为 3最小值为 0表示不进行重试最大值不做限制 |
### 约束条件

View File

@ -306,7 +306,7 @@ http 返回内容:
## taosAdapter 监控指标
taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标上报给 taosKeeper这些监控指标会被 taosKeeper 写入监控数据库,默认是 `log` 库,可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
taosAdapter 采集 REST/WebSocket 相关请求的监控指标。将监控指标上报给 taosKeeper这些监控指标会被 taosKeeper 写入监控数据库,默认是 `log` 库,可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
#### adapter\_requests 表
@ -331,7 +331,7 @@ taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标
| query\_in\_process | INT UNSIGNED | | 正在处理查询请求数 |
| write\_in\_process | INT UNSIGNED | | 正在处理写入请求数 |
| endpoint | VARCHAR | | 请求端点 |
| req\_type | NCHAR | TAG | 请求类型0 为 REST1 为 Websocket |
| req\_type | NCHAR | TAG | 请求类型0 为 REST1 为 WebSocket |
## 结果返回条数限制

View File

@ -5,14 +5,14 @@ toc_max_heading_level: 4
---
C/C++ 开发人员可以使用 TDengine 的客户端驱动,即 C/C++连接器 (以下都用 TDengine 客户端驱动表示),开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时需要包含 TDengine 头文件,里面列出了提供的 API 的函数原型;应用程序还要链接到所在平台上对应的动态库。
TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库,分别支持 Websocket 连接和原生连接。 Websocket 连接和原生连接的区别是 Websocket 连接方式不要求客户端和服务端版本完全匹配,而原生连接要求,在性能上 Websocket 连接方式也接近于原生连接,一般我们推荐使用 Websocket 连接方式。
TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库,分别支持 WebSocket 连接和原生连接。 WebSocket 连接和原生连接的区别是 WebSocket 连接方式不要求客户端和服务端版本完全匹配,而原生连接要求,在性能上 WebSocket 连接方式也接近于原生连接,一般我们推荐使用 WebSocket 连接方式。
下面我们分开介绍两种连接方式的使用方法。
## Websocket 连接方式
## WebSocket 连接方式
Websocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。
WebSocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。
```c
#include <taosws.h>
@ -44,7 +44,7 @@ TDengine 客户端驱动的动态库位于:
### 错误码
在 C 接口的设计中,错误码采用整数类型表示,每个错误码都对应一个特定的错误状态。如未特别说明,当 API 的返回值是整数时_0_ 代表成功,其它是代表失败原因的错误码,当返回值是指针时, _NULL_ 表示失败。
Websocket 连接方式单独的错误码在 `taosws.h` 中,
WebSocket 连接方式单独的错误码在 `taosws.h` 中,
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
@ -82,7 +82,7 @@ WebSocket 连接方式错误码只保留了原生连接错误码的后两个字
#### DSN
C/C++ Websocket 连接器通过 DSN 连接描述字符串来表示连接信息。
C/C++ WebSocket 连接器通过 DSN 连接描述字符串来表示连接信息。
DSN 描述字符串基本结构如下:
```text
@ -96,16 +96,16 @@ DSN 描述字符串基本结构如下:
- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接,支持如下驱动名:
- **taos**: 默认驱动,支持 SQL 执行,参数绑定,无模式写入。
- **tmq**: 使用 TMQ 订阅数据。
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。
- **http/ws**: 使用 Websocket 协议。
- **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 协议。
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。
- **http/ws**: 使用 WebSocket 协议。
- **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 协议。
- **username/password**: 用于创建连接的用户名及密码。
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时 Websocket 连接默认为 `localhost:6041` 。
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时 WebSocket 连接默认为 `localhost:6041` 。
- **database**: 指定默认连接的数据库名,可选参数。
- **params**:其他可选参数。
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test` 表示使用 Websocket`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test` 表示使用 WebSocket`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
#### 基础 API

View File

@ -33,14 +33,15 @@ REST 连接支持所有能运行 Java 的平台。
| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 |
| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: |
| 3.3.3 | 1. 解决了 Websocket statement 关闭导致的内存泄漏 | - |
| 3.3.2 | 1. 优化 Websocket 连接下的参数绑定性能2. 优化了对 mybatis 的支持 | - |
| 3.3.0 | 1. 优化 Websocket 连接下的数据传输性能2. 支持跳过 SSL 验证,默认关闭 | 3.3.2.0 及更高版本 |
| 3.4.0 | 1. 使用 jackson 库替换 fastjson 库2. WebSocket 采用独立协议标识3. 优化后台拉取线程使用,避免用户误用导致超时。 | - |
| 3.3.3 | 1. 解决了 WebSocket statement 关闭导致的内存泄漏 | - |
| 3.3.2 | 1. 优化 WebSocket 连接下的参数绑定性能2. 优化了对 mybatis 的支持 | - |
| 3.3.0 | 1. 优化 WebSocket 连接下的数据传输性能2. 支持跳过 SSL 验证,默认关闭 | 3.3.2.0 及更高版本 |
| 3.2.11 | 解决了 Native 连接关闭结果集 bug | - |
| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩2. Websocket 自动重连机制默认关闭3. Connection 类提供无模式写入的方法4. 优化了原生连接的数据拉取性能5. 修复了一些已知问题6.元数据获取函数可以返回支持的函数列表。 | - |
| 3.2.9 | 解决了 Websocket prepareStatement 关闭 bug | - |
| 3.2.8 | 优化了自动提交, 解决了 websocket 手动提交 bug, 优化 Websocket prepareStatement 使用一个连接, 元数据支持视图 | - |
| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型,增加 native 连接的时区设置支持。增加 websocket 自动重连功能。 | 3.2.0.0 及更高版本 |
| 3.2.10 | 1. REST/WebSocket 连接支持传输中的数据压缩2. WebSocket 自动重连机制默认关闭3. Connection 类提供无模式写入的方法4. 优化了原生连接的数据拉取性能5. 修复了一些已知问题6.元数据获取函数可以返回支持的函数列表。 | - |
| 3.2.9 | 解决了 WebSocket prepareStatement 关闭 bug | - |
| 3.2.8 | 优化了自动提交, 解决了 WebSocket 手动提交 bug, 优化 WebSocket prepareStatement 使用一个连接, 元数据支持视图 | - |
| 3.2.7 | 支持 VARBINARY 和 GEOMETRY 类型,增加 native 连接的时区设置支持。增加 WebSocket 自动重连功能。 | 3.2.0.0 及更高版本 |
| 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 |
| 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数,以及 unsubscribe() 方法。 | - |
| 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - |
@ -195,17 +196,14 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w
## API 参考
### JDBC 驱动
taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口提供了两个实现类RestfulDriver 和 TSDBDriver。
Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口,提供了 3 个实现类。
- WebSocket 连接使用驱动类 `com.taosdata.jdbc.ws.WebSocketDriver`。
- 原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
- REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。
#### URL 规范
TDengine 的 JDBC URL 规范格式为:
`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
对于建立连接,原生连接与 REST 连接有细微不同。 Websocket 和 REST 连接使用驱动类 `com.taosdata.jdbc.rs.RestfulDriver`。原生连接使用驱动类 `com.taosdata.jdbc.TSDBDriver`。
**注**REST 连接中增加 `batchfetch` 参数并设置为 true将开启 WebSocket 连接。
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
**原生连接**
`jdbc:TAOS://taosdemo.com:6030/power?user=root&password=taosdata`,使用了 JDBC 原生连接的 TSDBDriver建立了到 hostname 为 taosdemo.com端口为 6030TDengine 的默认端口),数据库名为 power 的连接。这个 URL
@ -234,23 +232,38 @@ TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可
> **注意**:这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件Linux OS 上默认值 /etc/taos/taos.cfg Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
**WebSocket 连接**
使用 JDBC WebSocket 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
**Websocket 和 REST 连接**
使用 JDBC Websocket 或 REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
1. driverClass 指定为“com.taosdata.jdbc.ws.WebSocketDriver”
2. jdbcUrl 以“jdbc:TAOS-WS://”开头;
3. 使用 6041 作为连接端口。
对于 WebSocket 连接url 中的配置参数如下:
- user登录 TDengine 用户名,默认值 'root'。
- password用户登录密码默认值 'taosdata'。
- charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。
- batchErrorIgnoretrue在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false不再执行失败 SQL 后的任何语句。默认值为false。
- httpConnectTimeout: 连接超时时间,单位 ms 默认值为 60000。
- messageWaitTimeout: 消息超时时间, 单位 ms 默认值为 60000。
- useSSL: 连接中是否使用 SSL。
**注意**部分配置项比如locale、timezone在 WebSocket 连接中不生效。
**REST 连接**
使用 JDBC REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要:
1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”
2. jdbcUrl 以“jdbc:TAOS-RS://”开头;
3. 使用 6041 作为连接端口。
对于 Websocket 和 REST 连接url 中的配置参数如下:
对于 REST 连接url 中的配置参数如下:
- user登录 TDengine 用户名,默认值 'root'。
- password用户登录密码默认值 'taosdata'。
- batchfetch: true在执行查询时批量拉取结果集false逐行拉取结果集。默认值为false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTPWebSocket 可以使 JDBC REST 连接支持大数据量查询,并提升查询性能。
- charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。
- batchErrorIgnoretrue在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false不再执行失败 SQL 后的任何语句。默认值为false。
- httpConnectTimeout: 连接超时时间,单位 ms 默认值为 60000。
- httpSocketTimeout: socket 超时时间,单位 ms默认值为 60000。仅在 batchfetch 设置为 false 时生效。
- messageWaitTimeout: 消息超时时间, 单位 ms 默认值为 60000。 仅在 batchfetch 设置为 true 时生效。
- httpSocketTimeout: socket 超时时间,单位 ms默认值为 60000。
- useSSL: 连接中是否使用 SSL。
- httpPoolSize: REST 并发请求大小,默认 20。
@ -272,7 +285,7 @@ TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可
properties 中的配置参数如下:
- TSDBDriver.PROPERTY_KEY_USER登录 TDengine 用户名,默认值 'root'。
- TSDBDriver.PROPERTY_KEY_PASSWORD用户登录密码默认值 'taosdata'。
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true在执行查询时批量拉取结果集false逐行拉取结果集。默认值为false。
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true在执行查询时批量拉取结果集false逐行拉取结果集。默认值为false。因历史原因使用 REST 连接时,若设置此参数为 true 会变成 WebSocket 连接。
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNOREtrue在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 sq 了。false不再执行失败 SQL 后的任何语句。默认值为false。
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径Linux OS 上默认值 `/etc/taos`Windows OS 上默认值 `C:/TDengine/cfg`。
- TSDBDriver.PROPERTY_KEY_CHARSET客户端使用的字符集默认值为系统字符集。
@ -280,16 +293,16 @@ properties 中的配置参数如下:
- TSDBDriver.PROPERTY_KEY_TIME_ZONE仅在使用 JDBC 原生连接时生效。 客户端使用的时区默认值为系统当前时区。因为历史的原因我们只支持POSIX标准的部分规范如UTC-8(代表中国上上海), GMT-8Asia/Shanghai 这几种形式。
- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间,单位 ms 默认值为 60000。仅在 REST 连接时生效。
- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间,单位 ms默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms 默认值为 60000。 仅 WebSocket 连接下有效。
- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 WebSocket/REST 连接时生效。
- TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小,默认 20。
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/Websocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。
> **注意**:启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效,对后面的 `use db` 语句切换数据库无效。
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔,单位毫秒,默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数,默认值 3仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 Websocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。
此外对 JDBC 原生连接,通过指定 URL 和 Properties 还可以指定其他参数比如日志级别、SQL 长度等。
@ -1154,7 +1167,7 @@ JDBC 驱动支持标准的 ResultSet 接口,提供了用于读取结果集中
PreparedStatement 允许使用预编译的 SQL 语句,这可以提高性能并提供参数化查询的能力,从而增加安全性。
JDBC 驱动提供了实现 PreparedStatement 接口的两个类:
1. 对应原生连接的 TSDBPreparedStatement
2. 对应 Websocket 连接的 TSWSPreparedStatement
2. 对应 WebSocket 连接的 TSWSPreparedStatement
因 JDBC 标准没有高性能绑定数据的接口TSDBPreparedStatement 和 TSWSPreparedStatement 都新增了一些方法,用来扩展参数绑定能力。
> **注意**:由于 PreparedStatement 继承了 Statement 接口,因此对于这部分重复的接口不再赘述,请参考 Statement 接口中对应描述。
@ -1347,8 +1360,8 @@ JDBC 标准不支持数据订阅,因此本章所有接口都是扩展接口。
- httpConnectTimeout: 创建连接超时参数,单位 ms默认为 5000 ms。仅在 WebSocket 连接下有效。
- messageWaitTimeout: 数据传输超时参数,单位 ms默认为 10000 ms。仅在 WebSocket 连接下有效。
- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 Websocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 Websocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用false: 不启用。默认为 false。
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔,单位毫秒,默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数,默认值 3仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。

View File

@ -18,9 +18,9 @@ import RequestId from "./_request_id.mdx";
## 连接方式
`taos` 提供两种建立连接的方式。一般我们推荐使用 **Websocket 连接**。
`taos` 提供两种建立连接的方式。一般我们推荐使用 **WebSocket 连接**。
- **原生连接**,它通过 TDengine 客户端驱动程序taosc连接 TDengine 运行实例。
- **Websocket 连接**,它通过 taosAdapter 的 Websocket 接口连接 TDengine 运行实例。
- **WebSocket 连接**,它通过 taosAdapter 的 WebSocket 接口连接 TDengine 运行实例。
你可以通过不同的 “特性(即 Cargo 关键字 `features`)” 来指定使用哪种连接器(默认同时支持)。
@ -29,13 +29,13 @@ import RequestId from "./_request_id.mdx";
## 支持的平台
原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。
Websocket 连接支持所有能运行 Rust 的平台。
WebSocket 连接支持所有能运行 Rust 的平台。
## 版本历史
| Rust 连接器版本 | TDengine 版本 | 主要功能 |
| :----------------: | :--------------: | :--------------------------------------------------: |
| v0.12.3 | 3.3.0.0 or later | 优化了 Websocket 查询和插入性能,支持了 VARBINARY 和 GEOMETRY 类型 |
| v0.12.3 | 3.3.0.0 or later | 优化了 WebSocket 查询和插入性能,支持了 VARBINARY 和 GEOMETRY 类型 |
| v0.12.0 | 3.2.3.0 or later | WS 支持压缩。 |
| v0.11.0 | 3.2.0.0 | TMQ 功能优化。 |
| v0.10.0 | 3.1.0.0 | WS endpoint 变更。 |
@ -115,15 +115,15 @@ DSN 描述字符串基本结构如下:
- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接,支持如下驱动名:
- **taos**: 使用 TDengine 连接器驱动,默认是使用 taos 驱动。
- **tmq**: 使用 TMQ 订阅数据。
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。
- **http/ws**: 使用 Websocket 创建连接。
- **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 连接。
- **protocol**: 显示指定以何种方式建立连接,例如:`taos+ws://localhost:6041` 指定以 WebSocket 方式建立连接。
- **http/ws**: 使用 WebSocket 创建连接。
- **https/wss**: 在 WebSocket 连接方式下显示启用 SSL/TLS 连接。
- **username/password**: 用于创建连接的用户名及密码。
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时(`taos://`),原生连接默认为 `localhost:6030`Websocket 连接默认为 `localhost:6041` 。
- **host/port**: 指定创建连接的服务器及端口,当不指定服务器地址及端口时(`taos://`),原生连接默认为 `localhost:6030`WebSocket 连接默认为 `localhost:6041` 。
- **database**: 指定默认连接的数据库名,可选参数。
- **params**:其他可选参数。
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test` 表示使用 Websocket`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
一个完整的 DSN 描述字符串示例如下:`taos+ws://localhost:6041/test` 表示使用 WebSocket`ws`)方式通过 `6041` 端口连接服务器 `localhost`,并指定默认数据库为 `test`。
#### TaosBuilder
TaosBuilder 结构体主要提供了根据 DSN 构建 Taos 对象的方法,还提供了检查连接,以及获取客户端版本号等功能。

View File

@ -14,10 +14,10 @@ import RequestId from "./_request_id.mdx";
Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-python)。
## 连接方式
`taospy`主要提供三种形式的连接器。一般我们推荐使用 **Websocket 连接**。
`taospy`主要提供三种形式的连接器。一般我们推荐使用 **WebSocket 连接**。
- **原生连接**,对应 `taospy` 包的 `taos` 模块。通过 TDengine 客户端驱动程序taosc原生连接 TDengine 实例支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。
- **REST 连接**,对应 `taospy` 包的 `taosrest` 模块。通过 taosAdapter 提供的 HTTP 接口连接 TDengine 实例,不支持 schemaless 和数据订阅等特性。
- **Websocket 连接**,对应 `taos-ws-py` 包,可以选装。通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例WebSocket 连接实现的功能集合和原生连接有少量不同。
- **WebSocket 连接**,对应 `taos-ws-py` 包,可以选装。通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例WebSocket 连接实现的功能集合和原生连接有少量不同。
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)
@ -48,9 +48,9 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con
|2.7.9|数据订阅支持获取消费进度和重置消费进度|
|2.7.8|新增 `execute_many`|
|Python Websocket Connector 版本|主要变化|
|Python WebSocket Connector 版本|主要变化|
|:----------------------------:|:-----:|
|0.3.2|优化 Websocket sql 查询和插入性能,修改 readme 和 文档,修复已知问题|
|0.3.2|优化 WebSocket sql 查询和插入性能,修改 readme 和 文档,修复已知问题|
|0.2.9|已知问题修复|
|0.2.5|1. 数据订阅支持获取消费进度和重置消费进度 <br/> 2. 支持 schemaless <br/> 3. 支持 STMT|
|0.2.4|数据订阅新增取消订阅方法|

View File

@ -14,7 +14,7 @@ Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-conne
## 连接方式
Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。
Node.js 连接器目前仅支持 WebSocket 连接器, 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)
@ -48,7 +48,7 @@ Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter
| 107 | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 |
| 108 | connection has been closed | 连接已经关闭,请检查 Connection 是否关闭后再次使用,或是连接是否正常。 |
| 109 | fetch block data parse fail | 获取到的查询数据,解析失败 |
| 110 | websocket connection has reached its maximum limit | Websocket 连接达到上限 |
| 110 | websocket connection has reached its maximum limit | WebSocket 连接达到上限 |
- [TDengine Node.js Connector Error Code](https://github.com/taosdata/taos-connector-node/blob/main/nodejs/src/common/wsError.ts)
- TDengine 其他功能模块的报错,请参考 [错误码](../../../reference/error-code)
@ -104,7 +104,7 @@ Node.js 连接器目前仅支持 Websocket 连接器, 其通过 taosAdapter
## API 参考
Node.js 连接器(`@tdengine/websocket`, 其通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例。
Node.js 连接器(`@tdengine/websocket`, 其通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例。
### URL 规范

View File

@ -14,7 +14,7 @@ import RequestId from "./_request_id.mdx";
`TDengine.Connector` 提供两种形式的连接器
* **原生连接**,通过 TDengine 客户端驱动程序taosc原生连接 TDengine 实例支持数据写入、查询、数据订阅、schemaless 接口和参数绑定接口等功能。
* **Websocket 连接**,通过 taosAdapter 提供的 Websocket 接口连接 TDengine 实例WebSocket 连接实现的功能集合和原生连接有少量不同。(自 v3.0.1 起)
* **WebSocket 连接**,通过 taosAdapter 提供的 WebSocket 接口连接 TDengine 实例WebSocket 连接实现的功能集合和原生连接有少量不同。(自 v3.0.1 起)
连接方式的详细介绍请参考:[连接方式](../../../develop/connect/#连接方式)

View File

@ -109,7 +109,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
## 版本历史
| taos_odbc版本 | 主要变化 | TDengine 版本 |
| taos_odbc 版本 | 主要变化 | TDengine 版本 |
| :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- |
| v1.1.0 | 1. 支持视图功能;<br/>2. 支持 VARBINARY/GEOMETRY 数据类型; | 3.3.3.0及更高版本 |
| v1.0.2 | 支持 CP1252 字符编码; | 3.2.3.0及更高版本 |
@ -145,7 +145,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
## API 参考
本节按功能分类汇总了 ODBC API关于完整的 ODBC API 参考,请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的ODBC程序员参考页面。
本节按功能分类汇总了 ODBC API关于完整的 ODBC API 参考,请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的 ODBC 程序员参考页面。
### 数据源和驱动程序管理

View File

@ -62,7 +62,7 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器
| **连接管理** | 支持 | 支持 | 支持 |
| **执行 SQL** | 支持 | 支持 | 支持 |
### 使用 Websocket 接口
### 使用 WebSocket 接口
| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** |
| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | --------- |

View File

@ -32,6 +32,8 @@ extern int32_t tsS3PageCacheSize;
extern int32_t tsS3UploadDelaySec;
int32_t s3Init();
int32_t s3Begin();
void s3End();
int32_t s3CheckCfg();
int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);

View File

@ -42,21 +42,21 @@ typedef enum {
} EEncryptScope;
// cluster
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern char tsVersionName[];
extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
//extern char tsAuthCode[];
extern char tsEncryptKey[];
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern char tsVersionName[];
extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
// extern char tsAuthCode[];
extern char tsEncryptKey[];
// common
extern int32_t tsMaxShellConns;
@ -71,6 +71,8 @@ extern int32_t tsTagFilterResCacheSize;
// queue & threads
extern int32_t tsNumOfRpcThreads;
extern int32_t tsNumOfRpcSessions;
extern int32_t tsShareConnLimit;
extern int32_t tsReadTimeout;
extern int32_t tsTimeToGetAvailableConn;
extern int32_t tsKeepAliveIdle;
extern int32_t tsNumOfCommitThreads;
@ -264,8 +266,8 @@ extern bool tsExperimental;
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs);
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl,
SArray *pArgs);
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc);
void taosCleanupCfg();

View File

@ -983,6 +983,7 @@ typedef struct SEpSet {
SEp eps[TSDB_MAX_REPLICA];
} SEpSet;
int32_t tEncodeSEpSet(SEncoder* pEncoder, const SEpSet* pEp);
int32_t tDecodeSEpSet(SDecoder* pDecoder, SEpSet* pEp);
int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);

View File

@ -329,6 +329,7 @@
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_RELEASE, "task-release", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_SCH_MSG)

45
include/libs/azure/az.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_AZURE_H_
#define _TD_AZURE_H_
#include "os.h"
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t azBegin();
void azEnd();
int32_t azCheckCfg();
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
void azDeleteObjectsByPrefix(const char *prefix);
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp);
int32_t azGetObjectsByPrefix(const char *prefix, const char *path);
int32_t azGetObjectToFile(const char *object_name, const char *fileName);
int32_t azDeleteObjects(const char *object_name[], int nobject);
#ifdef __cplusplus
}
#endif
#endif // _TD_AZURE_H_

58
include/libs/tcs/tcs.h Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TCS_H_
#define _TD_TCS_H_
#include "os.h"
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int8_t tsS3Enabled;
extern int8_t tsS3EnabledCfg;
extern int32_t tsS3UploadDelaySec;
extern int32_t tsS3BlockSize;
extern int32_t tsS3BlockCacheSize;
extern int32_t tsS3PageCacheSize;
extern int8_t tsS3StreamEnabled;
int32_t tcsInit();
void tcsUninit();
int32_t tcsCheckCfg();
int32_t tcsPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
int32_t tcsGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
void tcsDeleteObjectsByPrefix(const char *prefix);
int32_t tcsPutObjectFromFile2(const char *file, const char *object, int8_t withcp);
int32_t tcsGetObjectsByPrefix(const char *prefix, const char *path);
int32_t tcsDeleteObjects(const char *object_name[], int nobject);
int32_t tcsGetObjectToFile(const char *object_name, const char *fileName);
#ifdef __cplusplus
}
#endif
#endif // _TD_TCS_H_

View File

@ -63,6 +63,10 @@ typedef struct SRpcHandleInfo {
int8_t forbiddenIp;
int8_t notFreeAhandle;
int8_t compressed;
int64_t seqNum; // msg seq
int64_t qId; // queryId Get from client, other req's qId = -1;
int32_t refIdMgt;
int32_t msgType;
} SRpcHandleInfo;
typedef struct SRpcMsg {
@ -124,9 +128,13 @@ typedef struct SRpcInit {
int32_t connLimitLock;
int32_t timeToGetConn;
int8_t supportBatch; // 0: no batch, 1. batch
int32_t batchSize;
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
void *parent;
int32_t shareConnLimit;
int8_t shareConn; // 0: no share, 1. share
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
int8_t startReadTimer;
int64_t readTimeout; // s
void *parent;
} SRpcInit;
typedef struct {
@ -144,6 +152,7 @@ typedef struct {
SHashObj *args;
SRpcBrokenlinkVal brokenVal;
void (*freeFunc)(const void *arg);
int64_t st;
} SRpcCtx;
int32_t rpcInit();

View File

@ -130,14 +130,15 @@ int taosSetAutoDelFile(char *path);
bool lastErrorIsFileNotExist();
#ifdef BUILD_WITH_RAND_ERR
#define STUB_RAND_NETWORK_ERR(status) \
do { \
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
uint32_t r = taosRand() % tsRandErrDivisor; \
if ((r + 1) <= tsRandErrChance) { \
status = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
} \
} \
#define STUB_RAND_NETWORK_ERR(ret) \
do { \
if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \
uint32_t r = taosRand() % tsRandErrDivisor; \
if ((r + 1) <= tsRandErrChance) { \
ret = TSDB_CODE_RPC_NETWORK_UNAVAIL; \
uError("random network error: %s, %s", tstrerror(ret), __func__); \
} \
} \
while (0)
#else
#define STUB_RAND_NETWORK_ERR(status)

View File

@ -137,6 +137,7 @@ int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosSetSockOpt2(int32_t fd);
int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
@ -159,7 +160,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, st
int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
int32_t taosBlockSIGPIPE();
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t* ip);
int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip);
int32_t taosGetFqdn(char *);
void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr);

View File

@ -94,9 +94,9 @@ int32_t taosGetErrSize();
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
#define TSDB_CODE_RPC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0026)
#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027)
#define TSDB_CODE_RPC_ASYNC_IN_PROCESS TAOS_DEF_ERROR_CODE(0, 0x0028)
#define TSDB_CODE_RPC_NO_STATE TAOS_DEF_ERROR_CODE(0, 0x0029)
#define TSDB_CODE_RPC_STATE_DROPED TAOS_DEF_ERROR_CODE(0, 0x002A)
//common & util
#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //

View File

@ -57,6 +57,7 @@ extern int32_t rpcDebugFlag;
extern int32_t qDebugFlag;
extern int32_t stDebugFlag;
extern int32_t wDebugFlag;
extern int32_t azDebugFlag;
extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;

View File

@ -370,7 +370,10 @@ int32_t openTransporter(const char *user, const char *auth, int32_t numOfThread,
connLimitNum = TMAX(connLimitNum, 10);
connLimitNum = TMIN(connLimitNum, 1000);
rpcInit.connLimitNum = connLimitNum;
rpcInit.shareConnLimit = tsShareConnLimit;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
int32_t code = taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
if (TSDB_CODE_SUCCESS != code) {

View File

@ -410,7 +410,6 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
// int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, NULL, pSendMsg);
if (code) {
doRequestCallback(pRequest, code);
@ -1921,19 +1920,19 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
return NULL;
}
//TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
// const char* db, int dbLen, uint16_t port) {
// char ipStr[TSDB_EP_LEN] = {0};
// char dbStr[TSDB_DB_NAME_LEN] = {0};
// char userStr[TSDB_USER_LEN] = {0};
// char passStr[TSDB_PASSWORD_LEN] = {0};
// TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen,
// const char* db, int dbLen, uint16_t port) {
// char ipStr[TSDB_EP_LEN] = {0};
// char dbStr[TSDB_DB_NAME_LEN] = {0};
// char userStr[TSDB_USER_LEN] = {0};
// char passStr[TSDB_PASSWORD_LEN] = {0};
//
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
//}
// tstrncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
// tstrncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
// tstrncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
// tstrncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
// }
void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
@ -2301,7 +2300,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
(void)snprintf(varDataVal(dst), TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE, "%s",
(*((char*)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
} else {
tscError("doConvertJson error: invalid type:%d", jsonInnerType);
@ -2570,6 +2570,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.readTimeout = tsReadTimeout;
if (TSDB_CODE_SUCCESS != taosVersionStrToInt(version, &(rpcInit.compatibilityVer))) {
tscError("faild to convert taos version from str to int, errcode:%s", terrstr());
goto _OVER;

View File

@ -89,20 +89,8 @@ static void s3DumpCfgByEp(int8_t epIndex) {
int32_t s3CheckCfg() {
int32_t code = 0, lino = 0;
int8_t i = 0;
if (!tsS3Enabled) {
(void)fprintf(stderr, "s3 not configured.\n");
TAOS_RETURN(code);
}
code = s3Begin();
if (code != 0) {
(void)fprintf(stderr, "failed to initialize s3.\n");
TAOS_RETURN(code);
}
for (; i < tsS3EpNum; i++) {
for (int8_t i = 0; i < tsS3EpNum; i++) {
(void)fprintf(stdout, "test s3 ep (%d/%d):\n", i + 1, tsS3EpNum);
s3DumpCfgByEp(i);
@ -192,7 +180,7 @@ int32_t s3CheckCfg() {
(void)fprintf(stdout, "=================================================================\n");
}
s3End();
// s3End();
TAOS_RETURN(code);
}
@ -1529,6 +1517,8 @@ void s3EvictCache(const char *path, long object_size) {}
#include "cos_http_io.h"
#include "cos_log.h"
int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); }
int32_t s3Init() {
if (cos_http_io_initialize(NULL, 0) != COSE_OK) {
return -1;
@ -1967,6 +1957,10 @@ long s3Size(const char *object_name) {
#else
int32_t s3Init() { return 0; }
int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); }
void s3End() {}
int32_t s3CheckCfg() { return 0; }
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }

View File

@ -160,7 +160,11 @@ int32_t startRsync() {
code = system(cmd);
if (code != 0) {
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
if (errno == 0) {
return 0;
} else {
code = TAOS_SYSTEM_ERROR(errno);
}
} else {
uInfo("[rsync] cmd:%s start server successful", cmd);
}
@ -358,4 +362,4 @@ int32_t deleteRsync(const char* id) {
uDebug("[rsync] delete data:%s successful", id);
return 0;
}
}

View File

@ -56,6 +56,8 @@ int32_t tsShellActivityTimer = 3; // second
// queue & threads
int32_t tsNumOfRpcThreads = 1;
int32_t tsNumOfRpcSessions = 30000;
int32_t tsShareConnLimit = 8;
int32_t tsReadTimeout = 900;
int32_t tsTimeToGetAvailableConn = 500000;
int32_t tsKeepAliveIdle = 60;
@ -179,12 +181,12 @@ int32_t tsRedirectFactor = 2;
int32_t tsRedirectMaxPeriod = 1000;
int32_t tsMaxRetryWaitTime = 10000;
bool tsUseAdapter = false;
int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 10; // seconds
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 10; // seconds
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY;
char* tsSlowLogScopeString = "query";
char *tsSlowLogScopeString = "query";
int32_t tsSlowLogMaxLen = 4096;
int32_t tsTimeSeriesThreshold = 50;
bool tsMultiResultFunctionStarReturnTags = false;
@ -306,6 +308,7 @@ char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {"<appid>"};
int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false;
int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false};
int8_t tsS3Ablob = false;
int8_t tsS3StreamEnabled = false;
int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true};
@ -322,7 +325,6 @@ int32_t tsMaxTsmaNum = 3;
int32_t tsMaxTsmaCalcDelay = 600;
int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d
#define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \
if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \
@ -361,7 +363,7 @@ static int32_t taosSplitS3Cfg(SConfig *pCfg, const char *name, char gVarible[TSD
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, name);
char *strDup = NULL;
if ((strDup = taosStrdup(pItem->str))== NULL){
if ((strDup = taosStrdup(pItem->str)) == NULL) {
code = terrno;
goto _exit;
}
@ -435,6 +437,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
}
tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL);
tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL);
tsS3Ablob = (strstr(tsS3Endpoint[i], ".blob.core.windows.net") != NULL);
}
if (tsS3BucketName[0] != '<') {
@ -450,7 +453,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
struct SConfig *taosGetCfg() { return tsCfg; }
struct SConfig *taosGetCfg() {
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) {
@ -539,6 +544,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "azDebugFlag", azDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -572,7 +578,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(
cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
@ -600,16 +607,23 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(
cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH));
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE));
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000);
TAOS_CHECK_RETURN(
cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
@ -865,8 +879,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem = cfgGetItem(pCfg, "numOfRpcThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfRpcThreads = numOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS);
pItem->i32 = tsNumOfRpcThreads;
pItem->stype = stype;
}
@ -878,6 +891,20 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "shareConnLimit");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512);
pItem->i32 = tsShareConnLimit;
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "readTimeout");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7);
pItem->i32 = tsReadTimeout;
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "timeToGetAvailableConn");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
@ -1039,6 +1066,9 @@ static int32_t taosSetServerLogCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "wDebugFlag");
wDebugFlag = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "azDebugFlag");
azDebugFlag = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "sDebugFlag");
sDebugFlag = pItem->i32;
@ -1083,9 +1113,9 @@ int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope) {
int32_t slowScope = 0;
char* scope = NULL;
char *tmp = NULL;
while((scope = strsep(&pScopeStr, "|")) != NULL){
char *scope = NULL;
char *tmp = NULL;
while ((scope = strsep(&pScopeStr, "|")) != NULL) {
taosMemoryFreeClear(tmp);
tmp = taosStrdup(scope);
if (tmp == NULL) {
@ -1142,13 +1172,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "firstEp");
SEp firstEp = {0};
SEp firstEp = {0};
TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &firstEp));
(void)snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "firstEp", tsFirst, pItem->stype, true));
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "secondEp");
SEp secondEp = {0};
SEp secondEp = {0};
TAOS_CHECK_RETURN(taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? defaultFirstEp : pItem->str, &secondEp));
(void)snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
TAOS_CHECK_RETURN(cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype, true));
@ -1252,6 +1282,12 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
tsNumOfRpcSessions = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
tsShareConnLimit = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
tsReadTimeout = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
tsTimeToGetAvailableConn = pItem->i32;
@ -1353,6 +1389,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfRpcSessions");
tsNumOfRpcSessions = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "shareConnLimit");
tsShareConnLimit = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "readTimeout");
tsReadTimeout = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timeToGetAvailableConn");
tsTimeToGetAvailableConn = pItem->i32;
@ -1651,8 +1693,8 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag);
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SConfig *pCfg = NULL;
if (tsCfg == NULL) {
@ -1724,7 +1766,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *
TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit);
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit);
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit);
TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit);
if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
(void)printf("failed to load cfg since %s\n", tstrerror(code));
@ -1753,7 +1795,7 @@ _exit:
static int32_t taosCheckGlobalCfg() {
uint32_t ipv4 = 0;
int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4);
int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4);
if (code) {
uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code));
TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR);
@ -1858,7 +1900,7 @@ typedef struct {
static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) {
int32_t code = TSDB_CODE_CFG_NOT_FOUND;
char *name = pItem->name;
char *name = pItem->name;
for (int32_t d = 0; d < optionSize; ++d) {
const char *optName = pOptions[d].optionName;
if (strcasecmp(name, optName) != 0) continue;
@ -1951,13 +1993,14 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function
static OptionNameAndVar debugOptions[] = {
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
{"wDebugFlag", &wDebugFlag}, {"sDebugFlag", &sDebugFlag}, {"tsdbDebugFlag", &tsdbDebugFlag},
{"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"udfDebugFlag", &udfDebugFlag},
{"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag},
{"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag},
{"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag},
{"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug},
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
{"wDebugFlag", &wDebugFlag}, {"azDebugFlag", &azDebugFlag}, {"sDebugFlag", &sDebugFlag},
{"tsdbDebugFlag", &tsdbDebugFlag}, {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag},
{"udfDebugFlag", &udfDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag},
{"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
{"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
{"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag},
{"tqClientDebug", &tqClientDebug},
};
static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},
@ -2053,8 +2096,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
}
case 'f': {
if (strcasecmp("fqdn", name) == 0) {
SConfigItem* pFqdnItem = cfgGetItem(pCfg, "fqdn");
SConfigItem* pServerPortItem = cfgGetItem(pCfg, "serverPort");
SConfigItem *pFqdnItem = cfgGetItem(pCfg, "fqdn");
SConfigItem *pServerPortItem = cfgGetItem(pCfg, "serverPort");
SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
if (pFqdnItem == NULL || pServerPortItem == NULL || pFirstEpItem == NULL) {
uError("failed to get fqdn or serverPort or firstEp from cfg");
@ -2069,7 +2112,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
char defaultFirstEp[TSDB_EP_LEN] = {0};
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
SEp firstEp = {0};
SEp firstEp = {0};
TAOS_CHECK_GOTO(
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino,
_out);
@ -2109,8 +2152,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
}
case 'l': {
if (strcasecmp("locale", name) == 0) {
SConfigItem* pLocaleItem = cfgGetItem(pCfg, "locale");
SConfigItem* pCharsetItem = cfgGetItem(pCfg, "charset");
SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale");
SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset");
if (pLocaleItem == NULL || pCharsetItem == NULL) {
uError("failed to get locale or charset from cfg");
code = TSDB_CODE_CFG_NOT_FOUND;
@ -2183,7 +2226,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
char defaultFirstEp[TSDB_EP_LEN] = {0};
(void)snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
SEp firstEp = {0};
SEp firstEp = {0};
TAOS_CHECK_GOTO(
taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp), &lino,
_out);
@ -2314,7 +2357,7 @@ int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg,
// NOTE: set all command does not change the tmrDebugFlag
static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
if (flag < 0) TAOS_RETURN(TSDB_CODE_INVALID_PARA);
if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore
if (flag == 0) TAOS_RETURN(TSDB_CODE_SUCCESS); // just ignore
SArray *noNeedToSetVars = NULL;
SConfigItem *pItem = NULL;
@ -2335,6 +2378,7 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&azDebugFlag, "azDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars);
taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars);

View File

@ -24,6 +24,7 @@
#include "jemalloc/jemalloc.h"
#endif
#include "dmUtil.h"
#include "tcs.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -182,7 +183,6 @@ static void dmSetSignalHandle() {
}
#endif
}
extern bool generateNewMeta;
extern bool generateNewMeta;
@ -331,10 +331,9 @@ static int32_t dmCheckS3() {
int32_t code = 0;
SConfig *pCfg = taosGetCfg();
cfgDumpCfgS3(pCfg, 0, true);
#if defined(USE_S3)
extern int32_t s3CheckCfg();
code = s3CheckCfg();
#if defined(USE_S3)
code = tcsCheckCfg();
#endif
return code;
}

View File

@ -74,7 +74,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_RETRIEVE_IP_WHITE,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 0,
@ -249,7 +249,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_STATUS,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 0,
@ -322,7 +322,7 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
SRpcMsg rpcMsg = {.pCont = pHead,
.contLen = contLen,
.msgType = TDMT_MND_NOTIFY,
.info.ahandle = (void *)0x9527,
.info.ahandle = 0,
.info.notFreeAhandle = 1,
.info.refId = 0,
.info.noResp = 1,

View File

@ -47,7 +47,6 @@ static void *dmStatusThreadFp(void *param) {
return NULL;
}
extern SMonVloadInfo tsVinfo;
static void *dmStatusInfoThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
@ -73,19 +72,6 @@ static void *dmStatusInfoThreadFp(void *param) {
}
}
}
dDebug("begin to lock status info when thread exit");
if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to lock status info lock");
return NULL;
}
if (tsVinfo.pVloads != NULL) {
taosArrayDestroy(tsVinfo.pVloads);
tsVinfo.pVloads = NULL;
}
if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) {
dError("failed to unlock status info lock");
return NULL;
}
return NULL;
}

View File

@ -1,7 +1,7 @@
aux_source_directory(src IMPLEMENT_SRC)
add_library(dnode STATIC ${IMPLEMENT_SRC})
target_link_libraries(
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw tcs
)
IF (TD_ENTERPRISE)

View File

@ -20,6 +20,7 @@
#include "libs/function/tudf.h"
#include "tgrant.h"
#include "tcompare.h"
#include "tcs.h"
#include "tanal.h"
// clang-format on
@ -98,9 +99,9 @@ static bool dmDataSpaceAvailable() {
static int32_t dmCheckDiskSpace() {
// availability
int32_t code = 0;
code = osUpdate();
if(code != 0) {
code = 0; // ignore the error, just log it
code = osUpdate();
if (code != 0) {
code = 0; // ignore the error, just log it
dError("failed to update os info since %s", tstrerror(code));
}
if (!dmDataSpaceAvailable()) {
@ -163,13 +164,6 @@ static int32_t dmCheckDataDirVersionWrapper() {
}
return 0;
}
#if defined(USE_S3)
extern int32_t s3Begin();
extern void s3End();
extern int8_t tsS3Enabled;
#endif
int32_t dmInit() {
dInfo("start to init dnode env");
@ -187,7 +181,7 @@ int32_t dmInit() {
if ((code = dmInitDnode(dmInstance())) != 0) return code;
if ((code = InitRegexCache() != 0)) return code;
#if defined(USE_S3)
if ((code = s3Begin()) != 0) return code;
if ((code = tcsInit()) != 0) return code;
#endif
dInfo("dnode env is initialized");
@ -221,7 +215,7 @@ void dmCleanup() {
DestroyRegexCache();
#if defined(USE_S3)
s3End();
tcsUninit();
#endif
dInfo("dnode env is cleaned up");

View File

@ -219,6 +219,7 @@ int32_t dmInitVars(SDnode *pDnode) {
return 0;
}
extern SMonVloadInfo tsVinfo;
void dmClearVars(SDnode *pDnode) {
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
@ -254,6 +255,25 @@ void dmClearVars(SDnode *pDnode) {
(void)taosThreadRwlockUnlock(&pData->lock);
(void)taosThreadRwlockDestroy(&pData->lock);
dDebug("begin to lock status info when thread exit");
if (taosThreadMutexLock(&pData->statusInfolock) != 0) {
dError("failed to lock status info lock");
return;
}
if (tsVinfo.pVloads != NULL) {
taosArrayDestroy(tsVinfo.pVloads);
tsVinfo.pVloads = NULL;
}
if (taosThreadMutexUnlock(&pData->statusInfolock) != 0) {
dError("failed to unlock status info lock");
return;
}
if (taosThreadMutexDestroy(&pData->statusInfolock) != 0) {
dError("failed to destroy status info lock");
}
memset(&pData->statusInfolock, 0, sizeof(pData->statusInfolock));
(void)taosThreadMutexDestroy(&pDnode->mutex);
memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
}

View File

@ -16,8 +16,8 @@
#define _DEFAULT_SOURCE
#include "dmMgmt.h"
#include "qworker.h"
#include "tversion.h"
#include "tanal.h"
#include "tversion.h"
static inline void dmSendRsp(SRpcMsg *pMsg) {
if (rpcSendResponse(pMsg) != 0) {
@ -267,7 +267,7 @@ _OVER:
}
if (IsReq(pRpc)) {
SRpcMsg rsp = {.code = code, .info = pRpc->info};
SRpcMsg rsp = {.code = code, .info = pRpc->info, .msgType = pRpc->msgType + 1};
if (code == TSDB_CODE_MNODE_NOT_FOUND) {
dmBuildMnodeRedirectRsp(pDnode, &rsp);
}
@ -411,16 +411,19 @@ int32_t dmInitClient(SDnode *pDnode) {
rpcInit.noDelayFp = rpcNoDelayMsg;
int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3) / 2;
int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3);
connLimitNum = TMAX(connLimitNum, 10);
connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 2;
rpcInit.shareConn = 1;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.notWaitAvaliableConn = 0;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
@ -466,8 +469,10 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 2;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 0;
rpcInit.readTimeout = 0;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
@ -514,8 +519,11 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
rpcInit.connLimitNum = connLimitNum;
rpcInit.connLimitLock = 1;
rpcInit.supportBatch = 1;
rpcInit.batchSize = 8 * 1024;
rpcInit.shareConnLimit = tsShareConnLimit * 8;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
rpcInit.startReadTimer = 1;
rpcInit.readTimeout = tsReadTimeout;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);
}
@ -569,6 +577,7 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.parent = pDnode;
rpcInit.compressSize = tsCompressMsgSize;
rpcInit.shareConnLimit = tsShareConnLimit * 16;
if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) {
dError("failed to convert version string:%s to int", version);

View File

@ -237,7 +237,7 @@ static void mndPullupGrant(SMnode *pMnode) {
.pCont = pReq,
.contLen = contLen,
.info.notFreeAhandle = 1,
.info.ahandle = (void *)0x9527};
.info.ahandle = 0};
// TODO check return value
if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) {
mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__);

View File

@ -30,9 +30,9 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
(void)qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg);
}
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;
int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo *pInfo) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;
SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb, .pWorkerCb = pInfo->workerCb};
@ -67,26 +67,25 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg, SQueueInfo* pInfo) {
return code;
}
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void* p) {
static FORCE_INLINE void mnodeFreeSBatchRspMsg(void *p) {
if (NULL == p) {
return;
}
SBatchRspMsg* pRsp = (SBatchRspMsg*)p;
SBatchRspMsg *pRsp = (SBatchRspMsg *)p;
rpcFreeCont(pRsp->msg);
}
int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t rspSize = 0;
SBatchReq batchReq = {0};
SBatchMsg req = {0};
int32_t code = 0;
int32_t rspSize = 0;
SBatchReq batchReq = {0};
SBatchMsg req = {0};
SBatchRspMsg rsp = {0};
SBatchRsp batchRsp = {0};
SRpcMsg reqMsg = *pMsg;
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
SBatchRsp batchRsp = {0};
SRpcMsg reqMsg = *pMsg;
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -94,7 +93,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
goto _exit;
}
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
int32_t msgNum = taosArrayGetSize(batchReq.pMsgs);
if (msgNum >= MAX_META_MSG_IN_BATCH) {
code = TSDB_CODE_INVALID_MSG;
mError("too many msgs %d in mnode batch meta req", msgNum);
@ -108,7 +107,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
}
for (int32_t i = 0; i < msgNum; ++i) {
SBatchMsg* req = taosArrayGet(batchReq.pMsgs, i);
SBatchMsg *req = taosArrayGet(batchReq.pMsgs, i);
reqMsg.msgType = req->msgType;
reqMsg.pCont = req->msg;

View File

@ -4,7 +4,7 @@ aux_source_directory(. MNODE_ARBGROUP_TEST_SRC)
add_executable(arbgroupTest ${MNODE_ARBGROUP_TEST_SRC})
target_link_libraries(
arbgroupTest
PRIVATE dnode nodes planner gtest qcom
PRIVATE dnode nodes planner gtest qcom tcs
)
add_test(

View File

@ -119,6 +119,7 @@ if (${BUILD_CONTRIB})
vnode
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
@ -129,6 +130,7 @@ else()
vnode
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
@ -164,6 +166,7 @@ target_link_libraries(
PUBLIC tdb
PUBLIC audit
PUBLIC crypt
PUBLIC tcs
# PUBLIC bdb
# PUBLIC scalar

View File

@ -12,8 +12,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "functionMgt.h"
#include "tcs.h"
#include "tsdb.h"
#include "tsdbDataFileRW.h"
#include "tsdbIter.h"
@ -1258,7 +1258,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
}
if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) {
SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID};
SLastCol lastColTmp = {
.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID};
if ((code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, &lastColTmp)) != TSDB_CODE_SUCCESS) {
tsdbError("tsdb/cache: vgId:%d, put rocks failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino,
tstrerror(code));
@ -1705,8 +1706,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA
if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) {
code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0);
if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
tstrerror(code));
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code));
taosMemoryFreeClear(pToFree);
TAOS_CHECK_EXIT(code);
}
@ -3520,7 +3520,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
TAOS_CHECK_RETURN(s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock));
TAOS_CHECK_RETURN(tcsGetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock));
tsdbTrace("block:%p load from s3", *ppBlock);

View File

@ -14,7 +14,7 @@
*/
#include "tsdbFile2.h"
#include "cos.h"
#include "tcs.h"
#include "vnd.h"
// to_json
@ -318,7 +318,7 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
}
*(dot + 1) = 0;
s3DeleteObjectsByPrefix(object_name_prefix);
tcsDeleteObjectsByPrefix(object_name_prefix);
// remove local last chunk file
dot = strrchr(lc_path, '.');

View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "crypt.h"
#include "tcs.h"
#include "tsdb.h"
#include "tsdbDef.h"
#include "vnd.h"
@ -391,7 +391,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
code = tcsGetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
TSDB_CHECK_CODE(code, lino, _exit);
memcpy(buf + n, pBlock, nRead);

View File

@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "tcs.h"
#include "tsdb.h"
#include "tsdbFS2.h"
#include "vnd.h"
@ -426,35 +426,6 @@ static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3Kee
}
}
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
// TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) {
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
char *object_name = taosDirEntryBaseName(fname);
TAOS_CHECK_GOTO(s3PutObjectFromFile2(from->fname, object_name, 1), &lino, _exit);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
}
if (taosCloseFile(&fdFrom) != 0) {
tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));
}
return code;
}
static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
int32_t code = 0;
int32_t lino = 0;
@ -519,7 +490,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - fobj->f->lcn);
TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
}
// copy last chunk
@ -618,7 +589,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - 1);
TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
TAOS_CHECK_GOTO(tcsPutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
}
// copy last chunk
@ -741,8 +712,6 @@ _exit:
int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
int32_t code = 0;
extern int8_t tsS3EnabledCfg;
int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE);
if (expired && tsS3Enabled) {
tsdbWarn("s3 grant expired: %d", expired);

View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "sync.h"
#include "tcs.h"
#include "tsdb.h"
#include "vnd.h"
@ -327,7 +327,7 @@ void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) {
if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) {
char vnode_prefix[TSDB_FILENAME_LEN];
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId);
s3DeleteObjectsByPrefix(vnode_prefix);
tcsDeleteObjectsByPrefix(vnode_prefix);
}
}

View File

@ -22,4 +22,6 @@ add_subdirectory(stream)
add_subdirectory(planner)
add_subdirectory(qworker)
add_subdirectory(geometry)
add_subdirectory(command)
add_subdirectory(command)
add_subdirectory(azure)
add_subdirectory(tcs)

View File

@ -0,0 +1,32 @@
#if(${TD_LINUX})
aux_source_directory(src AZ_SRC)
add_library(az STATIC ${AZ_SRC})
if(${BUILD_S3})
add_definitions(-DUSE_S3)
target_link_libraries(
az
PUBLIC _azure_sdk
PUBLIC crypt
)
endif()
target_include_directories(
az
PUBLIC "${TD_SOURCE_DIR}/include/libs/azure"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
az
PUBLIC cjson
PUBLIC os
PUBLIC util
PUBLIC common
)
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})
#endif(${TD_LINUX})

View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_AZ_INT_H_
#define _TD_AZ_INT_H_
#include "os.h"
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
// clang-format off
#define azFatal(...) { if (azDebugFlag & DEBUG_FATAL) { taosPrintLog("AZR FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define azError(...) { if (azDebugFlag & DEBUG_ERROR) { taosPrintLog("AZR ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define azWarn(...) { if (azDebugFlag & DEBUG_WARN) { taosPrintLog("AZR WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define azInfo(...) { if (azDebugFlag & DEBUG_INFO) { taosPrintLog("AZR ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define azDebug(...) { if (azDebugFlag & DEBUG_DEBUG) { taosPrintLog("AZR ", DEBUG_DEBUG, azDebugFlag, __VA_ARGS__); }}
#define azTrace(...) { if (azDebugFlag & DEBUG_TRACE) { taosPrintLog("AZR ", DEBUG_TRACE, azDebugFlag, __VA_ARGS__); }}
// clang-format on
#ifdef __cplusplus
}
#endif
#endif // _TD_AZ_INT_H_

View File

@ -0,0 +1,191 @@
#pragma once
#include "azure/storage/blobs/blob_options.hpp"
#include <azure/core/io/body_stream.hpp>
#include <map>
#include <memory>
#include <type_traits>
namespace Azure {
namespace Storage {
namespace Blobs {
namespace _detail {
enum class AvroDatumType {
String,
Bytes,
Int,
Long,
Float,
Double,
Bool,
Null,
Record,
Enum,
Array,
Map,
Union,
Fixed,
};
class AvroStreamReader final {
public:
// position of a vector that lives through vector resizing
struct ReaderPos final {
const std::vector<uint8_t>* BufferPtr = nullptr;
size_t Offset = 0;
};
explicit AvroStreamReader(Core::IO::BodyStream& stream) : m_stream(&stream), m_pos{&m_streambuffer, 0} {}
AvroStreamReader(const AvroStreamReader&) = delete;
AvroStreamReader& operator=(const AvroStreamReader&) = delete;
int64_t ParseInt(const Core::Context& context);
void Advance(size_t n, const Core::Context& context);
// Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes
// available in m_streambuffer;
size_t Preload(size_t n, const Core::Context& context);
size_t TryPreload(size_t n, const Core::Context& context);
// discards data that's before m_pos
void Discard();
private:
size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; }
private:
Core::IO::BodyStream* m_stream;
std::vector<uint8_t> m_streambuffer;
ReaderPos m_pos;
friend class AvroDatum;
};
class AvroSchema final {
public:
static const AvroSchema StringSchema;
static const AvroSchema BytesSchema;
static const AvroSchema IntSchema;
static const AvroSchema LongSchema;
static const AvroSchema FloatSchema;
static const AvroSchema DoubleSchema;
static const AvroSchema BoolSchema;
static const AvroSchema NullSchema;
static AvroSchema RecordSchema(std::string name, const std::vector<std::pair<std::string, AvroSchema>>& fieldsSchema);
static AvroSchema ArraySchema(AvroSchema elementSchema);
static AvroSchema MapSchema(AvroSchema elementSchema);
static AvroSchema UnionSchema(std::vector<AvroSchema> schemas);
static AvroSchema FixedSchema(std::string name, int64_t size);
const std::string& Name() const { return m_name; }
AvroDatumType Type() const { return m_type; }
const std::vector<std::string>& FieldNames() const { return m_status->m_keys; }
AvroSchema ItemSchema() const { return m_status->m_schemas[0]; }
const std::vector<AvroSchema>& FieldSchemas() const { return m_status->m_schemas; }
size_t Size() const { return static_cast<size_t>(m_status->m_size); }
private:
explicit AvroSchema(AvroDatumType type) : m_type(type) {}
private:
AvroDatumType m_type;
std::string m_name;
struct SharedStatus {
std::vector<std::string> m_keys;
std::vector<AvroSchema> m_schemas;
int64_t m_size = 0;
};
std::shared_ptr<SharedStatus> m_status;
};
class AvroDatum final {
public:
AvroDatum() : m_schema(AvroSchema::NullSchema) {}
explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {}
void Fill(AvroStreamReader& reader, const Core::Context& context);
void Fill(AvroStreamReader::ReaderPos& data);
const AvroSchema& Schema() const { return m_schema; }
template <class T>
T Value() const;
struct StringView {
const uint8_t* Data = nullptr;
size_t Length = 0;
};
private:
AvroSchema m_schema;
AvroStreamReader::ReaderPos m_data;
};
using AvroMap = std::map<std::string, AvroDatum>;
class AvroRecord final {
public:
bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); }
const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); }
AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); }
const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); }
AvroDatum& FieldAt(size_t i) { return m_values.at(i); }
private:
size_t FindField(const std::string& key) const {
auto i = find(m_keys->begin(), m_keys->end(), key);
return i - m_keys->begin();
}
const std::vector<std::string>* m_keys = nullptr;
std::vector<AvroDatum> m_values;
friend class AvroDatum;
};
class AvroObjectContainerReader final {
public:
explicit AvroObjectContainerReader(Core::IO::BodyStream& stream);
bool End() const { return m_eof; }
// Calling Next() will invalidates the previous AvroDatum returned by this function and all
// AvroDatums propagated from there.
AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); }
private:
AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context);
private:
std::unique_ptr<AvroStreamReader> m_reader;
std::unique_ptr<AvroSchema> m_objectSchema;
std::string m_syncMarker;
int64_t m_remainingObjectInCurrentBlock = 0;
bool m_eof = false;
};
class AvroStreamParser final : public Core::IO::BodyStream {
public:
explicit AvroStreamParser(std::unique_ptr<Azure::Core::IO::BodyStream> inner,
std::function<void(int64_t, int64_t)> progressCallback,
std::function<void(BlobQueryError)> errorCallback)
: m_inner(std::move(inner)),
m_parser(*m_inner),
m_progressCallback(std::move(progressCallback)),
m_errorCallback(std::move(errorCallback)) {}
int64_t Length() const override { return -1; }
void Rewind() override { this->m_inner->Rewind(); }
private:
size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override;
private:
std::unique_ptr<Azure::Core::IO::BodyStream> m_inner;
AvroObjectContainerReader m_parser;
std::function<void(int64_t, int64_t)> m_progressCallback;
std::function<void(BlobQueryError)> m_errorCallback;
AvroDatum::StringView m_parserBuffer;
};
} // namespace _detail
} // namespace Blobs
} // namespace Storage
} // namespace Azure

View File

@ -0,0 +1,260 @@
#pragma once
#include "azure/storage/blobs/blob_client.hpp"
#include <cstdint>
#include <string>
#include <vector>
namespace Azure {
namespace Storage {
namespace Files {
namespace DataLake {
class FileClient;
}
} // namespace Files
} // namespace Storage
} // namespace Azure
namespace Azure {
namespace Storage {
namespace Blobs {
/**
* @brief The TDBlockBlobClient allows you to manipulate Azure Storage block blobs.
*
* Block blobs let you upload large blobs efficiently. Block blobs are comprised of blocks, each
* of which is identified by a block ID. You create or modify a block blob by writing a set of
* blocks and committing them by their block IDs. Each block can be a different size.
*
* When you upload a block to a blob in your storage account, it is associated with the specified
* block blob, but it does not become part of the blob until you commit a list of blocks that
* includes the new block's ID. New blocks remain in an uncommitted state until they are
* specifically committed or discarded. Writing a block does not update the last modified time of
* an existing blob.
*/
class TDBlockBlobClient final : public BlobClient {
public:
/**
* @brief Initialize a new instance of TDBlockBlobClient.
*
* @param connectionString A connection string includes the authentication information required
* for your application to access data in an Azure Storage account at runtime.
* @param blobContainerName The name of the container containing this blob.
* @param blobName The name of this blob.
* @param options Optional client options that define the transport pipeline policies for
* authentication, retries, etc., that are applied to every request.
* @return A new TDBlockBlobClient instance.
*/
static TDBlockBlobClient CreateFromConnectionString(const std::string& connectionString,
const std::string& blobContainerName, const std::string& blobName,
const BlobClientOptions& options = BlobClientOptions());
/**
* @brief Initialize a new instance of TDBlockBlobClient.
*
* @param blobUrl A URL
* referencing the blob that includes the name of the account, the name of the container, and
* the name of the blob.
* @param credential The shared key credential used to sign
* requests.
* @param options Optional client options that define the transport pipeline
* policies for authentication, retries, etc., that are applied to every request.
*/
explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<StorageSharedKeyCredential> credential,
const BlobClientOptions& options = BlobClientOptions());
/**
* @brief Initialize a new instance of TDBlockBlobClient.
*
* @param blobUrl A URL
* referencing the blob that includes the name of the account, the name of the container, and
* the name of the blob.
* @param credential The token credential used to sign requests.
* @param options Optional client options that define the transport pipeline policies for
* authentication, retries, etc., that are applied to every request.
*/
explicit TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<Core::Credentials::TokenCredential> credential,
const BlobClientOptions& options = BlobClientOptions());
/**
* @brief Initialize a new instance of TDBlockBlobClient.
*
* @param blobUrl A URL
* referencing the blob that includes the name of the account, the name of the container, and
* the name of the blob, and possibly also a SAS token.
* @param options Optional client
* options that define the transport pipeline policies for authentication, retries, etc., that
* are applied to every request.
*/
explicit TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options = BlobClientOptions());
/**
* @brief Initializes a new instance of the TDBlockBlobClient class with an identical URL
* source but the specified snapshot timestamp.
*
* @param snapshot The snapshot
* identifier.
* @return A new TDBlockBlobClient instance.
* @remarks Pass empty string to remove the snapshot returning the base blob.
*/
TDBlockBlobClient WithSnapshot(const std::string& snapshot) const;
/**
* @brief Creates a clone of this instance that references a version ID rather than the base
* blob.
*
* @param versionId The version ID returning a URL to the base blob.
* @return A new TDBlockBlobClient instance.
* @remarks Pass empty string to remove the version ID returning the base blob.
*/
TDBlockBlobClient WithVersionId(const std::string& versionId) const;
/**
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
* an existing block blob overwrites any existing metadata on the blob.
*
* @param content A BodyStream containing the content to upload.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A UploadBlockBlobResult describing the state of the updated block blob.
*/
Azure::Response<Models::UploadBlockBlobResult> Upload(
Azure::Core::IO::BodyStream& content, const UploadBlockBlobOptions& options = UploadBlockBlobOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
* an existing block blob overwrites any existing metadata on the blob.
*
* @param buffer A memory buffer containing the content to upload.
* @param bufferSize Size of the memory buffer.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A UploadBlockBlobFromResult describing the state of the updated block blob.
*/
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
const uint8_t* buffer, size_t bufferSize,
const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Creates a new block blob, or updates the content of an existing block blob. Updating
* an existing block blob overwrites any existing metadata on the blob.
*
* @param fileName A file containing the content to upload.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A UploadBlockBlobFromResult describing the state of the updated block blob.
*/
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
const std::string& fileName, const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
Azure::Response<Models::UploadBlockBlobFromResult> UploadFrom(
const std::string& fileName, int64_t offset, int64_t size,
const UploadBlockBlobFromOptions& options = UploadBlockBlobFromOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Creates a new Block Blob where the contents of the blob are read from a given URL.
*
* @param sourceUri Specifies the URL of the source blob.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A UploadBlockBlobFromUriResult describing the state of the updated block blob.
*/
Azure::Response<Models::UploadBlockBlobFromUriResult> UploadFromUri(
const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options = UploadBlockBlobFromUriOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Creates a new block as part of a block blob's staging area to be eventually
* committed via the CommitBlockList operation.
*
* @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the
* string must be less than or equal to 64 bytes in size.
* @param content A BodyStream containing the content to upload.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A StageBlockResult describing the state of the updated block.
*/
Azure::Response<Models::StageBlockResult> StageBlock(
const std::string& blockId, Azure::Core::IO::BodyStream& content,
const StageBlockOptions& options = StageBlockOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Creates a new block to be committed as part of a blob where the contents are read from
* the sourceUri.
*
* @param blockId A valid Base64 string value that identifies the block. Prior to encoding, the
* string must be less than or equal to 64 bytes in size.
* @param sourceUri Specifies the uri of the source
* blob. The value may be a uri of up to 2 KB in length that specifies a blob. The source blob
* must either be public or must be authenticated via a shared access signature. If the source
* blob is public, no authentication is required to perform the operation.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A StageBlockFromUriResult describing the state of the updated block blob.
*/
Azure::Response<Models::StageBlockFromUriResult> StageBlockFromUri(
const std::string& blockId, const std::string& sourceUri,
const StageBlockFromUriOptions& options = StageBlockFromUriOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Writes a blob by specifying the list of block IDs that make up the blob. In order to
* be written as part of a blob, a block must have been successfully written to the server in a
* prior StageBlock operation. You can call CommitBlockList to update a blob by uploading only
* those blocks that have changed, then committing the new and existing blocks together. You can
* do this by specifying whether to commit a block from the committed block list or from the
* uncommitted block list, or to commit the most recently uploaded version of the block,
* whichever list it may belong to.
*
* @param blockIds Base64 encoded block IDs to indicate that make up the blob.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A CommitBlobBlockListResult describing the state of the updated block blob.
*/
Azure::Response<Models::CommitBlockListResult> CommitBlockList(
const std::vector<std::string>& blockIds, const CommitBlockListOptions& options = CommitBlockListOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Retrieves the list of blocks that have been uploaded as part of a block blob. There
* are two block lists maintained for a blob. The Committed Block list has blocks that have been
* successfully committed to a given blob with CommitBlockList. The Uncommitted Block list has
* blocks that have been uploaded for a blob using StageBlock, but that have not yet been
* committed.
*
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A GetBlobBlockListResult describing requested block list.
*/
Azure::Response<Models::GetBlockListResult> GetBlockList(
const GetBlockListOptions& options = GetBlockListOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
/**
* @brief Returns the result of a query against the blob.
*
* @param querySqlExpression The query expression in SQL.
* @param options Optional parameters to execute this function.
* @param context Context for cancelling long running operations.
* @return A QueryBlobResult describing the query result.
*/
Azure::Response<Models::QueryBlobResult> Query(const std::string& querySqlExpression,
const QueryBlobOptions& options = QueryBlobOptions(),
const Azure::Core::Context& context = Azure::Core::Context()) const;
explicit TDBlockBlobClient(BlobClient blobClient);
private:
friend class BlobClient;
friend class Files::DataLake::DataLakeFileClient;
};
} // namespace Blobs
} // namespace Storage
} // namespace Azure

View File

@ -0,0 +1,534 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "az.h"
#include "azInt.h"
#include "os.h"
#include "taoserror.h"
#include "tglobal.h"
#if defined(USE_S3)
#include <azure/core.hpp>
#include <azure/storage/blobs.hpp>
#include "td_block_blob_client.hpp"
// Add appropriate using namespace directives
using namespace Azure::Storage;
using namespace Azure::Storage::Blobs;
extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[TSDB_FQDN_LEN];
extern int8_t tsS3Enabled;
extern int8_t tsS3EpNum;
int32_t azBegin() { return TSDB_CODE_SUCCESS; }
void azEnd() {}
static void checkPrint(const char *fmt, ...) {
va_list arg_ptr;
va_start(arg_ptr, fmt);
(void)vfprintf(stderr, fmt, arg_ptr);
va_end(arg_ptr);
}
static void azDumpCfgByEp(int8_t epIndex) {
// clang-format off
checkPrint(
"%-24s %s\n"
"%-24s %s\n"
"%-24s %s\n"
"%-24s %s\n"
"%-24s %s\n"
"%-24s %s\n",
"hostName", tsS3Hostname[epIndex],
"bucketName", tsS3BucketName,
"protocol", "https only",
"uristyle", "path only",
"accessKey", tsS3AccessKeyId[epIndex],
"accessKeySecret", tsS3AccessKeySecret[epIndex]);
// clang-format on
}
static int32_t azListBucket(char const *bucketname) {
int32_t code = 0;
const std::string delimiter = "/";
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
std::string accountURL = tsS3Hostname[0];
accountURL = "https://" + accountURL;
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = bucketname;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
Azure::Storage::Blobs::ListBlobsOptions options;
options.Prefix = "s3";
checkPrint("objects:\n");
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
for (const auto &blob : pageResult.Blobs) {
checkPrint("%s\n", blob.Name.c_str());
}
}
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
e.ReasonPhrase.c_str());
code = TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
int32_t azCheckCfg() {
int32_t code = 0, lino = 0;
azDumpCfgByEp(0);
// test put
char testdata[17] = "0123456789abcdef";
const char *objectname[] = {"s3test.txt"};
char path[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]);
} else {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]);
}
uint8_t *pBlock = NULL;
int c_offset = 10;
int c_len = 6;
char buf[7] = {0};
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
checkPrint("failed to open test file: %s.\n", path);
TAOS_CHECK_GOTO(terrno, &lino, _next);
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
checkPrint("failed to write test file: %s.\n", path);
TAOS_CHECK_GOTO(terrno, &lino, _next);
}
if (taosFsyncFile(fp) < 0) {
checkPrint("failed to fsync test file: %s.\n", path);
TAOS_CHECK_GOTO(terrno, &lino, _next);
}
(void)taosCloseFile(&fp);
checkPrint("\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
code = azPutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
checkPrint("put object %s : failed.\n", objectname[0]);
TAOS_CHECK_GOTO(code, &lino, _next);
}
checkPrint("put object %s: success.\n\n", objectname[0]);
// list buckets
checkPrint("start to list bucket %s by prefix s3.\n", tsS3BucketName);
code = azListBucket(tsS3BucketName);
if (code != 0) {
checkPrint("listing bucket %s : failed.\n", tsS3BucketName);
TAOS_CHECK_GOTO(code, &lino, _next);
}
checkPrint("listing bucket %s: success.\n\n", tsS3BucketName);
// test range get
checkPrint("start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
code = azGetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
checkPrint("get object %s : failed.\n", objectname[0]);
TAOS_CHECK_GOTO(code, &lino, _next);
}
(void)memcpy(buf, pBlock, c_len);
taosMemoryFree(pBlock);
checkPrint("object content: %s\n", buf);
checkPrint("get object %s: success.\n\n", objectname[0]);
// delete test object
checkPrint("start to delete object: %s.\n", objectname[0]);
// code = azDeleteObjectsByPrefix(objectname[0]);
azDeleteObjectsByPrefix(objectname[0]);
/*
if (code != 0) {
(void)fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
TAOS_CHECK_GOTO(code, &lino, _next);
}
*/
checkPrint("delete object %s: success.\n\n", objectname[0]);
_next:
if (fp) {
(void)taosCloseFile(&fp);
}
if (TSDB_CODE_SUCCESS != code) {
checkPrint("s3 check failed, code: %d, line: %d.\n", code, lino);
}
checkPrint("=================================================================\n");
TAOS_RETURN(code);
}
static int32_t azPutObjectFromFileOffsetImpl(const char *file, const char *object_name, int64_t offset, int64_t size) {
int32_t code = 0;
std::string endpointUrl = tsS3Hostname[0];
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
std::string accountURL = tsS3Hostname[0];
accountURL = "https://" + accountURL;
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = tsS3BucketName;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
std::string blobName = "blob.txt";
uint8_t blobContent[] = "Hello Azure!";
// Create the block blob client
// BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
// TDBlockBlobClient blobClient(containerClient.GetBlobClient(blobName));
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
blobClient.UploadFrom(file, offset, size);
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s: Status Code: %d, Reason Phrase: %s", __func__, static_cast<int>(e.StatusCode), e.ReasonPhrase.c_str());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
int32_t code = 0;
try {
code = azPutObjectFromFileOffsetImpl(file, object_name, offset, size);
} catch (const std::exception &e) {
azError("%s: Reason Phrase: %s", __func__, e.what());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
static int32_t azGetObjectBlockImpl(const char *object_name, int64_t offset, int64_t size, bool check,
uint8_t **ppBlock) {
int32_t code = TSDB_CODE_SUCCESS;
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
std::string accountURL = tsS3Hostname[0];
uint8_t *buf = NULL;
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
accountURL = "https://" + accountURL;
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = tsS3BucketName;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
Blobs::DownloadBlobToOptions options;
options.Range = Azure::Core::Http::HttpRange();
options.Range.Value().Offset = offset;
options.Range.Value().Length = size;
buf = (uint8_t *)taosMemoryCalloc(1, size);
if (!buf) {
return terrno;
}
auto res = blobClient.DownloadTo(buf, size, options);
if (check && res.Value.ContentRange.Length.Value() != size) {
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
*ppBlock = buf;
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
e.ReasonPhrase.c_str());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
if (buf) {
taosMemoryFree(buf);
}
*ppBlock = NULL;
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
static int32_t azGetObjectBlockRetry(const char *object_name, int64_t offset, int64_t size, bool check,
uint8_t **ppBlock) {
int32_t code = TSDB_CODE_SUCCESS;
// May use an exponential backoff policy for retries with 503
int retryCount = 0;
static int maxRetryCount = 5;
static int minRetryInterval = 1000; // ms
static int maxRetryInterval = 3000; // ms
_retry:
code = azGetObjectBlockImpl(object_name, offset, size, check, ppBlock);
if (TSDB_CODE_SUCCESS != code && retryCount++ < maxRetryCount) {
taosMsleep(taosRand() % (maxRetryInterval - minRetryInterval + 1) + minRetryInterval);
uInfo("%s: 0x%x(%s) and retry get object", __func__, code, tstrerror(code));
goto _retry;
}
TAOS_RETURN(code);
}
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
int32_t code = TSDB_CODE_SUCCESS;
try {
code = azGetObjectBlockRetry(object_name, offset, size, check, ppBlock);
} catch (const std::exception &e) {
azError("%s: Reason Phrase: %s", __func__, e.what());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
static void azDeleteObjectsByPrefixImpl(const char *prefix) {
const std::string delimiter = "/";
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
std::string accountURL = tsS3Hostname[0];
accountURL = "https://" + accountURL;
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = tsS3BucketName;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
Azure::Storage::Blobs::ListBlobsOptions options;
options.Prefix = prefix;
std::set<std::string> listBlobs;
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
for (const auto &blob : pageResult.Blobs) {
listBlobs.insert(blob.Name);
}
}
for (auto blobName : listBlobs) {
auto blobClient = containerClient.GetAppendBlobClient(blobName);
blobClient.Delete();
}
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
e.ReasonPhrase.c_str());
}
}
void azDeleteObjectsByPrefix(const char *prefix) {
int32_t code = TSDB_CODE_SUCCESS;
try {
azDeleteObjectsByPrefixImpl(prefix);
} catch (const std::exception &e) {
azError("%s: Reason Phrase: %s", __func__, e.what());
}
}
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) {
int32_t code = 0, lino = 0;
uint64_t contentLength = 0;
if (taosStatFile(file, (int64_t *)&contentLength, NULL, NULL) < 0) {
azError("ERROR: %s Failed to stat file %s: ", __func__, file);
TAOS_RETURN(terrno);
}
code = azPutObjectFromFileOffset(file, object, 0, contentLength);
if (code != 0) {
azError("ERROR: %s Failed to put file %s: ", __func__, file);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
_exit:
if (code) {
azError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return 0;
}
int32_t azGetObjectToFile(const char *object_name, const char *fileName) {
int32_t code = TSDB_CODE_SUCCESS;
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
std::string accountURL = tsS3Hostname[0];
accountURL = "https://" + accountURL;
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = tsS3BucketName;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name));
auto res = blobClient.DownloadTo(fileName);
if (res.Value.ContentRange.Length.Value() <= 0) {
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
e.ReasonPhrase.c_str());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
int32_t azGetObjectsByPrefix(const char *prefix, const char *path) {
const std::string delimiter = "/";
std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0];
std::string accountURL = tsS3Hostname[0];
accountURL = "https://" + accountURL;
try {
auto sharedKeyCredential = std::make_shared<StorageSharedKeyCredential>(accountName, accountKey);
BlobServiceClient blobServiceClient(accountURL, sharedKeyCredential);
std::string containerName = tsS3BucketName;
auto containerClient = blobServiceClient.GetBlobContainerClient(containerName);
Azure::Storage::Blobs::ListBlobsOptions options;
options.Prefix = prefix;
std::set<std::string> listBlobs;
for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) {
for (const auto &blob : pageResult.Blobs) {
listBlobs.insert(blob.Name);
}
}
for (auto blobName : listBlobs) {
const char *tmp = strchr(blobName.c_str(), '/');
tmp = (tmp == NULL) ? blobName.c_str() : tmp + 1;
char fileName[PATH_MAX] = {0};
if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) {
(void)snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp);
} else {
(void)snprintf(fileName, PATH_MAX, "%s%s", path, tmp);
}
if (azGetObjectToFile(blobName.c_str(), fileName)) {
TAOS_RETURN(TSDB_CODE_FAILED);
}
}
} catch (const Azure::Core::RequestFailedException &e) {
azError("%s failed at line %d since %d(%s)", __func__, __LINE__, static_cast<int>(e.StatusCode),
e.ReasonPhrase.c_str());
TAOS_RETURN(TSDB_CODE_FAILED);
}
return 0;
}
int32_t azDeleteObjects(const char *object_name[], int nobject) {
for (int i = 0; i < nobject; ++i) {
azDeleteObjectsByPrefix(object_name[i]);
}
return 0;
}
#else
int32_t azBegin() { return TSDB_CODE_SUCCESS; }
void azEnd() {}
int32_t azCheckCfg() { return TSDB_CODE_SUCCESS; }
int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
return TSDB_CODE_SUCCESS;
}
int32_t azGetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
return TSDB_CODE_SUCCESS;
}
void azDeleteObjectsByPrefix(const char *prefix) {}
int32_t azPutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; }
int32_t azGetObjectToFile(const char *object_name, const char *fileName) { return 0; }
int32_t azDeleteObjects(const char *object_name[], int nobject) { return 0; }
#endif

View File

@ -0,0 +1,531 @@
#if defined(USE_S3)
#include <td_avro_parser.h>
#include <azure/core/azure_assert.hpp>
#include <azure/core/internal/json/json.hpp>
#include <algorithm>
#include <cstring>
namespace Azure {
namespace Storage {
namespace Blobs {
namespace _detail {
namespace {
int64_t parseInt(AvroStreamReader::ReaderPos& data) {
uint64_t r = 0;
int nb = 0;
while (true) {
uint8_t c = (*data.BufferPtr)[data.Offset++];
r = r | ((static_cast<uint64_t>(c) & 0x7f) << (nb * 7));
if (c & 0x80) {
++nb;
continue;
}
break;
}
return static_cast<int64_t>(r >> 1) ^ -static_cast<int64_t>(r & 0x01);
}
AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) {
const static std::map<std::string, AvroSchema> BuiltinNameSchemaMap = {
{"string", AvroSchema::StringSchema}, {"bytes", AvroSchema::BytesSchema}, {"int", AvroSchema::IntSchema},
{"long", AvroSchema::LongSchema}, {"float", AvroSchema::FloatSchema}, {"double", AvroSchema::DoubleSchema},
{"boolean", AvroSchema::BoolSchema}, {"null", AvroSchema::NullSchema}, {"string", AvroSchema::StringSchema},
};
std::map<std::string, AvroSchema> nameSchemaMap = BuiltinNameSchemaMap;
std::function<AvroSchema(const Core::Json::_internal::json& obj)> parseSchemaFromJsonObject;
parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema {
if (obj.is_string()) {
auto typeName = obj.get<std::string>();
return nameSchemaMap.find(typeName)->second;
} else if (obj.is_array()) {
std::vector<AvroSchema> unionSchemas;
for (const auto& s : obj) {
unionSchemas.push_back(parseSchemaFromJsonObject(s));
}
return AvroSchema::UnionSchema(std::move(unionSchemas));
} else if (obj.is_object()) {
if (obj.count("namespace") != 0) {
throw std::runtime_error("Namespace isn't supported yet in Avro schema.");
}
if (obj.count("aliases") != 0) {
throw std::runtime_error("Alias isn't supported yet in Avro schema.");
}
auto typeName = obj["type"].get<std::string>();
auto i = nameSchemaMap.find(typeName);
if (i != nameSchemaMap.end()) {
return i->second;
}
if (typeName == "record") {
std::vector<std::pair<std::string, AvroSchema>> fieldsSchema;
for (const auto& field : obj["fields"]) {
fieldsSchema.push_back(
std::make_pair(field["name"].get<std::string>(), parseSchemaFromJsonObject(field["type"])));
}
const std::string recordName = obj["name"].get<std::string>();
auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema));
nameSchemaMap.insert(std::make_pair(recordName, recordSchema));
return recordSchema;
} else if (typeName == "enum") {
throw std::runtime_error("Enum type isn't supported yet in Avro schema.");
} else if (typeName == "array") {
return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"]));
} else if (typeName == "map") {
return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"]));
} else if (typeName == "fixed") {
const std::string fixedName = obj["name"].get<std::string>();
auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get<int64_t>());
nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema));
return fixedSchema;
} else {
throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema.");
}
}
AZURE_UNREACHABLE_CODE();
};
auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end());
return parseSchemaFromJsonObject(jsonRoot);
}
} // namespace
int64_t AvroStreamReader::ParseInt(const Core::Context& context) {
uint64_t r = 0;
int nb = 0;
while (true) {
Preload(1, context);
uint8_t c = m_streambuffer[m_pos.Offset++];
r = r | ((static_cast<uint64_t>(c) & 0x7f) << (nb * 7));
if (c & 0x80) {
++nb;
continue;
}
break;
}
return static_cast<int64_t>(r >> 1) ^ -static_cast<int64_t>(r & 0x01);
}
void AvroStreamReader::Advance(size_t n, const Core::Context& context) {
Preload(n, context);
m_pos.Offset += n;
}
size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) {
size_t oldAvailable = AvailableBytes();
while (true) {
size_t newAvailable = TryPreload(n, context);
if (newAvailable >= n) {
return newAvailable;
}
if (oldAvailable == newAvailable) {
throw std::runtime_error("Unexpected EOF of Avro stream.");
}
oldAvailable = newAvailable;
}
AZURE_UNREACHABLE_CODE();
}
size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) {
size_t availableBytes = AvailableBytes();
if (availableBytes >= n) {
return availableBytes;
}
const size_t MinRead = 4096;
size_t tryReadSize = (std::max)(n, MinRead);
size_t currSize = m_streambuffer.size();
m_streambuffer.resize(m_streambuffer.size() + tryReadSize);
size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context);
m_streambuffer.resize(currSize + actualReadSize);
return AvailableBytes();
}
void AvroStreamReader::Discard() {
constexpr size_t MinimumReleaseMemory = 128 * 1024;
if (m_pos.Offset < MinimumReleaseMemory) {
return;
}
const size_t availableBytes = AvailableBytes();
std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes);
m_streambuffer.resize(availableBytes);
m_pos.Offset = 0;
}
const AvroSchema AvroSchema::StringSchema(AvroDatumType::String);
const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes);
const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int);
const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long);
const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float);
const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double);
const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool);
const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null);
AvroSchema AvroSchema::RecordSchema(std::string name,
const std::vector<std::pair<std::string, AvroSchema>>& fieldsSchema) {
AvroSchema recordSchema(AvroDatumType::Record);
recordSchema.m_name = std::move(name);
recordSchema.m_status = std::make_shared<SharedStatus>();
for (auto& i : fieldsSchema) {
recordSchema.m_status->m_keys.push_back(i.first);
recordSchema.m_status->m_schemas.push_back(i.second);
}
return recordSchema;
}
AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) {
AvroSchema arraySchema(AvroDatumType::Array);
arraySchema.m_status = std::make_shared<SharedStatus>();
arraySchema.m_status->m_schemas.push_back(std::move(elementSchema));
return arraySchema;
}
AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) {
AvroSchema mapSchema(AvroDatumType::Map);
mapSchema.m_status = std::make_shared<SharedStatus>();
mapSchema.m_status->m_schemas.push_back(std::move(elementSchema));
return mapSchema;
}
AvroSchema AvroSchema::UnionSchema(std::vector<AvroSchema> schemas) {
AvroSchema unionSchema(AvroDatumType::Union);
unionSchema.m_status = std::make_shared<SharedStatus>();
unionSchema.m_status->m_schemas = std::move(schemas);
return unionSchema;
}
AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) {
AvroSchema fixedSchema(AvroDatumType::Fixed);
fixedSchema.m_name = std::move(name);
fixedSchema.m_status = std::make_shared<SharedStatus>();
fixedSchema.m_status->m_size = size;
return fixedSchema;
}
void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) {
m_data = reader.m_pos;
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
int64_t stringSize = reader.ParseInt(context);
reader.Advance(static_cast<size_t>(stringSize), context);
} else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long ||
m_schema.Type() == AvroDatumType::Enum) {
reader.ParseInt(context);
} else if (m_schema.Type() == AvroDatumType::Float) {
reader.Advance(4, context);
} else if (m_schema.Type() == AvroDatumType::Double) {
reader.Advance(8, context);
} else if (m_schema.Type() == AvroDatumType::Bool) {
reader.Advance(1, context);
} else if (m_schema.Type() == AvroDatumType::Null) {
reader.Advance(0, context);
} else if (m_schema.Type() == AvroDatumType::Record) {
for (const auto& s : m_schema.FieldSchemas()) {
AvroDatum(s).Fill(reader, context);
}
} else if (m_schema.Type() == AvroDatumType::Array) {
while (true) {
int64_t numElementsInBlock = reader.ParseInt(context);
if (numElementsInBlock == 0) {
break;
} else if (numElementsInBlock < 0) {
int64_t blockSize = reader.ParseInt(context);
reader.Advance(static_cast<size_t>(blockSize), context);
} else {
for (auto i = 0; i < numElementsInBlock; ++i) {
AvroDatum(m_schema.ItemSchema()).Fill(reader, context);
}
}
}
} else if (m_schema.Type() == AvroDatumType::Map) {
while (true) {
int64_t numElementsInBlock = reader.ParseInt(context);
if (numElementsInBlock == 0) {
break;
} else if (numElementsInBlock < 0) {
int64_t blockSize = reader.ParseInt(context);
reader.Advance(static_cast<size_t>(blockSize), context);
} else {
for (int64_t i = 0; i < numElementsInBlock; ++i) {
AvroDatum(AvroSchema::StringSchema).Fill(reader, context);
AvroDatum(m_schema.ItemSchema()).Fill(reader, context);
}
}
}
} else if (m_schema.Type() == AvroDatumType::Union) {
int64_t i = reader.ParseInt(context);
AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]).Fill(reader, context);
} else if (m_schema.Type() == AvroDatumType::Fixed) {
reader.Advance(m_schema.Size(), context);
} else {
AZURE_UNREACHABLE_CODE();
}
}
void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) {
m_data = data;
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
int64_t stringSize = parseInt(data);
data.Offset += static_cast<size_t>(stringSize);
} else if (m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long ||
m_schema.Type() == AvroDatumType::Enum) {
parseInt(data);
} else if (m_schema.Type() == AvroDatumType::Float) {
data.Offset += 4;
} else if (m_schema.Type() == AvroDatumType::Double) {
data.Offset += 8;
} else if (m_schema.Type() == AvroDatumType::Bool) {
data.Offset += 1;
} else if (m_schema.Type() == AvroDatumType::Null) {
data.Offset += 0;
} else if (m_schema.Type() == AvroDatumType::Record) {
for (const auto& s : m_schema.FieldSchemas()) {
AvroDatum(s).Fill(data);
}
} else if (m_schema.Type() == AvroDatumType::Array) {
while (true) {
int64_t numElementsInBlock = parseInt(data);
if (numElementsInBlock == 0) {
break;
} else if (numElementsInBlock < 0) {
int64_t blockSize = parseInt(data);
data.Offset += static_cast<size_t>(blockSize);
} else {
for (auto i = 0; i < numElementsInBlock; ++i) {
AvroDatum(m_schema.ItemSchema()).Fill(data);
}
}
}
} else if (m_schema.Type() == AvroDatumType::Map) {
while (true) {
int64_t numElementsInBlock = parseInt(data);
if (numElementsInBlock == 0) {
break;
} else if (numElementsInBlock < 0) {
int64_t blockSize = parseInt(data);
data.Offset += static_cast<size_t>(blockSize);
} else {
for (int64_t i = 0; i < numElementsInBlock; ++i) {
AvroDatum(AvroSchema::StringSchema).Fill(data);
AvroDatum(m_schema.ItemSchema()).Fill(data);
}
}
}
} else if (m_schema.Type() == AvroDatumType::Union) {
int64_t i = parseInt(data);
AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]).Fill(data);
} else if (m_schema.Type() == AvroDatumType::Fixed) {
data.Offset += m_schema.Size();
} else {
AZURE_UNREACHABLE_CODE();
}
}
template <>
AvroDatum::StringView AvroDatum::Value() const {
auto data = m_data;
if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) {
const int64_t length = parseInt(data);
const uint8_t* start = &(*data.BufferPtr)[data.Offset];
StringView ret{start, static_cast<size_t>(length)};
data.Offset += static_cast<size_t>(length);
return ret;
}
if (m_schema.Type() == AvroDatumType::Fixed) {
const size_t fixedSize = m_schema.Size();
const uint8_t* start = &(*data.BufferPtr)[data.Offset];
StringView ret{start, fixedSize};
data.Offset += fixedSize;
return ret;
}
AZURE_UNREACHABLE_CODE();
}
template <>
std::string AvroDatum::Value() const {
auto stringView = Value<StringView>();
return std::string(stringView.Data, stringView.Data + stringView.Length);
}
template <>
std::vector<uint8_t> AvroDatum::Value() const {
auto stringView = Value<StringView>();
return std::vector<uint8_t>(stringView.Data, stringView.Data + stringView.Length);
}
template <>
int64_t AvroDatum::Value() const {
auto data = m_data;
return parseInt(data);
}
template <>
int32_t AvroDatum::Value() const {
return static_cast<int32_t>(Value<int64_t>());
}
template <>
bool AvroDatum::Value() const {
return Value<int64_t>();
}
template <>
std::nullptr_t AvroDatum::Value() const {
return nullptr;
}
template <>
AvroRecord AvroDatum::Value() const {
auto data = m_data;
AvroRecord r;
r.m_keys = &m_schema.FieldNames();
for (const auto& schema : m_schema.FieldSchemas()) {
auto datum = AvroDatum(schema);
datum.Fill(data);
r.m_values.push_back(std::move(datum));
}
return r;
}
template <>
AvroMap AvroDatum::Value() const {
auto data = m_data;
AvroMap m;
while (true) {
int64_t numElementsInBlock = parseInt(data);
if (numElementsInBlock == 0) {
break;
}
if (numElementsInBlock < 0) {
numElementsInBlock = -numElementsInBlock;
parseInt(data);
}
for (int64_t i = 0; i < numElementsInBlock; ++i) {
auto keyDatum = AvroDatum(AvroSchema::StringSchema);
keyDatum.Fill(data);
auto valueDatum = AvroDatum(m_schema.ItemSchema());
valueDatum.Fill(data);
m[keyDatum.Value<std::string>()] = valueDatum;
}
}
return m;
}
template <>
AvroDatum AvroDatum::Value() const {
auto data = m_data;
if (m_schema.Type() == AvroDatumType::Union) {
int64_t i = parseInt(data);
auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast<size_t>(i)]);
datum.Fill(data);
return datum;
}
AZURE_UNREACHABLE_CODE();
}
AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream)
: m_reader(std::make_unique<AvroStreamReader>(stream)) {}
AvroDatum AvroObjectContainerReader::NextImpl(const AvroSchema* schema, const Core::Context& context) {
AZURE_ASSERT_FALSE(m_eof);
static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16);
if (!schema) {
static AvroSchema FileHeaderSchema = []() {
std::vector<std::pair<std::string, AvroSchema>> fieldsSchema;
fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4)));
fieldsSchema.push_back(std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema)));
fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema));
return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema));
}();
auto fileHeaderDatum = AvroDatum(FileHeaderSchema);
fileHeaderDatum.Fill(*m_reader, context);
auto fileHeader = fileHeaderDatum.Value<AvroRecord>();
if (fileHeader.Field("magic").Value<std::string>() != "Obj\01") {
throw std::runtime_error("Invalid Avro object container magic.");
}
AvroMap meta = fileHeader.Field("meta").Value<AvroMap>();
std::string objectSchemaJson = meta["avro.schema"].Value<std::string>();
std::string codec = "null";
if (meta.count("avro.codec") != 0) {
codec = meta["avro.codec"].Value<std::string>();
}
if (codec != "null") {
throw std::runtime_error("Unsupported Avro codec: " + codec);
}
m_syncMarker = fileHeader.Field("sync").Value<std::string>();
m_objectSchema = std::make_unique<AvroSchema>(ParseSchemaFromJsonString(objectSchemaJson));
schema = m_objectSchema.get();
}
if (m_remainingObjectInCurrentBlock == 0) {
m_reader->Discard();
m_remainingObjectInCurrentBlock = m_reader->ParseInt(context);
int64_t ObjectsSize = m_reader->ParseInt(context);
m_reader->Preload(static_cast<size_t>(ObjectsSize), context);
}
auto objectDatum = AvroDatum(*m_objectSchema);
objectDatum.Fill(*m_reader, context);
if (--m_remainingObjectInCurrentBlock == 0) {
auto markerDatum = AvroDatum(SyncMarkerSchema);
markerDatum.Fill(*m_reader, context);
auto marker = markerDatum.Value<std::string>();
if (marker != m_syncMarker) {
throw std::runtime_error("Sync marker doesn't match.");
}
m_eof = m_reader->TryPreload(1, context) == 0;
}
return objectDatum;
}
size_t AvroStreamParser::OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) {
if (m_parserBuffer.Length != 0) {
size_t bytesToCopy = (std::min)(m_parserBuffer.Length, count);
std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy);
m_parserBuffer.Data += bytesToCopy;
m_parserBuffer.Length -= bytesToCopy;
return bytesToCopy;
}
while (!m_parser.End()) {
auto datum = m_parser.Next(context);
if (datum.Schema().Type() == AvroDatumType::Union) {
datum = datum.Value<AvroDatum>();
}
if (datum.Schema().Type() != AvroDatumType::Record) {
continue;
}
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") {
auto record = datum.Value<AvroRecord>();
auto dataDatum = record.Field("data");
m_parserBuffer = dataDatum.Value<AvroDatum::StringView>();
return OnRead(buffer, count, context);
}
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" && m_progressCallback) {
auto record = datum.Value<AvroRecord>();
auto bytesScanned = record.Field("bytesScanned").Value<int64_t>();
auto totalBytes = record.Field("totalBytes").Value<int64_t>();
m_progressCallback(bytesScanned, totalBytes);
}
if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" && m_errorCallback) {
auto record = datum.Value<AvroRecord>();
BlobQueryError e;
e.Name = record.Field("name").Value<std::string>();
e.Description = record.Field("description").Value<std::string>();
e.IsFatal = record.Field("fatal").Value<bool>();
e.Position = record.Field("position").Value<int64_t>();
m_errorCallback(std::move(e));
}
}
return 0;
}
} // namespace _detail
} // namespace Blobs
} // namespace Storage
} // namespace Azure
#endif

View File

@ -0,0 +1,625 @@
#if defined(USE_S3)
#include "td_block_blob_client.hpp"
#include <azure/core/platform.hpp>
#if defined(AZ_PLATFORM_WINDOWS)
#if !defined(WIN32_LEAN_AND_MEAN)
#define WIN32_LEAN_AND_MEAN
#endif
#if !defined(NOMINMAX)
#define NOMINMAX
#endif
#include <windows.h>
#endif
#include <td_avro_parser.h>
#include <azure/core/io/body_stream.hpp>
#include <azure/storage/common/crypt.hpp>
#include <azure/storage/common/internal/concurrent_transfer.hpp>
#include <azure/storage/common/internal/constants.hpp>
#include <azure/storage/common/internal/file_io.hpp>
#include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
#include <azure/storage/common/storage_common.hpp>
#include <azure/storage/common/storage_exception.hpp>
namespace Azure {
namespace Storage {
namespace Blobs {
TDBlockBlobClient TDBlockBlobClient::CreateFromConnectionString(const std::string& connectionString,
const std::string& blobContainerName,
const std::string& blobName,
const BlobClientOptions& options) {
TDBlockBlobClient newClient(
BlobClient::CreateFromConnectionString(connectionString, blobContainerName, blobName, options));
return newClient;
}
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<StorageSharedKeyCredential> credential,
const BlobClientOptions& options)
: BlobClient(blobUrl, std::move(credential), options) {}
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl,
std::shared_ptr<Core::Credentials::TokenCredential> credential,
const BlobClientOptions& options)
: BlobClient(blobUrl, std::move(credential), options) {}
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options)
: BlobClient(blobUrl, options) {}
TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {}
TDBlockBlobClient TDBlockBlobClient::WithSnapshot(const std::string& snapshot) const {
TDBlockBlobClient newClient(*this);
if (snapshot.empty()) {
newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQuerySnapshot);
} else {
newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQuerySnapshot,
_internal::UrlEncodeQueryParameter(snapshot));
}
return newClient;
}
TDBlockBlobClient TDBlockBlobClient::WithVersionId(const std::string& versionId) const {
TDBlockBlobClient newClient(*this);
if (versionId.empty()) {
newClient.m_blobUrl.RemoveQueryParameter(_internal::HttpQueryVersionId);
} else {
newClient.m_blobUrl.AppendQueryParameter(_internal::HttpQueryVersionId,
_internal::UrlEncodeQueryParameter(versionId));
}
return newClient;
}
Azure::Response<Models::UploadBlockBlobResult> TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content,
const UploadBlockBlobOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions;
if (options.TransactionalContentHash.HasValue()) {
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
}
}
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
protocolLayerOptions.Tier = options.AccessTier;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
if (options.ImmutabilityPolicy.HasValue()) {
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
}
protocolLayerOptions.LegalHold = options.HasLegalHold;
return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
}
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
const uint8_t* buffer, size_t bufferSize, const UploadBlockBlobFromOptions& options,
const Azure::Core::Context& context) const {
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
constexpr int64_t MaxBlockNumber = 50000;
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
if (static_cast<uint64_t>(options.TransferOptions.SingleUploadThreshold) > (std::numeric_limits<size_t>::max)()) {
throw Azure::Core::RequestFailedException("Single upload threshold is too big");
}
if (bufferSize <= static_cast<size_t>(options.TransferOptions.SingleUploadThreshold)) {
Azure::Core::IO::MemoryBodyStream contentStream(buffer, bufferSize);
UploadBlockBlobOptions uploadBlockBlobOptions;
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
uploadBlockBlobOptions.Metadata = options.Metadata;
uploadBlockBlobOptions.Tags = options.Tags;
uploadBlockBlobOptions.AccessTier = options.AccessTier;
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
return Upload(contentStream, uploadBlockBlobOptions, context);
}
int64_t chunkSize;
if (options.TransferOptions.ChunkSize.HasValue()) {
chunkSize = options.TransferOptions.ChunkSize.Value();
} else {
int64_t minChunkSize = (bufferSize + MaxBlockNumber - 1) / MaxBlockNumber;
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
}
if (chunkSize > MaxStageBlockSize) {
throw Azure::Core::RequestFailedException("Block size is too big.");
}
std::vector<std::string> blockIds;
auto getBlockId = [](int64_t id) {
constexpr size_t BlockIdLength = 64;
std::string blockId = std::to_string(id);
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
};
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
Azure::Core::IO::MemoryBodyStream contentStream(buffer + offset, static_cast<size_t>(length));
StageBlockOptions chunkOptions;
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
if (chunkId == numChunks - 1) {
blockIds.resize(static_cast<size_t>(numChunks));
}
};
_internal::ConcurrentTransfer(0, bufferSize, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc);
for (size_t i = 0; i < blockIds.size(); ++i) {
blockIds[i] = getBlockId(static_cast<int64_t>(i));
}
CommitBlockListOptions commitBlockListOptions;
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
commitBlockListOptions.Metadata = options.Metadata;
commitBlockListOptions.Tags = options.Tags;
commitBlockListOptions.AccessTier = options.AccessTier;
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
Models::UploadBlockBlobFromResult ret;
ret.ETag = std::move(commitBlockListResponse.Value.ETag);
ret.LastModified = std::move(commitBlockListResponse.Value.LastModified);
ret.VersionId = std::move(commitBlockListResponse.Value.VersionId);
ret.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
ret.EncryptionKeySha256 = std::move(commitBlockListResponse.Value.EncryptionKeySha256);
ret.EncryptionScope = std::move(commitBlockListResponse.Value.EncryptionScope);
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(ret),
std::move(commitBlockListResponse.RawResponse));
}
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
const std::string& fileName, const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const {
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
constexpr int64_t MaxBlockNumber = 50000;
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
{
Azure::Core::IO::FileBodyStream contentStream(fileName);
if (contentStream.Length() <= options.TransferOptions.SingleUploadThreshold) {
UploadBlockBlobOptions uploadBlockBlobOptions;
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
uploadBlockBlobOptions.Metadata = options.Metadata;
uploadBlockBlobOptions.Tags = options.Tags;
uploadBlockBlobOptions.AccessTier = options.AccessTier;
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
return Upload(contentStream, uploadBlockBlobOptions, context);
}
}
std::vector<std::string> blockIds;
auto getBlockId = [](int64_t id) {
constexpr size_t BlockIdLength = 64;
std::string blockId = std::to_string(id);
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
};
_internal::FileReader fileReader(fileName);
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length);
StageBlockOptions chunkOptions;
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
if (chunkId == numChunks - 1) {
blockIds.resize(static_cast<size_t>(numChunks));
}
};
int64_t chunkSize;
if (options.TransferOptions.ChunkSize.HasValue()) {
chunkSize = options.TransferOptions.ChunkSize.Value();
} else {
int64_t minChunkSize = (fileReader.GetFileSize() + MaxBlockNumber - 1) / MaxBlockNumber;
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
}
if (chunkSize > MaxStageBlockSize) {
throw Azure::Core::RequestFailedException("Block size is too big.");
}
_internal::ConcurrentTransfer(0, fileReader.GetFileSize(), chunkSize, options.TransferOptions.Concurrency,
uploadBlockFunc);
for (size_t i = 0; i < blockIds.size(); ++i) {
blockIds[i] = getBlockId(static_cast<int64_t>(i));
}
CommitBlockListOptions commitBlockListOptions;
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
commitBlockListOptions.Metadata = options.Metadata;
commitBlockListOptions.Tags = options.Tags;
commitBlockListOptions.AccessTier = options.AccessTier;
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
Models::UploadBlockBlobFromResult result;
result.ETag = commitBlockListResponse.Value.ETag;
result.LastModified = commitBlockListResponse.Value.LastModified;
result.VersionId = commitBlockListResponse.Value.VersionId;
result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256;
result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope;
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
std::move(commitBlockListResponse.RawResponse));
}
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
const std::string& fileName, int64_t offset, int64_t size, const UploadBlockBlobFromOptions& options,
const Azure::Core::Context& context) const {
_internal::FileReader fileReader(fileName);
{
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, size);
if (size <= options.TransferOptions.SingleUploadThreshold) {
UploadBlockBlobOptions uploadBlockBlobOptions;
uploadBlockBlobOptions.HttpHeaders = options.HttpHeaders;
uploadBlockBlobOptions.Metadata = options.Metadata;
uploadBlockBlobOptions.Tags = options.Tags;
uploadBlockBlobOptions.AccessTier = options.AccessTier;
uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold;
return Upload(contentStream, uploadBlockBlobOptions, context);
}
}
std::vector<std::string> blockIds;
auto getBlockId = [](int64_t id) {
constexpr size_t BlockIdLength = 64;
std::string blockId = std::to_string(id);
blockId = std::string(BlockIdLength - blockId.length(), '0') + blockId;
return Azure::Core::Convert::Base64Encode(std::vector<uint8_t>(blockId.begin(), blockId.end()));
};
auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(fileReader.GetHandle(), offset, length);
StageBlockOptions chunkOptions;
auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
if (chunkId == numChunks - 1) {
blockIds.resize(static_cast<size_t>(numChunks));
}
};
constexpr int64_t DefaultStageBlockSize = 4 * 1024 * 1024ULL;
constexpr int64_t MaxStageBlockSize = 4000 * 1024 * 1024ULL;
constexpr int64_t MaxBlockNumber = 50000;
constexpr int64_t BlockGrainSize = 1 * 1024 * 1024;
int64_t chunkSize;
if (options.TransferOptions.ChunkSize.HasValue()) {
chunkSize = options.TransferOptions.ChunkSize.Value();
} else {
int64_t minChunkSize = (size + MaxBlockNumber - 1) / MaxBlockNumber;
minChunkSize = (minChunkSize + BlockGrainSize - 1) / BlockGrainSize * BlockGrainSize;
chunkSize = (std::max)(DefaultStageBlockSize, minChunkSize);
}
if (chunkSize > MaxStageBlockSize) {
throw Azure::Core::RequestFailedException("Block size is too big.");
}
_internal::ConcurrentTransfer(offset, size, chunkSize, options.TransferOptions.Concurrency, uploadBlockFunc);
for (size_t i = 0; i < blockIds.size(); ++i) {
blockIds[i] = getBlockId(static_cast<int64_t>(i));
}
CommitBlockListOptions commitBlockListOptions;
commitBlockListOptions.HttpHeaders = options.HttpHeaders;
commitBlockListOptions.Metadata = options.Metadata;
commitBlockListOptions.Tags = options.Tags;
commitBlockListOptions.AccessTier = options.AccessTier;
commitBlockListOptions.ImmutabilityPolicy = options.ImmutabilityPolicy;
commitBlockListOptions.HasLegalHold = options.HasLegalHold;
auto commitBlockListResponse = CommitBlockList(blockIds, commitBlockListOptions, context);
Models::UploadBlockBlobFromResult result;
result.ETag = commitBlockListResponse.Value.ETag;
result.LastModified = commitBlockListResponse.Value.LastModified;
result.VersionId = commitBlockListResponse.Value.VersionId;
result.IsServerEncrypted = commitBlockListResponse.Value.IsServerEncrypted;
result.EncryptionKeySha256 = commitBlockListResponse.Value.EncryptionKeySha256;
result.EncryptionScope = commitBlockListResponse.Value.EncryptionScope;
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
std::move(commitBlockListResponse.RawResponse));
}
Azure::Response<Models::UploadBlockBlobFromUriResult> TDBlockBlobClient::UploadFromUri(
const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::UploadBlockBlobFromUriOptions protocolLayerOptions;
protocolLayerOptions.CopySource = sourceUri;
protocolLayerOptions.CopySourceBlobProperties = options.CopySourceBlobProperties;
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
protocolLayerOptions.Tier = options.AccessTier;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch;
protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch;
protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince;
protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince;
protocolLayerOptions.SourceIfTags = options.SourceAccessConditions.TagConditions;
if (options.TransactionalContentHash.HasValue()) {
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value;
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value;
}
}
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
protocolLayerOptions.CopySourceTags = options.CopySourceTagsMode;
if (!options.SourceAuthorization.empty()) {
protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization;
}
return _detail::BlockBlobClient::UploadFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
}
Azure::Response<Models::StageBlockResult> TDBlockBlobClient::StageBlock(const std::string& blockId,
Azure::Core::IO::BodyStream& content,
const StageBlockOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::StageBlockBlobBlockOptions protocolLayerOptions;
protocolLayerOptions.BlockId = blockId;
if (options.TransactionalContentHash.HasValue()) {
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
}
}
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
return _detail::BlockBlobClient::StageBlock(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
}
Azure::Response<Models::StageBlockFromUriResult> TDBlockBlobClient::StageBlockFromUri(
const std::string& blockId, const std::string& sourceUri, const StageBlockFromUriOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::StageBlockBlobBlockFromUriOptions protocolLayerOptions;
protocolLayerOptions.BlockId = blockId;
protocolLayerOptions.SourceUrl = sourceUri;
if (options.SourceRange.HasValue()) {
std::string rangeStr = "bytes=" + std::to_string(options.SourceRange.Value().Offset) + "-";
if (options.SourceRange.Value().Length.HasValue()) {
rangeStr += std::to_string(options.SourceRange.Value().Offset + options.SourceRange.Value().Length.Value() - 1);
}
protocolLayerOptions.SourceRange = rangeStr;
}
if (options.TransactionalContentHash.HasValue()) {
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
protocolLayerOptions.SourceContentMD5 = options.TransactionalContentHash.Value().Value;
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
protocolLayerOptions.SourceContentcrc64 = options.TransactionalContentHash.Value().Value;
}
}
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.SourceIfModifiedSince = options.SourceAccessConditions.IfModifiedSince;
protocolLayerOptions.SourceIfUnmodifiedSince = options.SourceAccessConditions.IfUnmodifiedSince;
protocolLayerOptions.SourceIfMatch = options.SourceAccessConditions.IfMatch;
protocolLayerOptions.SourceIfNoneMatch = options.SourceAccessConditions.IfNoneMatch;
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
if (!options.SourceAuthorization.empty()) {
protocolLayerOptions.CopySourceAuthorization = options.SourceAuthorization;
}
return _detail::BlockBlobClient::StageBlockFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
}
Azure::Response<Models::CommitBlockListResult> TDBlockBlobClient::CommitBlockList(
const std::vector<std::string>& blockIds, const CommitBlockListOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::CommitBlockBlobBlockListOptions protocolLayerOptions;
protocolLayerOptions.Blocks.Latest = blockIds;
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
protocolLayerOptions.Tier = options.AccessTier;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
if (options.ImmutabilityPolicy.HasValue()) {
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
}
protocolLayerOptions.LegalHold = options.HasLegalHold;
return _detail::BlockBlobClient::CommitBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
}
Azure::Response<Models::GetBlockListResult> TDBlockBlobClient::GetBlockList(const GetBlockListOptions& options,
const Azure::Core::Context& context) const {
_detail::BlockBlobClient::GetBlockBlobBlockListOptions protocolLayerOptions;
protocolLayerOptions.ListType = options.ListType;
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
return _detail::BlockBlobClient::GetBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions,
_internal::WithReplicaStatus(context));
}
/*
Azure::Response<Models::QueryBlobResult> TDBlockBlobClient::Query(const std::string& querySqlExpression,
const QueryBlobOptions& options,
const Azure::Core::Context& context) const {
_detail::BlobClient::QueryBlobOptions protocolLayerOptions;
protocolLayerOptions.QueryRequest.QueryType = Models::_detail::QueryRequestQueryType::SQL;
protocolLayerOptions.QueryRequest.Expression = querySqlExpression;
if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) {
Models::_detail::DelimitedTextConfiguration c;
c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator;
c.ColumnSeparator = options.InputTextConfiguration.m_columnSeparator;
c.FieldQuote = options.InputTextConfiguration.m_quotationCharacter;
c.EscapeChar = options.InputTextConfiguration.m_escapeCharacter;
c.HeadersPresent = options.InputTextConfiguration.m_hasHeaders;
Models::_detail::QuerySerialization q;
q.Format.Type = options.InputTextConfiguration.m_format;
q.Format.DelimitedTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) {
Models::_detail::JsonTextConfiguration c;
c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator;
Models::_detail::QuerySerialization q;
q.Format.Type = options.InputTextConfiguration.m_format;
q.Format.JsonTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
} else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) {
Models::_detail::ParquetConfiguration c;
Models::_detail::QuerySerialization q;
q.Format.Type = options.InputTextConfiguration.m_format;
q.Format.ParquetTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.InputSerialization = std::move(q);
} else if (options.InputTextConfiguration.m_format.ToString().empty()) {
} else {
AZURE_UNREACHABLE_CODE();
}
if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) {
Models::_detail::DelimitedTextConfiguration c;
c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator;
c.ColumnSeparator = options.OutputTextConfiguration.m_columnSeparator;
c.FieldQuote = options.OutputTextConfiguration.m_quotationCharacter;
c.EscapeChar = options.OutputTextConfiguration.m_escapeCharacter;
c.HeadersPresent = options.OutputTextConfiguration.m_hasHeaders;
Models::_detail::QuerySerialization q;
q.Format.Type = options.OutputTextConfiguration.m_format;
q.Format.DelimitedTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) {
Models::_detail::JsonTextConfiguration c;
c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator;
Models::_detail::QuerySerialization q;
q.Format.Type = options.OutputTextConfiguration.m_format;
q.Format.JsonTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) {
Models::_detail::ParquetConfiguration c;
Models::_detail::QuerySerialization q;
q.Format.Type = options.OutputTextConfiguration.m_format;
q.Format.ParquetTextConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
} else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Arrow) {
Models::_detail::ArrowConfiguration c;
c.Schema = options.OutputTextConfiguration.m_schema;
Models::_detail::QuerySerialization q;
q.Format.Type = options.OutputTextConfiguration.m_format;
q.Format.ArrowConfiguration = std::move(c);
protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q);
} else if (options.InputTextConfiguration.m_format.ToString().empty()) {
} else {
AZURE_UNREACHABLE_CODE();
}
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
if (m_customerProvidedKey.HasValue()) {
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.EncryptionScope = m_encryptionScope;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
auto response =
_detail::BlobClient::Query(*m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context));
const auto statusCode = response.RawResponse->GetStatusCode();
const auto reasonPhrase = response.RawResponse->GetReasonPhrase();
const auto requestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderRequestId) != 0
? response.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId)
: std::string();
const auto clientRequestId = response.RawResponse->GetHeaders().count(_internal::HttpHeaderClientRequestId) != 0
? response.RawResponse->GetHeaders().at(_internal::HttpHeaderClientRequestId)
: std::string();
auto defaultErrorHandler = [statusCode, reasonPhrase, requestId, clientRequestId](BlobQueryError e) {
if (e.IsFatal) {
StorageException exception("Fatal " + e.Name + " at " + std::to_string(e.Position));
exception.StatusCode = statusCode;
exception.ReasonPhrase = reasonPhrase;
exception.RequestId = requestId;
exception.ClientRequestId = clientRequestId;
exception.ErrorCode = e.Name;
exception.Message = e.Description;
throw exception;
}
};
response.Value.BodyStream =
std::make_unique<_detail::AvroStreamParser>(std::move(response.Value.BodyStream), options.ProgressHandler,
options.ErrorHandler ? options.ErrorHandler : defaultErrorHandler);
return response;
}
*/
} // namespace Blobs
} // namespace Storage
} // namespace Azure
#endif

View File

@ -0,0 +1,22 @@
if (TD_LINUX)
aux_source_directory(. AZ_TEST_SRC)
add_executable(azTest ${AZ_TEST_SRC})
target_include_directories(azTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/azure"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(azTest
az
gtest_main
)
enable_testing()
add_test(
NAME az_test
COMMAND azTest
)
endif(TD_LINUX)

View File

@ -0,0 +1,201 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <cstring>
#include <iostream>
#include <queue>
#include "az.h"
extern int8_t tsS3Enabled;
int32_t azInitEnv() {
int32_t code = 0;
extern int8_t tsS3EpNum;
extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[TSDB_FQDN_LEN];
/* TCS parameter format
tsS3Hostname[0] = "<endpoint>/<account-name>.blob.core.windows.net";
tsS3AccessKeyId[0] = "<access-key-id/account-name>";
tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
tsS3BucketName = "<bucket/container-name>";
*/
const char *hostname = "<endpoint>/<account-name>.blob.core.windows.net";
const char *accessKeyId = "<access-key-id/account-name>";
const char *accessKeySecret = "<access-key-secret/account-key>";
const char *bucketName = "<bucket/container-name>";
if (hostname[0] != '<') {
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
} else {
const char *accountId = getenv("ablob_account_id");
if (!accountId) {
return -1;
}
const char *accountSecret = getenv("ablob_account_secret");
if (!accountSecret) {
return -1;
}
const char *containerName = getenv("ablob_container");
if (!containerName) {
return -1;
}
TAOS_STRCPY(&tsS3Hostname[0][0], accountId);
TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net");
TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId);
TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret);
TAOS_STRCPY(tsS3BucketName, containerName);
}
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
tsS3Enabled = true;
return code;
}
// TEST(AzTest, DISABLED_InterfaceTest) {
TEST(AzTest, InterfaceTest) {
int code = 0;
bool check = false;
bool withcp = false;
code = azInitEnv();
if (code) {
std::cout << "ablob env init failed with: " << code << std::endl;
return;
}
GTEST_ASSERT_EQ(code, 0);
GTEST_ASSERT_EQ(tsS3Enabled, 1);
code = azBegin();
GTEST_ASSERT_EQ(code, 0);
code = azCheckCfg();
GTEST_ASSERT_EQ(code, 0);
const int size = 4096;
char data[size] = {0};
for (int i = 0; i < size / 2; ++i) {
data[i * 2 + 1] = 1;
}
const char object_name[] = "azut.bin";
char path[PATH_MAX] = {0};
char path_download[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
} else {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
}
tstrncpy(path_download, path, strlen(path) + 1);
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
GTEST_ASSERT_NE(fp, nullptr);
int n = taosWriteFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
code = azPutObjectFromFileOffset(path, object_name, 0, size);
GTEST_ASSERT_EQ(code, 0);
uint8_t *pBlock = NULL;
code = azGetObjectBlock(object_name, 0, size, check, &pBlock);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
}
taosMemoryFree(pBlock);
code = azGetObjectToFile(object_name, path_download);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
azDeleteObjectsByPrefix(object_name);
// list object to check
code = azPutObjectFromFile2(path, object_name, withcp);
GTEST_ASSERT_EQ(code, 0);
code = azGetObjectsByPrefix(object_name, tsTempDir);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
const char *object_name_arr[] = {object_name};
code = azDeleteObjects(object_name_arr, 1);
GTEST_ASSERT_EQ(code, 0);
azEnd();
}

View File

@ -226,7 +226,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
}
if (TSDB_CODE_SUCCESS != pOperator->pTaskInfo->code) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(pOperator->pTaskInfo->code));
T_LONG_JMP(pTaskInfo->env, pOperator->pTaskInfo->code);
}
if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
@ -530,6 +530,16 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
int32_t index = pWrapper->sourceIndex;
SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index);
int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, index);
if (pRpcHandle != NULL) {
int32_t ret = asyncFreeConnById(pExchangeInfo->pTransporter, *pRpcHandle);
if (ret != 0) {
qDebug("failed to free rpc handle, code:%s, %p", tstrerror(ret), pExchangeInfo);
}
*pRpcHandle = -1;
}
if (!pSourceDataInfo) {
return terrno;
}

View File

@ -3293,8 +3293,44 @@ static int32_t rewriteIsTrue(SNode* pSrc, SNode** pIsTrue) {
return TSDB_CODE_SUCCESS;
}
extern int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX];
static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) {
if (commonType->type < TSDB_DATA_TYPE_NULL || commonType->type >= TSDB_DATA_TYPE_MAX ||
newType->type < TSDB_DATA_TYPE_NULL || newType->type >= TSDB_DATA_TYPE_MAX) {
return TSDB_CODE_INVALID_PARA;
}
int8_t type1 = commonType->type;
int8_t type2 = newType->type;
int8_t resultType;
if (type1 < type2) {
resultType = gDisplyTypes[type1][type2];
} else {
resultType = gDisplyTypes[type2][type1];
}
if (resultType == -1) {
return TSDB_CODE_SCALAR_CONVERT_ERROR;
}
if (commonType->type == newType->type) {
commonType->bytes = TMAX(commonType->bytes, newType->bytes);
return TSDB_CODE_SUCCESS;
}
if (resultType == commonType->type){
return TSDB_CODE_SUCCESS;
}
if(resultType == newType->type) {
*commonType = *newType;
return TSDB_CODE_SUCCESS;
}
commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]);
if(resultType == TSDB_DATA_TYPE_VARCHAR && (IS_FLOAT_TYPE(commonType->type) || IS_FLOAT_TYPE(newType->type))) {
commonType->bytes += TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE];
}
commonType->type = resultType;
return TSDB_CODE_SUCCESS;
}
static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseWhen) {
bool first = true;
bool allNullThen = true;
SNode* pNode = NULL;
FOREACH(pNode, pCaseWhen->pWhenThenList) {
@ -3313,10 +3349,18 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW
continue;
}
allNullThen = false;
if (first || dataTypeComp(&pCaseWhen->node.resType, &pThenExpr->resType) < 0) {
pCaseWhen->node.resType = pThenExpr->resType;
pCxt->errCode = selectCommonType(&pCaseWhen->node.resType, &pThenExpr->resType);
if(TSDB_CODE_SUCCESS != pCxt->errCode){
return DEAL_RES_ERROR;
}
}
SExprNode* pElseExpr = (SExprNode*)pCaseWhen->pElse;
if (NULL != pElseExpr) {
pCxt->errCode = selectCommonType(&pCaseWhen->node.resType, &pElseExpr->resType);
if(TSDB_CODE_SUCCESS != pCxt->errCode) {
return DEAL_RES_ERROR;
}
first = false;
}
if (allNullThen) {

View File

@ -1030,6 +1030,31 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0};
int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIME NCHA UTINY USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
/*NULL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, -1, -1, 8,
/*BOOL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 5, 10, 11, 12, 13, 14, 8, -1, -1, -1, -1, 8,
/*TINY*/ 0, 0, 2, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, 8, -1, -1, -1, -1, 8,
/*SMAL*/ 0, 0, 0, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, 8, -1, -1, -1, -1, 8,
/*INT */ 0, 0, 0, 0, 4, 5, 8, 8, 8, 5, 10, 4, 4, 5, 8, 8, -1, -1, -1, -1, 8,
/*BIGI*/ 0, 0, 0, 0, 0, 5, 8, 8, 8, 5, 10, 5, 5, 5, 8, 8, -1, -1, -1, -1, 8,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 6, 7, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 7, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 8, 8, 8, 8, 8, -1, -1, -1, -1, 8,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 5, 5, 5, 8, 8, -1, -1, -1, -1, 8,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10, 10, -1, -1, -1, -1, 10,
/*UTINY*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 13, 14, 8, -1, -1, -1, -1, 8,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 8, -1, -1, -1, -1, 8,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 8, -1, -1, -1, -1, 8,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, -1, -1, -1, -1, 8,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -1, -1, -1, -1, 8,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -1, -1, -1, -1,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20
};
int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
if (type1 == type2) {
return 0;

View File

@ -504,6 +504,10 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
code);
// called if drop task rsp received code
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
if (pMsg->handle == NULL) {
qError("sch handle is NULL, may be already released and mem lea");
}
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);

View File

@ -3,6 +3,7 @@ add_library(stream STATIC ${STREAM_SRC})
target_include_directories(
stream
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream"
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
@ -11,7 +12,7 @@ if(${BUILD_WITH_ROCKSDB})
if (${BUILD_CONTRIB})
target_link_libraries(
stream
PUBLIC rocksdb tdb
PUBLIC rocksdb tdb tcs
PRIVATE os util transport qcom executor wal index
)
target_include_directories(
@ -30,13 +31,13 @@ if(${BUILD_WITH_ROCKSDB})
)
target_link_libraries(
stream
PUBLIC rocksdb tdb
PUBLIC rocksdb tdb tcs
PRIVATE os util transport qcom executor wal index
)
else()
target_link_libraries(
stream
PUBLIC rocksdb tdb
PUBLIC rocksdb tdb tcs
PRIVATE os util transport qcom executor wal index
)
target_include_directories(
@ -58,4 +59,3 @@ endif(${BUILD_WITH_ROCKSDB})
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST})

View File

@ -5079,8 +5079,8 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) {
}
char content[256] = {0};
nBytes = tsnprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest, p->curChkpId,
"processVer", processId);
nBytes = tsnprintf(content, sizeof(content), META_ON_S3_FORMATE, p->pCurrent, p->curChkpId, p->pManifest,
p->curChkpId, "processVer", processId);
if (nBytes <= 0 || nBytes >= sizeof(content)) {
code = TSDB_CODE_OUT_OF_RANGE;
stError("chkp failed to format meta file: %s, reason: invalid msg", dstDir);

View File

@ -13,10 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "rsync.h"
#include "streamBackendRocksdb.h"
#include "streamInt.h"
#include "tcs.h"
static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName);
static int32_t deleteCheckpointFile(const char* id, const char* name);
@ -343,7 +343,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
// And if we don't start a new timer, and the lost of checkpoint-trigger message may cause the whole checkpoint
// procedure to be stucked.
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1);
int8_t old = atomic_val_compare_exchange_8(&pTmrInfo->isActive, 0, 1);
if (old == 0) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
@ -352,7 +352,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"trigger-recv-monitor");
pTmrInfo->launchChkptId = pActiveInfo->activeId;
} else { // already launched, do nothing
} else { // already launched, do nothing
stError("s-task:%s previous checkpoint-trigger monitor tmr is set, not start new one", pTask->id.idStr);
}
}
@ -373,10 +373,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
} else { // only one task exists, no need to dispatch downstream info
code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId,
-1);
code =
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1);
streamFreeQitem((SStreamQueueItem*)pBlock);
}
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@ -399,8 +399,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (taskLevel == TASK_LEVEL__SINK) {
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num);
streamFreeQitem((SStreamQueueItem*)pBlock);
code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
} else { // source & agg tasks need to forward the checkpoint msg downwards
code = streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
} else { // source & agg tasks need to forward the checkpoint msg downwards
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
code = flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock);
if (code) {
@ -445,7 +445,7 @@ static int32_t processCheckpointReadyHelp(SActiveCheckpointInfo* pInfo, int32_t
.transId = pInfo->transId,
.streamId = streamId,
.downstreamNodeId = downstreamNodeId};
void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
if (p == NULL) {
stError("s-task:%s failed to set checkpoint ready recv msg, code:%s", id, tstrerror(terrno));
return terrno;
@ -560,8 +560,8 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
}
streamMutexUnlock(&pInfo->lock);
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%"PRId64", current checkpointId:%"PRId64,
pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64,
pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
}
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
@ -575,8 +575,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
if (pReq->checkpointId <= pInfo->checkpointId) {
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " Ver:%" PRId64
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64
" transId:%d ignored",
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64 " transId:%d ignored",
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
pReq->transId);
streamMutexUnlock(&pTask->lock);
@ -623,7 +622,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
}
bool valid = (pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer &&
pInfo->processedVer <= pReq->checkpointVer);
pInfo->processedVer <= pReq->checkpointVer);
if (!valid) {
stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64
@ -908,7 +907,7 @@ static int32_t doChkptStatusCheck(SStreamTask* pTask) {
if (pTmrInfo->launchChkptId != pActiveInfo->activeId) {
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64
", quit, ref:%d",
", quit, ref:%d",
id, vgId, pTmrInfo->launchChkptId, ref);
return -1;
}
@ -1005,7 +1004,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
int32_t numOfNotSend = 0;
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
@ -1023,7 +1022,8 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
}
if (++pTmrInfo->activeCounter < 50) {
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor");
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"trigger-recv-monitor");
return;
}
@ -1201,8 +1201,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher;
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId};
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) { // pause the stream task, if memory not enough
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) { // pause the stream task, if memory not enough
code = terrno;
}
} else {
@ -1213,8 +1213,8 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
}
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId};
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) { // pause the stream task, if memory not enough
void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) { // pause the stream task, if memory not enough
code = terrno;
break;
}
@ -1288,11 +1288,11 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
static int32_t uploadCheckpointToS3(const char* id, const char* path) {
int32_t code = 0;
int32_t nBytes = 0;
/*
if (s3Init() != 0) {
return TSDB_CODE_THIRDPARTY_ERROR;
}
*/
TdDirPtr pDir = taosOpenDir(path);
if (pDir == NULL) {
return terrno;
@ -1325,11 +1325,11 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) {
break;
}
code = s3PutObjectFromFile2(filename, object, 0);
code = tcsPutObjectFromFile2(filename, object, 0);
if (code != 0) {
stError("[s3] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
stError("[tcs] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
} else {
stDebug("[s3] upload checkpoint:%s", filename);
stDebug("[tcs] upload checkpoint:%s", filename);
}
}
@ -1355,7 +1355,7 @@ int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char
taosMemoryFree(buf);
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t code = s3GetObjectToFile(buf, dstName);
int32_t code = tcsGetObjectToFile(buf, dstName);
if (code != 0) {
taosMemoryFree(buf);
return TAOS_SYSTEM_ERROR(errno);
@ -1418,7 +1418,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path, int64_t che
if (strlen(tsSnodeAddress) != 0) {
return downloadByRsync(id, path, checkpointId);
} else if (tsS3StreamEnabled) {
return s3GetObjectsByPrefix(id, path);
return tcsGetObjectsByPrefix(id, path);
}
return 0;
@ -1432,7 +1432,7 @@ int32_t deleteCheckpoint(const char* id) {
if (strlen(tsSnodeAddress) != 0) {
return deleteRsync(id);
} else if (tsS3StreamEnabled) {
s3DeleteObjectsByPrefix(id);
tcsDeleteObjectsByPrefix(id);
}
return 0;
}
@ -1446,7 +1446,7 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
}
char* tmp = object;
int32_t code = s3DeleteObjects((const char**)&tmp, 1);
int32_t code = tcsDeleteObjects((const char**)&tmp, 1);
if (code != 0) {
return TSDB_CODE_THIRDPARTY_ERROR;
}
@ -1488,4 +1488,4 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) {
streamMutexUnlock(&pTask->lock);
return code;
}
}

View File

@ -101,4 +101,4 @@ IF(NOT TD_DARWIN)
NAME backendTest
COMMAND backendTest
)
ENDIF ()
ENDIF ()

View File

@ -0,0 +1,22 @@
aux_source_directory(src TCS_SRC)
add_library(tcs STATIC ${TCS_SRC})
target_include_directories(
tcs
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
tcs
PUBLIC az
PUBLIC common
# PUBLIC cjson
# PUBLIC os
# PUBLIC util
# PUBLIC crypt
)
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TCS_INT_H_
#define _TD_TCS_INT_H_
#include "os.h"
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int8_t tsS3Ablob;
typedef enum {
TOS_PROTO_NIL,
TOS_PROTO_S3,
TOS_PROTO_ABLOB,
} STosProto;
typedef struct {
int32_t (*Begin)();
void (*End)();
int32_t (*CheckCfg)();
int32_t (*PutObjectFromFileOffset)(const char* file, const char* object_name, int64_t offset, int64_t size);
int32_t (*GetObjectBlock)(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock);
void (*DeleteObjectsByPrefix)(const char* prefix);
int32_t (*PutObjectFromFile2)(const char* file, const char* object, int8_t withcp);
int32_t (*GetObjectsByPrefix)(const char* prefix, const char* path);
int32_t (*DeleteObjects)(const char* object_name[], int nobject);
int32_t (*GetObjectToFile)(const char* object_name, const char* fileName);
} STcs;
extern STcs tcs;
#ifdef __cplusplus
}
#endif
#endif // _TD_TCS_INT_H_

105
source/libs/tcs/src/tcs.c Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcs.h"
#include "os.h"
#include "taoserror.h"
#include "tcsInt.h"
#include "tglobal.h"
#include "az.h"
#include "cos.h"
int32_t tcsInit() {
int32_t code = 0;
STosProto proto = tsS3Ablob ? TOS_PROTO_ABLOB : TOS_PROTO_S3;
if (TOS_PROTO_S3 == proto) {
tcs.Begin = s3Begin;
tcs.End = s3End;
tcs.CheckCfg = s3CheckCfg;
tcs.PutObjectFromFileOffset = s3PutObjectFromFileOffset;
tcs.GetObjectBlock = s3GetObjectBlock;
tcs.DeleteObjectsByPrefix = s3DeleteObjectsByPrefix;
tcs.PutObjectFromFile2 = s3PutObjectFromFile2;
tcs.GetObjectsByPrefix = s3GetObjectsByPrefix;
tcs.DeleteObjects = s3DeleteObjects;
tcs.GetObjectToFile = s3GetObjectToFile;
} else if (TOS_PROTO_ABLOB == proto) {
tcs.Begin = azBegin;
tcs.End = azEnd;
tcs.CheckCfg = azCheckCfg;
tcs.PutObjectFromFileOffset = azPutObjectFromFileOffset;
tcs.GetObjectBlock = azGetObjectBlock;
tcs.DeleteObjectsByPrefix = azDeleteObjectsByPrefix;
tcs.PutObjectFromFile2 = azPutObjectFromFile2;
tcs.GetObjectsByPrefix = azGetObjectsByPrefix;
tcs.DeleteObjects = azDeleteObjects;
tcs.GetObjectToFile = azGetObjectToFile;
} else {
code = TSDB_CODE_INVALID_PARA;
return code;
}
code = tcs.Begin();
return code;
}
void tcsUninit() { tcs.End(); }
int32_t tcsCheckCfg() {
int32_t code = 0;
if (!tsS3Enabled) {
(void)fprintf(stderr, "tcs not configured.\n");
TAOS_RETURN(code);
}
code = tcsInit();
if (code != 0) {
(void)fprintf(stderr, "failed to initialize tcs.\n");
TAOS_RETURN(code);
}
code = tcs.CheckCfg();
if (code != 0) {
(void)fprintf(stderr, "failed to check tcs.\n");
TAOS_RETURN(code);
}
tcsUninit();
return code;
}
int32_t tcsPutObjectFromFileOffset(const char* file, const char* object_name, int64_t offset, int64_t size) {
return tcs.PutObjectFromFileOffset(file, object_name, offset, size);
}
int32_t tcsGetObjectBlock(const char* object_name, int64_t offset, int64_t size, bool check, uint8_t** ppBlock) {
return tcs.GetObjectBlock(object_name, offset, size, check, ppBlock);
}
void tcsDeleteObjectsByPrefix(const char* prefix) { return tcs.DeleteObjectsByPrefix(prefix); }

View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcs.h"
#include "tcsInt.h"
STcs tcs;
int32_t tcsPutObjectFromFile2(const char* file, const char* object, int8_t withcp) {
return tcs.PutObjectFromFile2(file, object, withcp);
}
int32_t tcsGetObjectsByPrefix(const char* prefix, const char* path) { return tcs.GetObjectsByPrefix(prefix, path); }
int32_t tcsDeleteObjects(const char* object_name[], int nobject) { return tcs.DeleteObjects(object_name, nobject); }
int32_t tcsGetObjectToFile(const char* object_name, const char* fileName) {
return tcs.GetObjectToFile(object_name, fileName);
}

View File

@ -0,0 +1,22 @@
if (TD_LINUX)
aux_source_directory(. TCS_TEST_SRC)
add_executable(tcsTest ${TCS_TEST_SRC})
target_include_directories(tcsTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/tcs"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(tcsTest
tcs
gtest_main
)
enable_testing()
add_test(
NAME tcs_test
COMMAND tcsTest
)
endif()

View File

@ -0,0 +1,351 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <cstring>
#include <iostream>
#include <queue>
#include "tcs.h"
#include "tcsInt.h"
int32_t tcsInitEnv(int8_t isBlob) {
int32_t code = 0;
extern char tsS3Hostname[][TSDB_FQDN_LEN];
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
extern char tsS3BucketName[TSDB_FQDN_LEN];
/* TCS parameter format
tsS3Hostname[0] = "<endpoint>/<account-name>.blob.core.windows.net";
tsS3AccessKeyId[0] = "<access-key-id/account-name>";
tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
tsS3BucketName = "<bucket/container-name>";
*/
tsS3Ablob = isBlob;
if (isBlob) {
const char *hostname = "<endpoint>/<account-name>.blob.core.windows.net";
const char *accessKeyId = "<access-key-id/account-name>";
const char *accessKeySecret = "<access-key-secret/account-key>";
const char *bucketName = "<bucket/container-name>";
if (hostname[0] != '<') {
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
} else {
const char *accountId = getenv("ablob_account_id");
if (!accountId) {
return -1;
}
const char *accountSecret = getenv("ablob_account_secret");
if (!accountSecret) {
return -1;
}
const char *containerName = getenv("ablob_container");
if (!containerName) {
return -1;
}
TAOS_STRCPY(&tsS3Hostname[0][0], accountId);
TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net");
TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId);
TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret);
TAOS_STRCPY(tsS3BucketName, containerName);
}
} else {
/*
const char *hostname = "endpoint/<account-name>.blob.core.windows.net";
const char *accessKeyId = "<access-key-id/account-name>";
const char *accessKeySecret = "<access-key-secret/account-key>";
const char *bucketName = "<bucket/container-name>";
*/
// const char *hostname = "http://192.168.1.52:9000";
// const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel";
// const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX";
// const char *bucketName = "test-bucket";
const char *hostname = "192.168.1.52:9000";
const char *accessKeyId = "fGPPyYjzytw05nw44ViA";
const char *accessKeySecret = "vK1VcwxgSOykicx6hk8fL1x15uEtyDSFU3w4hTaZ";
const char *bucketName = "ci-bucket19";
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
// setup s3 env
extern int8_t tsS3EpNum;
extern int8_t tsS3Https[TSDB_MAX_EP_NUM];
tsS3EpNum = 1;
tsS3Https[0] = false;
}
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
tsS3Enabled = true;
return code;
}
// TEST(TcsTest, DISABLED_InterfaceTest) {
TEST(TcsTest, InterfaceTest) {
int code = 0;
bool check = false;
bool withcp = false;
code = tcsInitEnv(true);
if (code) {
std::cout << "ablob env init failed with: " << code << std::endl;
return;
}
GTEST_ASSERT_EQ(code, 0);
GTEST_ASSERT_EQ(tsS3Enabled, 1);
GTEST_ASSERT_EQ(tsS3Ablob, 1);
code = tcsInit();
GTEST_ASSERT_EQ(code, 0);
code = tcsCheckCfg();
GTEST_ASSERT_EQ(code, 0);
const int size = 4096;
char data[size] = {0};
for (int i = 0; i < size / 2; ++i) {
data[i * 2 + 1] = 1;
}
const char object_name[] = "tcsut.bin";
char path[PATH_MAX] = {0};
char path_download[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
} else {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
}
tstrncpy(path_download, path, strlen(path) + 1);
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
GTEST_ASSERT_NE(fp, nullptr);
int n = taosWriteFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
code = tcsPutObjectFromFileOffset(path, object_name, 0, size);
GTEST_ASSERT_EQ(code, 0);
uint8_t *pBlock = NULL;
code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
}
taosMemoryFree(pBlock);
code = tcsGetObjectToFile(object_name, path_download);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
tcsDeleteObjectsByPrefix(object_name);
// list object to check
code = tcsPutObjectFromFile2(path, object_name, withcp);
GTEST_ASSERT_EQ(code, 0);
code = tcsGetObjectsByPrefix(object_name, tsTempDir);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
const char *object_name_arr[] = {object_name};
code = tcsDeleteObjects(object_name_arr, 1);
GTEST_ASSERT_EQ(code, 0);
tcsUninit();
}
// TEST(TcsTest, DISABLED_InterfaceNonBlobTest) {
TEST(TcsTest, InterfaceNonBlobTest) {
int code = 0;
bool check = false;
bool withcp = false;
code = tcsInitEnv(false);
GTEST_ASSERT_EQ(code, 0);
GTEST_ASSERT_EQ(tsS3Enabled, 1);
GTEST_ASSERT_EQ(tsS3Ablob, 0);
code = tcsInit();
GTEST_ASSERT_EQ(code, 0);
code = tcsCheckCfg();
GTEST_ASSERT_EQ(code, 0);
const int size = 4096;
char data[size] = {0};
for (int i = 0; i < size / 2; ++i) {
data[i * 2 + 1] = 1;
}
const char object_name[] = "tcsut.bin";
char path[PATH_MAX] = {0};
char path_download[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
} else {
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
}
tstrncpy(path_download, path, strlen(path) + 1);
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
GTEST_ASSERT_NE(fp, nullptr);
int n = taosWriteFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
code = tcsPutObjectFromFileOffset(path, object_name, 0, size);
GTEST_ASSERT_EQ(code, 0);
uint8_t *pBlock = NULL;
code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
}
taosMemoryFree(pBlock);
code = tcsGetObjectToFile(object_name, path_download);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
tcsDeleteObjectsByPrefix(object_name);
// list object to check
code = tcsPutObjectFromFile2(path, object_name, withcp);
GTEST_ASSERT_EQ(code, 0);
code = tcsGetObjectsByPrefix(object_name, tsTempDir);
GTEST_ASSERT_EQ(code, 0);
{
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
GTEST_ASSERT_NE(fp, nullptr);
(void)memset(data, 0, size);
int64_t n = taosReadFile(fp, data, size);
GTEST_ASSERT_EQ(n, size);
code = taosCloseFile(&fp);
GTEST_ASSERT_EQ(code, 0);
for (int i = 0; i < size / 2; ++i) {
GTEST_ASSERT_EQ(data[i * 2], 0);
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
}
}
const char *object_name_arr[] = {object_name};
code = tcsDeleteObjects(object_name_arr, 1);
GTEST_ASSERT_EQ(code, 0);
tcsUninit();
}

View File

@ -114,6 +114,7 @@ typedef struct SExHandle {
void* handle;
int64_t refId;
void* pThrd;
int8_t pThrdIdx;
queue q;
int8_t inited;
SRWLatch latch;
@ -135,57 +136,33 @@ typedef struct SCvtAddr {
bool cvt;
} SCvtAddr;
typedef struct {
SEpSet epSet; // ip list provided by app
SEpSet origEpSet;
void* ahandle; // handle provided by app
tmsg_t msgType; // message type
int8_t connType; // connection type cli/srv
STransCtx appCtx; //
STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
STransSyncMsg* pSyncMsg; // for syncchronous with timeout API
int64_t syncMsgRef;
SCvtAddr cvtAddr;
int32_t retryMinInterval;
int32_t retryMaxInterval;
int32_t retryStepFactor;
int64_t retryMaxTimeout;
int64_t retryInitTimestamp;
int64_t retryNextInterval;
bool retryInit;
int32_t retryStep;
int8_t epsetRetryCnt;
int32_t retryCode;
void* task;
int hThrdIdx;
} STransConnCtx;
#pragma pack(push, 1)
typedef struct {
int8_t inUse;
int8_t numOfEps;
SEp eps[];
} SReqEpSet;
#define TRANS_VER 2
typedef struct {
char version : 4; // RPC version
char comp : 2; // compression algorithm, 0:no compression 1:lz4
char noResp : 2; // noResp bits, 0: resp, 1: resp
char persist : 2; // persist handle,0: no persit, 1: persist handle
char release : 2;
char version : 4; // RPC version
char comp : 2; // compression algorithm, 0:no compression 1:lz4
char noResp : 2; // noResp bits, 0: resp, 1: resp
char withUserInfo : 2; // 0: sent user info or not
char secured : 2;
char spi : 2;
char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset
uint64_t timestamp;
char user[TSDB_UNI_LEN];
int32_t compatibilityVer;
uint32_t magicNum;
STraceId traceId;
uint64_t ahandle; // ahandle assigned by client
uint32_t code; // del later
int64_t qid;
uint32_t code; // del later
uint32_t msgType;
int32_t msgLen;
int64_t seqNum;
uint8_t content[0]; // message body starts from here
} STransMsgHead;
@ -206,6 +183,35 @@ typedef struct {
#pragma pack(pop)
int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet);
int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet);
int32_t transValidReqEpset(SReqEpSet* pReqEpSet);
typedef struct {
SReqEpSet* epSet; // ip list provided by app
SReqEpSet* origEpSet;
void* ahandle; // handle provided by app
tmsg_t msgType; // message type
STransCtx userCtx; //
STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
STransSyncMsg* pSyncMsg; // for syncchronous with timeout API
int64_t syncMsgRef;
SCvtAddr* pCvtAddr;
int64_t retryInitTimestamp;
int64_t retryNextInterval;
int64_t retryMaxTimeout;
int32_t retryMinInterval;
int32_t retryMaxInterval;
int32_t retryStepFactor;
int32_t retryStep;
int32_t retryCode;
int8_t retryInit;
int8_t epsetRetryCnt;
} SReqCtx;
typedef enum { Normal, Quit, Release, Register, Update, FreeById } STransMsgType;
typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus;
@ -272,24 +278,24 @@ bool transAsyncPoolIsEmpty(SAsyncPool* pool);
} \
} while (0)
#define ASYNC_CHECK_HANDLE(exh1, id) \
do { \
if (id > 0) { \
SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), id); \
if (exh2 == NULL || id != exh2->refId) { \
tDebug("ref:%" PRId64 " already released", id); \
code = terrno; \
goto _return1; \
} \
} else { \
tDebug("invalid handle to release"); \
goto _return2; \
} \
#define ASYNC_CHECK_HANDLE(idMgt, id, exh1) \
do { \
if (id > 0) { \
SExHandle* exh2 = transAcquireExHandle(idMgt, id); \
if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) { \
tError("handle not match, exh1:%p, exh2:%p, refId:%"PRId64"", exh1, exh2, id); \
code = TSDB_CODE_INVALID_MSG; \
goto _return1; \
} \
} else { \
tError("invalid handle to release"); \
goto _return2; \
} \
} while (0)
int32_t transInitBuffer(SConnBuffer* buf);
int32_t transClearBuffer(SConnBuffer* buf);
int32_t transDestroyBuffer(SConnBuffer* buf);
void transDestroyBuffer(SConnBuffer* buf);
int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
bool transReadComplete(SConnBuffer* connBuf);
int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf);
@ -300,30 +306,31 @@ int32_t transSetConnOption(uv_tcp_t* stream, int keepalive);
void transRefSrvHandle(void* handle);
void transUnrefSrvHandle(void* handle);
void transRefCliHandle(void* handle);
void transUnrefCliHandle(void* handle);
void transRefCliHandle(void* handle);
int32_t transUnrefCliHandle(void* handle);
int32_t transGetRefCount(void* handle);
int32_t transReleaseCliHandle(void* handle);
int32_t transReleaseSrvHandle(void* handle);
int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx);
int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp);
int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated,
int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx);
int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp);
int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated,
int32_t timeoutMs);
int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId);
int32_t transFreeConnById(void* shandle, int64_t transpointId);
int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId);
int32_t transFreeConnById(void* pInit, int64_t transpointId);
int32_t transSendResponse(const STransMsg* msg);
int32_t transRegisterMsg(const STransMsg* msg);
int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn);
int32_t transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func);
int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn);
int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func);
int32_t transSockInfo2Str(struct sockaddr* sockname, char* dst);
void transSockInfo2Str(struct sockaddr* sockname, char* dst);
int32_t transAllocHandle(int64_t* refId);
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
void transCloseClient(void* arg);
void transCloseServer(void* arg);
@ -336,33 +343,29 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key);
void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType);
// request list
typedef struct STransReq {
queue q;
uv_write_t wreq;
} STransReq;
void transReqQueueInit(queue* q);
void* transReqQueuePush(queue* q);
void* transReqQueueRemove(void* arg);
void transReqQueueClear(queue* q);
typedef struct SWriteReq {
queue node; // req queue node
void* conn;
} SWriteReq;
// queue sending msgs
typedef struct {
SArray* q;
void (*freeFunc)(const void* arg);
queue node;
void (*freeFunc)(void* arg);
int32_t size;
} STransQueue;
/*
* init queue
* note: queue'size is small, default 1
*/
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg));
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(void* arg));
/*
* put arg into queue
* if queue'size > 1, return false; else return true
*/
bool transQueuePush(STransQueue* queue, void* arg);
void transQueuePush(STransQueue* queue, void* arg);
/*
* the size of queue
*/
@ -375,10 +378,25 @@ void* transQueuePop(STransQueue* queue);
* get ith from queue
*/
void* transQueueGet(STransQueue* queue, int i);
/*
* head elm from queue
*/
void* tranQueueHead(STransQueue* q);
/*
* remove all match elm from queue
*/
void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size);
/*
* rm ith from queue
*/
void* transQueueRm(STransQueue* queue, int i);
/*
* remove el from queue
*/
void transQueueRemove(STransQueue* q, void* e);
/*
* queue empty or not
*/
@ -418,9 +436,9 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg));
SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
void transDQCancel(SDelayQueue* queue, SDelayTask* task);
bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b);
bool transEpSetIsEqual2(SEpSet* a, SEpSet* b);
bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b);
/*
* init global func
*/
@ -432,14 +450,14 @@ void transPrintEpSet(SEpSet* pEpSet);
void transFreeMsg(void* msg);
int32_t transCompressMsg(char* msg, int32_t len);
int32_t transDecompressMsg(char** msg, int32_t len);
int32_t transDecompressMsg(char** msg, int32_t* len);
int32_t transOpenRefMgt(int size, void (*func)(void*));
void transCloseRefMgt(int32_t refMgt);
int64_t transAddExHandle(int32_t refMgt, void* p);
int32_t transRemoveExHandle(int32_t refMgt, int64_t refId);
void transRemoveExHandle(int32_t refMgt, int64_t refId);
void* transAcquireExHandle(int32_t refMgt, int64_t refId);
int32_t transReleaseExHandle(int32_t refMgt, int64_t refId);
void transReleaseExHandle(int32_t refMgt, int64_t refId);
void transDestroyExHandle(void* handle);
int32_t transGetRefMgt();
@ -465,6 +483,33 @@ int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf);
int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf);
int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf);
enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING };
#if defined(WINDOWS) || defined(DARWIN)
#define BUFFER_LIMIT 1
#define STATE_BUFFER_LIMIT 1
#else
#define BUFFER_LIMIT 4
#define STATE_BUFFER_LIMIT 8
#endif
#define HEAP_MISS_HIT_LIMIT 100000
#define READ_TIMEOUT 100000
typedef struct {
queue node; // queue for write
queue q; // queue for reqs
uv_write_t wreq;
void* arg;
} SWReqsWrapper;
int32_t initWQ(queue* wq);
void destroyWQ(queue* wq);
uv_write_t* allocWReqFromWQ(queue* wq, void* arg);
void freeWReqToWQ(queue* wq, SWReqsWrapper* w);
int32_t transSetReadOption(uv_handle_t* handle);
#ifdef __cplusplus
}
#endif

View File

@ -32,8 +32,8 @@
extern "C" {
#endif
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit);
void taosCloseServer(void* arg);
void taosCloseClient(void* arg);
@ -70,12 +70,16 @@ typedef struct {
int32_t connLimitNum;
int8_t connLimitLock; // 0: no lock. 1. lock
int8_t supportBatch; // 0: no batch, 1: support batch
int32_t batchSize;
int32_t shareConnLimit;
int8_t optBatchFetch;
int32_t timeToGetConn;
int index;
void* parent;
void* tcphandle; // returned handle from TCP initialization
int64_t refId;
int8_t shareConn;
int8_t startReadTimer;
int64_t readTimeout;
TdThreadMutex mutex;
} SRpcInfo;

View File

@ -15,13 +15,13 @@
#include "transComm.h"
void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = {
void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* pInit) = {
transInitServer, transInitClient};
void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient};
void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle};
void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle};
void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, NULL};
int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle};
@ -42,6 +42,8 @@ void* rpcOpen(const SRpcInit* pInit) {
if (pRpc == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _end);
}
pRpc->startReadTimer = pInit->startReadTimer;
if (pInit->label) {
int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label);
memcpy(pRpc->label, pInit->label, len);
@ -77,7 +79,15 @@ void* rpcOpen(const SRpcInit* pInit) {
pRpc->connLimitLock = pInit->connLimitLock;
pRpc->supportBatch = pInit->supportBatch;
pRpc->batchSize = pInit->batchSize;
pRpc->shareConnLimit = pInit->shareConnLimit;
if (pRpc->shareConnLimit <= 0) {
pRpc->shareConnLimit = BUFFER_LIMIT;
}
pRpc->readTimeout = pInit->readTimeout;
if (pRpc->readTimeout < 0) {
pRpc->readTimeout = INT64_MAX;
}
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
if (pRpc->numOfThreads <= 0) {
@ -115,6 +125,8 @@ void* rpcOpen(const SRpcInit* pInit) {
int64_t refId = transAddExHandle(transGetInstMgt(), pRpc);
void* tmp = transAcquireExHandle(transGetInstMgt(), refId);
pRpc->refId = refId;
pRpc->shareConn = pInit->shareConn;
return (void*)refId;
_end:
taosMemoryFree(pRpc);
@ -127,9 +139,8 @@ void rpcClose(void* arg) {
if (arg == NULL) {
return;
}
TAOS_UNUSED(transRemoveExHandle(transGetInstMgt(), (int64_t)arg));
TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)arg));
transRemoveExHandle(transGetInstMgt(), (int64_t)arg);
transReleaseExHandle(transGetInstMgt(), (int64_t)arg);
tInfo("end to close rpc");
return;
}
@ -175,29 +186,29 @@ void* rpcReallocCont(void* ptr, int64_t contLen) {
return st + TRANS_MSG_OVERHEAD;
}
int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
return transSendRequest(shandle, pEpSet, pMsg, NULL);
int32_t rpcSendRequest(void* pInit, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
return transSendRequest(pInit, pEpSet, pMsg, NULL);
}
int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
int32_t rpcSendRequestWithCtx(void* pInit, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) {
return transSendRequest(shandle, pEpSet, pMsg, pCtx);
return transSendRequest(pInit, pEpSet, pMsg, pCtx);
} else {
return transSendRequestWithId(shandle, pEpSet, pMsg, pRid);
return transSendRequestWithId(pInit, pEpSet, pMsg, pRid);
}
}
int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) {
return transSendRequestWithId(shandle, pEpSet, pReq, transpointId);
int32_t rpcSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) {
return transSendRequestWithId(pInit, pEpSet, pReq, transpointId);
}
int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
return transSendRecv(shandle, pEpSet, pMsg, pRsp);
int32_t rpcSendRecv(void* pInit, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
return transSendRecv(pInit, pEpSet, pMsg, pRsp);
}
int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated,
int32_t rpcSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated,
int32_t timeoutMs) {
return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs);
return transSendRecvWithTimeout(pInit, pEpSet, pMsg, pRsp, epUpdated, timeoutMs);
}
int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); }
int32_t rpcFreeConnById(void* pInit, int64_t connId) { return transFreeConnById(pInit, connId); }
int32_t rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); }

File diff suppressed because it is too large Load Diff

View File

@ -15,7 +15,7 @@
#include "transComm.h"
#define BUFFER_CAP 4096
#define BUFFER_CAP 8 * 1024
static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
@ -59,7 +59,7 @@ int32_t transCompressMsg(char* msg, int32_t len) {
taosMemoryFree(buf);
return ret;
}
int32_t transDecompressMsg(char** msg, int32_t len) {
int32_t transDecompressMsg(char** msg, int32_t* len) {
STransMsgHead* pHead = (STransMsgHead*)(*msg);
if (pHead->comp == 0) return 0;
@ -68,16 +68,18 @@ int32_t transDecompressMsg(char** msg, int32_t len) {
STransCompMsg* pComp = (STransCompMsg*)pCont;
int32_t oriLen = htonl(pComp->contLen);
char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead));
int32_t tlen = *len;
char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead));
if (buf == NULL) {
return terrno;
}
STransMsgHead* pNewHead = (STransMsgHead*)buf;
int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content,
len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
tlen - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead));
*len = oriLen + sizeof(STransMsgHead);
pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead));
taosMemoryFree(pHead);
@ -95,13 +97,12 @@ void transFreeMsg(void* msg) {
tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD);
taosMemoryFree((char*)msg - sizeof(STransMsgHead));
}
int transSockInfo2Str(struct sockaddr* sockname, char* dst) {
void transSockInfo2Str(struct sockaddr* sockname, char* dst) {
struct sockaddr_in addr = *(struct sockaddr_in*)sockname;
char buf[20] = {0};
int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf));
sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port));
return r;
}
int32_t transInitBuffer(SConnBuffer* buf) {
buf->buf = taosMemoryCalloc(1, BUFFER_CAP);
@ -116,10 +117,9 @@ int32_t transInitBuffer(SConnBuffer* buf) {
buf->invalid = 0;
return 0;
}
int32_t transDestroyBuffer(SConnBuffer* p) {
void transDestroyBuffer(SConnBuffer* p) {
taosMemoryFree(p->buf);
p->buf = NULL;
return 0;
}
int32_t transClearBuffer(SConnBuffer* buf) {
@ -184,7 +184,7 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) {
}
}
} else {
tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG));
tError("failed to reset buffer, total:%d, len:%d since %s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG));
return TSDB_CODE_INVALID_MSG;
}
return 0;
@ -281,7 +281,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs
async->data = item;
err = uv_async_init(loop, async, cb);
if (err != 0) {
tError("failed to init async, reason:%s", uv_err_name(err));
tError("failed to init async since %s", uv_err_name(err));
code = TSDB_CODE_THIRDPARTY_ERROR;
break;
}
@ -333,14 +333,16 @@ int transAsyncSend(SAsyncPool* pool, queue* q) {
SAsyncItem* item = async->data;
if (taosThreadMutexLock(&item->mtx) != 0) {
tError("failed to lock mutex");
tError("failed to lock mutex since %s", tstrerror(terrno));
return terrno;
}
QUEUE_PUSH(&item->qmsg, q);
TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx));
int ret = uv_async_send(async);
if (ret != 0) {
tError("failed to send async,reason:%s", uv_err_name(ret));
tError("failed to send async since %s", uv_err_name(ret));
return TSDB_CODE_THIRDPARTY_ERROR;
}
return 0;
@ -348,15 +350,17 @@ int transAsyncSend(SAsyncPool* pool, queue* q) {
void transCtxInit(STransCtx* ctx) {
// init transCtx
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK);
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
void transCtxCleanup(STransCtx* ctx) {
if (ctx->args == NULL) {
if (ctx == NULL || ctx->args == NULL) {
return;
}
STransCtxVal* iter = taosHashIterate(ctx->args, NULL);
while (iter) {
int32_t* type = taosHashGetKey(iter, NULL);
tDebug("free msg type %s dump func", TMSG_INFO(*type));
ctx->freeFunc(iter->val);
iter = taosHashIterate(ctx->args, iter);
}
@ -385,7 +389,7 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) {
int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
if (code != 0) {
tError("failed to put val to hash, reason:%s", tstrerror(code));
tError("failed to put val to hash since %s", tstrerror(code));
}
iter = taosHashIterate(src->args, iter);
}
@ -415,120 +419,93 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) {
return ret;
}
void transReqQueueInit(queue* q) {
// init req queue
QUEUE_INIT(q);
}
void* transReqQueuePush(queue* q) {
STransReq* req = taosMemoryCalloc(1, sizeof(STransReq));
if (req == NULL) {
return NULL;
}
req->wreq.data = req;
QUEUE_PUSH(q, &req->q);
return &req->wreq;
}
void* transReqQueueRemove(void* arg) {
void* ret = NULL;
uv_write_t* wreq = arg;
STransReq* req = wreq ? wreq->data : NULL;
if (req == NULL) return NULL;
QUEUE_REMOVE(&req->q);
ret = wreq && wreq->handle ? wreq->handle->data : NULL;
taosMemoryFree(req);
return ret;
}
void transReqQueueClear(queue* q) {
while (!QUEUE_IS_EMPTY(q)) {
queue* h = QUEUE_HEAD(q);
QUEUE_REMOVE(h);
STransReq* req = QUEUE_DATA(h, STransReq, q);
taosMemoryFree(req);
}
}
int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) {
queue->q = taosArrayInit(2, sizeof(void*));
if (queue->q == NULL) {
return terrno;
}
queue->freeFunc = (void (*)(const void*))freeFunc;
int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) {
QUEUE_INIT(&wq->node);
wq->freeFunc = (void (*)(void*))freeFunc;
wq->size = 0;
return 0;
}
bool transQueuePush(STransQueue* queue, void* arg) {
if (queue->q == NULL) {
return true;
}
if (taosArrayPush(queue->q, &arg) == NULL) {
return false;
}
if (taosArrayGetSize(queue->q) > 1) {
return false;
}
return true;
void transQueuePush(STransQueue* q, void* arg) {
queue* node = arg;
QUEUE_PUSH(&q->node, node);
q->size++;
}
void* transQueuePop(STransQueue* queue) {
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
return NULL;
}
void* ptr = taosArrayGetP(queue->q, 0);
taosArrayRemove(queue->q, 0);
return ptr;
}
int32_t transQueueSize(STransQueue* queue) {
if (queue->q == NULL) {
return 0;
}
return taosArrayGetSize(queue->q);
}
void* transQueueGet(STransQueue* queue, int i) {
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
return NULL;
}
if (i >= taosArrayGetSize(queue->q)) {
return NULL;
}
void* transQueuePop(STransQueue* q) {
if (q->size == 0) return NULL;
void* ptr = taosArrayGetP(queue->q, i);
return ptr;
queue* head = QUEUE_HEAD(&q->node);
QUEUE_REMOVE(head);
q->size--;
return head;
}
int32_t transQueueSize(STransQueue* q) { return q->size; }
void* transQueueGet(STransQueue* q, int idx) {
if (q->size == 0) return NULL;
while (idx-- > 0) {
queue* node = QUEUE_NEXT(&q->node);
if (node == &q->node) return NULL;
}
return NULL;
}
void* transQueueRm(STransQueue* queue, int i) {
if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
return NULL;
}
if (i >= taosArrayGetSize(queue->q)) {
return NULL;
}
void* ptr = taosArrayGetP(queue->q, i);
taosArrayRemove(queue->q, i);
return ptr;
}
bool transQueueEmpty(STransQueue* queue) {
if (queue->q == NULL) {
return true;
}
return taosArrayGetSize(queue->q) == 0;
}
void transQueueClear(STransQueue* queue) {
if (queue->freeFunc != NULL) {
for (int i = 0; i < taosArrayGetSize(queue->q); i++) {
void* p = taosArrayGetP(queue->q, i);
queue->freeFunc(p);
void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) {
queue* d = dst;
queue* node = QUEUE_NEXT(&q->node);
while (node != &q->node) {
queue* next = QUEUE_NEXT(node);
if (filter && filter(node, arg)) {
QUEUE_REMOVE(node);
q->size--;
QUEUE_PUSH(d, node);
if (--size == 0) {
break;
}
}
node = next;
}
taosArrayClear(queue->q);
}
void transQueueDestroy(STransQueue* queue) {
transQueueClear(queue);
taosArrayDestroy(queue->q);
void* tranQueueHead(STransQueue* q) {
if (q->size == 0) return NULL;
queue* head = QUEUE_HEAD(&q->node);
return head;
}
void* transQueueRm(STransQueue* q, int i) {
// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) {
// return NULL;
// }
// if (i >= taosArrayGetSize(queue->q)) {
// return NULL;
// }
// void* ptr = taosArrayGetP(queue->q, i);
// taosArrayRemove(queue->q, i);
// return ptr;
return NULL;
}
void transQueueRemove(STransQueue* q, void* e) {
if (q->size == 0) return;
queue* node = e;
QUEUE_REMOVE(node);
q->size--;
}
bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; }
void transQueueClear(STransQueue* q) {
while (!QUEUE_IS_EMPTY(&q->node)) {
queue* h = QUEUE_HEAD(&q->node);
QUEUE_REMOVE(h);
if (q->freeFunc != NULL) (q->freeFunc)(h);
q->size--;
}
}
void transQueueDestroy(STransQueue* q) { transQueueClear(q); }
static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
SDelayTask* arg1 = container_of(a, SDelayTask, node);
SDelayTask* arg2 = container_of(b, SDelayTask, node);
@ -690,7 +667,13 @@ void transPrintEpSet(SEpSet* pEpSet) {
len += tsnprintf(buf + len, sizeof(buf) - len, "}");
tTrace("%s, inUse:%d", buf, pEpSet->inUse);
}
bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) {
if (a == NULL && b == NULL) {
return true;
} else if (a == NULL || b == NULL) {
return false;
}
if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) {
return false;
}
@ -701,7 +684,7 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
}
return true;
}
bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) {
bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) {
if (a->numOfEps != b->numOfEps) {
return false;
}
@ -757,28 +740,31 @@ int64_t transAddExHandle(int32_t refMgt, void* p) {
// acquire extern handle
return taosAddRef(refMgt, p);
}
int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) {
void transRemoveExHandle(int32_t refMgt, int64_t refId) {
// acquire extern handle
return taosRemoveRef(refMgt, refId);
int32_t code = taosRemoveRef(refMgt, refId);
if (code != 0) {
tTrace("failed to remove %" PRId64 " from resetId:%d", refId, refMgt);
}
}
void* transAcquireExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle
return (void*)taosAcquireRef(refMgt, refId);
}
int32_t transReleaseExHandle(int32_t refMgt, int64_t refId) {
void transReleaseExHandle(int32_t refMgt, int64_t refId) {
// release extern handle
return taosReleaseRef(refMgt, refId);
int32_t code = taosReleaseRef(refMgt, refId);
if (code != 0) {
tTrace("failed to release %" PRId64 " from resetId:%d", refId, refMgt);
}
}
void transDestroyExHandle(void* handle) {
if (handle == NULL) {
return;
}
SExHandle* eh = handle;
if (!QUEUE_IS_EMPTY(&eh->q)) {
tDebug("handle %p mem leak", handle);
}
tDebug("free exhandle %p", handle);
tDebug("trans destroy sid:%" PRId64 ", memory %p", eh->refId, handle);
taosMemoryFree(handle);
}
@ -841,7 +827,7 @@ int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) {
int32_t err = uv_inet_ntop(AF_INET, &addr, buf, 32);
if (err != 0) {
tError("failed to convert ip to string, reason:%s", uv_strerror(err));
tError("failed to convert ip to string since %s", uv_strerror(err));
return TSDB_CODE_THIRDPARTY_ERROR;
}
@ -890,3 +876,113 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) {
// STUB_RAND_NETWORK_ERR(status)
// return status;
// }
int32_t initWQ(queue* wq) {
int32_t code = 0;
QUEUE_INIT(wq);
for (int i = 0; i < 4; i++) {
SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper));
if (w == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _exception);
}
w->wreq.data = w;
w->arg = NULL;
QUEUE_INIT(&w->node);
QUEUE_PUSH(wq, &w->q);
}
return 0;
_exception:
destroyWQ(wq);
return code;
}
void destroyWQ(queue* wq) {
while (!QUEUE_IS_EMPTY(wq)) {
queue* h = QUEUE_HEAD(wq);
QUEUE_REMOVE(h);
SWReqsWrapper* w = QUEUE_DATA(h, SWReqsWrapper, q);
taosMemoryFree(w);
}
}
uv_write_t* allocWReqFromWQ(queue* wq, void* arg) {
if (!QUEUE_IS_EMPTY(wq)) {
queue* node = QUEUE_HEAD(wq);
QUEUE_REMOVE(node);
SWReqsWrapper* w = QUEUE_DATA(node, SWReqsWrapper, q);
w->arg = arg;
QUEUE_INIT(&w->node);
return &w->wreq;
} else {
SWReqsWrapper* w = taosMemoryCalloc(1, sizeof(SWReqsWrapper));
if (w == NULL) {
return NULL;
}
w->wreq.data = w;
w->arg = arg;
QUEUE_INIT(&w->node);
return &w->wreq;
}
}
void freeWReqToWQ(queue* wq, SWReqsWrapper* w) {
QUEUE_INIT(&w->node);
QUEUE_PUSH(wq, &w->q);
}
int32_t transSetReadOption(uv_handle_t* handle) {
int32_t code = 0;
int32_t fd;
int ret = uv_fileno((uv_handle_t*)handle, &fd);
if (ret != 0) {
tWarn("failed to get fd since %s", uv_err_name(ret));
return TSDB_CODE_THIRDPARTY_ERROR;
}
code = taosSetSockOpt2(fd);
return code;
}
int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) {
if (pEpset == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pReqEpSet == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t size = sizeof(SReqEpSet) + sizeof(SEp) * pEpset->numOfEps;
SReqEpSet* pReq = (SReqEpSet*)taosMemoryCalloc(1, size);
if (pReq == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy((char*)pReq, (char*)pEpset, size);
// clear previous
taosMemoryFree(*pReqEpSet);
if (transValidReqEpset(pReq) != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
return TSDB_CODE_INVALID_PARA;
}
*pReqEpSet = pReq;
return TSDB_CODE_SUCCESS;
}
int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet) {
if (pReqEpSet == NULL) {
return TSDB_CODE_INVALID_PARA;
}
memcpy((char*)pEpSet, (char*)pReqEpSet, sizeof(SReqEpSet) + sizeof(SEp) * pReqEpSet->numOfEps);
return TSDB_CODE_SUCCESS;
}
int32_t transValidReqEpset(SReqEpSet* pReqEpSet) {
if (pReqEpSet == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pReqEpSet->numOfEps == 0 || pReqEpSet->numOfEps > TSDB_MAX_EP_NUM || pReqEpSet->inUse >= TSDB_MAX_EP_NUM) {
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
}

File diff suppressed because it is too large Load Diff

View File

@ -117,7 +117,7 @@ int main(int argc, char *argv[]) {
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.connLimitNum = 10;
rpcInit.connLimitLock = 1;
rpcInit.batchSize = 16 * 1024;
rpcInit.shareConnLimit = 16 * 1024;
rpcInit.supportBatch = 1;
rpcDebugFlag = 135;

View File

@ -395,10 +395,11 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
SetFilePointer(h, 0, NULL, FILE_END);
}
if (h == INVALID_HANDLE_VALUE) {
DWORD dwError = GetLastError();
DWORD dwError = GetLastError();
terrno = TAOS_SYSTEM_WINAPI_ERROR(dwError);
// LPVOID lpMsgBuf;
// FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf, 0,
// FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf,
// 0,
// NULL);
// printf("CreateFile failed with error %d: %s", dwError, (char *)lpMsgBuf);
// LocalFree(lpMsgBuf);
@ -915,7 +916,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
}
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
int32_t code = fstat(pFile->fd, &fileStat);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
@ -983,7 +984,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
}
#ifdef _TD_DARWIN_64
if(lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) {
if (lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -1015,7 +1016,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
}
return writeLen;
#else // for linux
#else // for linux
int64_t leftbytes = size;
int64_t sentbytes;
@ -1126,7 +1127,7 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
if ((*ppFile)->hFile != NULL) {
// FlushFileBuffers((*ppFile)->hFile);
if (!CloseHandle((*ppFile)->hFile)) {
terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError());
code = -1;
}
(*ppFile)->hFile = NULL;
@ -1469,7 +1470,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
while (!feof(pSrcFile->fp)) {
len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
if (len > 0) {
if(gzwrite(dstFp, data, len) == 0) {
if (gzwrite(dstFp, data, len) == 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
ret = terrno;
goto cmp_end;
@ -1564,4 +1565,4 @@ int taosSetAutoDelFile(char *path) {
}
return 0;
#endif
}
}

View File

@ -308,6 +308,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
uint32_t r = taosRand() % tsRandErrDivisor;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("random memory error: %s, %s", tstrerror(terrno), __func__);
return NULL;
}
}

Some files were not shown because too many files have changed in this diff Show More