diff --git a/cmake/cmake.version b/cmake/cmake.version index edc51f206c..fe35fbe7bd 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.6.0.alpha") + SET(TD_VER_NUMBER "3.1.0.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index 43713117f9..2a6cd9ecf7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -20,6 +20,11 @@ The source code for the Python connector is hosted on [GitHub](https://github.co - The [supported platforms](/reference/connector/#supported-platforms) for the native connection are the same as the ones supported by the TDengine client. - REST connections are supported on all platforms that can run Python. +### Supported features + +- Native connections support all the core features of TDengine, including connection management, SQL execution, bind interface, subscriptions, and schemaless writing. +- REST connections support features such as connection management and SQL execution. (SQL execution allows you to: manage databases, tables, and supertables, write data, query data, create continuous queries, etc.). + ## Version selection We recommend using the latest version of `taospy`, regardless of the version of TDengine. @@ -64,10 +69,23 @@ All exceptions from the Python Connector are thrown directly. Applications shoul {{#include docs/examples/python/handle_exception.py}} ``` -## Supported features +## TDengine DataType vs. Python DataType -- Native connections support all the core features of TDengine, including connection management, SQL execution, bind interface, subscriptions, and schemaless writing. -- REST connections support features such as connection management and SQL execution. (SQL execution allows you to: manage databases, tables, and supertables, write data, query data, create continuous queries, etc.). +TDengine currently supports timestamp, number, character, Boolean type, and the corresponding type conversion with Python is as follows: + +|TDengine DataType|Python DataType| +|:---------------:|:-------------:| +|TIMESTAMP|datetime| +|INT|int| +|BIGINT|int| +|FLOAT|float| +|DOUBLE|int| +|SMALLINT|int| +|TINYINT|int| +|BOOL|bool| +|BINARY|str| +|NCHAR|str| +|JSON|str| ## Installation @@ -544,7 +562,7 @@ Connector support data subscription. For more information about subscroption, pl The `consumer` in the connector contains the subscription api. -#### Create Consumer +##### Create Consumer The syntax for creating a consumer is `consumer = Consumer(configs)`. For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/). @@ -554,7 +572,7 @@ from taos.tmq import Consumer consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) ``` -#### Subscribe topics +##### Subscribe topics The `subscribe` function is used to subscribe to a list of topics. @@ -562,7 +580,7 @@ The `subscribe` function is used to subscribe to a list of topics. consumer.subscribe(['topic1', 'topic2']) ``` -#### Consume +##### Consume The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. @@ -580,7 +598,7 @@ while True: print(block.fetchall()) ``` -#### assignment +##### assignment The `assignment` function is used to get the assignment of the topic. @@ -588,7 +606,7 @@ The `assignment` function is used to get the assignment of the topic. assignments = consumer.assignment() ``` -#### Seek +##### Seek The `seek` function is used to reset the assignment of the topic. @@ -597,7 +615,7 @@ tp = TopicPartition(topic='topic1', partition=0, offset=0) consumer.seek(tp) ``` -#### After consuming data +##### After consuming data You should unsubscribe to the topics and close the consumer after consuming. @@ -606,13 +624,13 @@ consumer.unsubscribe() consumer.close() ``` -#### Tmq subscription example +##### Tmq subscription example ```python {{#include docs/examples/python/tmq_example.py}} ``` -#### assignment and seek example +##### assignment and seek example ```python {{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} @@ -624,7 +642,7 @@ consumer.close() In addition to native connections, the connector also supports subscriptions via websockets. -#### Create Consumer +##### Create Consumer The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/#create-a-consumer). @@ -634,7 +652,7 @@ import taosws consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` -#### subscribe topics +##### subscribe topics The `subscribe` function is used to subscribe to a list of topics. @@ -642,7 +660,7 @@ The `subscribe` function is used to subscribe to a list of topics. consumer.subscribe(['topic1', 'topic2']) ``` -#### Consume +##### Consume The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. @@ -659,7 +677,7 @@ while True: print(row) ``` -#### assignment +##### assignment The `assignment` function is used to get the assignment of the topic. @@ -667,7 +685,7 @@ The `assignment` function is used to get the assignment of the topic. assignments = consumer.assignment() ``` -#### Seek +##### Seek The `seek` function is used to reset the assignment of the topic. @@ -675,7 +693,7 @@ The `seek` function is used to reset the assignment of the topic. consumer.seek(topic='topic1', partition=0, offset=0) ``` -#### After consuming data +##### After consuming data You should unsubscribe to the topics and close the consumer after consuming. @@ -684,13 +702,13 @@ consumer.unsubscribe() consumer.close() ``` -#### Subscription example +##### Subscription example ```python {{#include docs/examples/python/tmq_websocket_example.py}} ``` -#### Assignment and seek example +##### Assignment and seek example ```python {{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} @@ -706,19 +724,19 @@ Connector support schemaless insert. -Simple insert +##### Simple insert ```python {{#include docs/examples/python/schemaless_insert.py}} ``` -Insert with ttl argument +##### Insert with ttl argument ```python {{#include docs/examples/python/schemaless_insert_ttl.py}} ``` -Insert with req_id argument +##### Insert with req_id argument ```python {{#include docs/examples/python/schemaless_insert_req_id.py}} @@ -728,19 +746,19 @@ Insert with req_id argument -Simple insert +##### Simple insert ```python {{#include docs/examples/python/schemaless_insert_raw.py}} ``` -Insert with ttl argument +##### Insert with ttl argument ```python {{#include docs/examples/python/schemaless_insert_raw_ttl.py}} ``` -Insert with req_id argument +##### Insert with req_id argument ```python {{#include docs/examples/python/schemaless_insert_raw_req_id.py}} @@ -756,7 +774,7 @@ The Python connector provides a parameter binding api for inserting data. Simila -#### Create Stmt +##### Create Stmt Call the `statement` method in `Connection` to create the `stmt` for parameter binding. @@ -767,7 +785,7 @@ conn = taos.connect() stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") ``` -#### parameter binding +##### parameter binding Call the `new_multi_binds` function to create the parameter list for parameter bindings. @@ -797,7 +815,7 @@ Call the `bind_param` (for a single row) method or the `bind_param_batch` (for m stmt.bind_param_batch(params) ``` -#### execute sql +##### execute sql Call `execute` method to execute sql. @@ -805,13 +823,13 @@ Call `execute` method to execute sql. stmt.execute() ``` -#### Close Stmt +##### Close Stmt ``` stmt.close() ``` -#### Example +##### Example ```python {{#include docs/examples/python/stmt_example.py}} @@ -820,7 +838,7 @@ stmt.close() -#### Create Stmt +##### Create Stmt Call the `statement` method in `Connection` to create the `stmt` for parameter binding. @@ -831,7 +849,7 @@ conn = taosws.connect('taosws://localhost:6041/test') stmt = conn.statement() ``` -#### Prepare sql +##### Prepare sql Call `prepare` method in stmt to prepare sql. @@ -839,7 +857,7 @@ Call `prepare` method in stmt to prepare sql. stmt.prepare("insert into t1 values (?, ?, ?, ?)") ``` -#### parameter binding +##### parameter binding Call the `bind_param` method to bind parameters. @@ -858,7 +876,7 @@ Call the `add_batch` method to add parameters to the batch. stmt.add_batch() ``` -#### execute sql +##### execute sql Call `execute` method to execute sql. @@ -866,13 +884,13 @@ Call `execute` method to execute sql. stmt.execute() ``` -#### Close Stmt +##### Close Stmt ``` stmt.close() ``` -#### Example +##### Example ```python {{#include docs/examples/python/stmt_websocket_example.py}} diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f4d9ba8e42..a5c1553402 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.6.0 + + + ## 3.0.5.1 diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index c3ec224354..0b9f2d75a7 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -21,7 +21,12 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con - 原生连接[支持的平台](../#支持的平台)和 TDengine 客户端支持的平台一致。 - REST 连接支持所有能运行 Python 的平台。 -## 版本选择 +### 支持的功能 + +- 原生连接支持 TDengine 的所有核心功能, 包括: 连接管理、执行 SQL、参数绑定、订阅、无模式写入(schemaless)。 +- REST 连接支持的功能包括:连接管理、执行 SQL。 (通过执行 SQL 可以: 管理数据库、管理表和超级表、写入数据、查询数据、创建连续查询等)。 + +## 历史版本 无论使用什么版本的 TDengine 都建议使用最新版本的 `taospy`。 @@ -65,12 +70,25 @@ Python Connector 的所有数据库操作如果出现异常,都会直接抛出 {{#include docs/examples/python/handle_exception.py}} ``` -## 支持的功能 +TDengine DataType 和 Python DataType -- 原生连接支持 TDengine 的所有核心功能, 包括: 连接管理、执行 SQL、参数绑定、订阅、无模式写入(schemaless)。 -- REST 连接支持的功能包括:连接管理、执行 SQL。 (通过执行 SQL 可以: 管理数据库、管理表和超级表、写入数据、查询数据、创建连续查询等)。 +TDengine 目前支持时间戳、数字、字符、布尔类型,与 Python 对应类型转换如下: -## 安装 +|TDengine DataType|Python DataType| +|:---------------:|:-------------:| +|TIMESTAMP|datetime| +|INT|int| +|BIGINT|int| +|FLOAT|float| +|DOUBLE|int| +|SMALLINT|int| +|TINYINT|int| +|BOOL|bool| +|BINARY|str| +|NCHAR|str| +|JSON|str| + +## 安装步骤 ### 安装前准备 @@ -383,7 +401,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -#### Connection 类的使用 +##### Connection 类的使用 `Connection` 类既包含对 PEP249 Connection 接口的实现(如:cursor方法和 close 方法),也包含很多扩展功能(如: execute、 query、schemaless_insert 和 subscribe 方法。 @@ -547,7 +565,7 @@ RestClient 类是对于 REST API 的直接封装。它只包含一个 sql() 方 `Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API。 -#### 创建 Consumer +##### 创建 Consumer 创建 Consumer 语法为 `consumer = Consumer(configs)`,参数定义请参考 [数据订阅文档](../../develop/tmq/#%E5%88%9B%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85-consumer)。 @@ -557,7 +575,7 @@ from taos.tmq import Consumer consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) ``` -#### 订阅 topics +##### 订阅 topics Consumer API 的 `subscribe` 方法用于订阅 topics,consumer 支持同时订阅多个 topic。 @@ -565,7 +583,7 @@ Consumer API 的 `subscribe` 方法用于订阅 topics,consumer 支持同时 consumer.subscribe(['topic1', 'topic2']) ``` -#### 消费数据 +##### 消费数据 Consumer API 的 `poll` 方法用于消费数据,`poll` 方法接收一个 float 类型的超时时间,超时时间单位为秒(s),`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 @@ -583,7 +601,7 @@ while True: print(block.fetchall()) ``` -#### 获取消费进度 +##### 获取消费进度 Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度,返回结果类型为 TopicPartition 列表。 @@ -591,7 +609,7 @@ Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic assignments = consumer.assignment() ``` -#### 重置消费进度 +##### 指定订阅 Offset Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置,方法参数类型为 TopicPartition。 @@ -600,7 +618,7 @@ tp = TopicPartition(topic='topic1', partition=0, offset=0) consumer.seek(tp) ``` -#### 结束消费 +##### 关闭订阅 消费结束后,应当取消订阅,并关闭 Consumer。 @@ -609,13 +627,13 @@ consumer.unsubscribe() consumer.close() ``` -#### tmq 订阅示例代码 +##### 完整示例 ```python {{#include docs/examples/python/tmq_example.py}} ``` -#### 获取和重置消费进度示例代码 +##### 获取和重置消费进度示例代码 ```python {{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} @@ -629,7 +647,7 @@ consumer.close() taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。 -#### 创建 Consumer +##### 创建 Consumer 创建 Consumer 语法为 `consumer = Consumer(conf=configs)`,使用时需要指定 `td.connect.websocket.scheme` 参数值为 "ws",参数定义请参考 [数据订阅文档](../../develop/tmq/#%E5%88%9B%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85-consumer)。 @@ -639,7 +657,7 @@ import taosws consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` -#### 订阅 topics +##### 订阅 topics Consumer API 的 `subscribe` 方法用于订阅 topics,consumer 支持同时订阅多个 topic。 @@ -647,7 +665,7 @@ Consumer API 的 `subscribe` 方法用于订阅 topics,consumer 支持同时 consumer.subscribe(['topic1', 'topic2']) ``` -#### 消费数据 +##### 消费数据 Consumer API 的 `poll` 方法用于消费数据,`poll` 方法接收一个 float 类型的超时时间,超时时间单位为秒(s),`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 @@ -664,7 +682,7 @@ while True: print(row) ``` -#### 获取消费进度 +##### 获取消费进度 Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度,返回结果类型为 TopicPartition 列表。 @@ -672,7 +690,7 @@ Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic assignments = consumer.assignment() ``` -#### 重置消费进度 +##### 重置消费进度 Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置。 @@ -680,7 +698,7 @@ Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位 consumer.seek(topic='topic1', partition=0, offset=0) ``` -#### 结束消费 +##### 结束消费 消费结束后,应当取消订阅,并关闭 Consumer。 @@ -689,7 +707,7 @@ consumer.unsubscribe() consumer.close() ``` -#### tmq 订阅示例代码 +##### tmq 订阅示例代码 ```python {{#include docs/examples/python/tmq_websocket_example.py}} @@ -697,7 +715,7 @@ consumer.close() 连接器提供了 `assignment` 接口,用于获取 topic assignment 的功能,可以查询订阅的 topic 的消费进度,并提供 `seek` 接口,用于重置 topic 的消费进度。 -#### 获取和重置消费进度示例代码 +##### 获取和重置消费进度示例代码 ```python {{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} @@ -713,19 +731,19 @@ consumer.close() -简单写入 +##### 简单写入 ```python {{#include docs/examples/python/schemaless_insert.py}} ``` -带有 ttl 参数的写入 +##### 带有 ttl 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_ttl.py}} ``` -带有 req_id 参数的写入 +##### 带有 req_id 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_req_id.py}} @@ -735,19 +753,19 @@ consumer.close() -简单写入 +##### 简单写入 ```python {{#include docs/examples/python/schemaless_insert_raw.py}} ``` -带有 ttl 参数的写入 +##### 带有 ttl 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_raw_ttl.py}} ``` -带有 req_id 参数的写入 +##### 带有 req_id 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_raw_req_id.py}} @@ -763,7 +781,7 @@ TDengine 的 Python 连接器支持参数绑定风格的 Prepare API 方式写 -#### 创建 stmt +##### 创建 stmt Python 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt,该方法接收 sql 字符串作为参数,sql 字符串目前仅支持用 `?` 来代表绑定的参数。 @@ -774,7 +792,7 @@ conn = taos.connect() stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") ``` -#### 参数绑定 +##### 参数绑定 调用 `new_multi_binds` 函数创建 params 列表,用于参数绑定。 @@ -804,7 +822,7 @@ params[15].timestamp([None, None, 1626861392591]) stmt.bind_param_batch(params) ``` -#### 执行 sql +##### 执行 sql 调用 stmt 的 `execute` 方法执行 sql @@ -812,7 +830,7 @@ stmt.bind_param_batch(params) stmt.execute() ``` -#### 关闭 stmt +##### 关闭 stmt 最后需要关闭 stmt。 @@ -820,7 +838,7 @@ stmt.execute() stmt.close() ``` -#### 示例代码 +##### 示例代码 ```python {{#include docs/examples/python/stmt_example.py}} @@ -829,7 +847,7 @@ stmt.close() -#### 创建 stmt +##### 创建 stmt Python WebSocket 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt,该方法接收 sql 字符串作为参数,sql 字符串目前仅支持用 `?` 来代表绑定的参数。 @@ -840,7 +858,7 @@ conn = taosws.connect('taosws://localhost:6041/test') stmt = conn.statement() ``` -#### 解析 sql +##### 解析 sql 调用 stmt 的 `prepare` 方法来解析 insert 语句。 @@ -848,7 +866,7 @@ stmt = conn.statement() stmt.prepare("insert into t1 values (?, ?, ?, ?)") ``` -#### 参数绑定 +##### 参数绑定 调用 stmt 的 `bind_param` 方法绑定参数。 @@ -867,7 +885,7 @@ stmt.bind_param([ stmt.add_batch() ``` -#### 执行 sql +##### 执行 sql 调用 stmt 的 `execute` 方法执行 sql @@ -875,7 +893,7 @@ stmt.add_batch() stmt.execute() ``` -#### 关闭 stmt +##### 关闭 stmt 最后需要关闭 stmt。 @@ -883,7 +901,7 @@ stmt.execute() stmt.close() ``` -#### 示例代码 +##### 示例代码 ```python {{#include docs/examples/python/stmt_websocket_example.py}} diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index ae47388566..557552bc1c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.6.0 + + + ## 3.0.5.1 diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 081383f89b..0622b01f2b 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -152,7 +152,7 @@ function wgetFile { file=$1 versionPath=$2 sourceP=$3 -nasServerIP="192.168.1.131" +nasServerIP="192.168.1.213" packagePath="/nas/TDengine/v${versionPath}/${verMode}" if [ -f ${file} ];then echoColor YD "${file} already exists ,it will delete it and download it again " diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index a12f8051ba..1e70ce4a1c 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -47,9 +47,7 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { } static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - terrno = TSDB_CODE_SUCCESS; - + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); @@ -74,13 +72,12 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t capacity = 0; int64_t readSize = 0; char* buf = NULL; - bool firstTrial = pFileInfo->fileSize < fileSize; int64_t offset = TMIN(pFileInfo->fileSize, fileSize); - int64_t offsetForward = offset - stepSize + walCkHeadSz - 1; - int64_t offsetBackward = offset; int64_t retVer = -1; int64_t lastEntryBeginOffset = 0; int64_t lastEntryEndOffset = 0; + int64_t recordLen = 0; + bool forwardStage = false; // check recover size if (2 * tsWalFsyncDataSizeLimit + offset < end) { @@ -91,14 +88,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // search for the valid last WAL entry, e.g. block by block while (1) { - offset = (firstTrial) ? TMIN(fileSize, offsetForward + stepSize - walCkHeadSz + 1) - : TMAX(0, offsetBackward - stepSize + walCkHeadSz - 1); + offset = (lastEntryEndOffset > 0) ? offset : TMAX(0, offset - stepSize + walCkHeadSz - 1); end = TMIN(offset + stepSize, fileSize); - if (firstTrial) { - offsetForward = offset; - } else { - offsetBackward = offset; - } readSize = end - offset; capacity = readSize + sizeof(magic); @@ -129,7 +120,16 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t pos = 0; SWalCkHead* logContent = NULL; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + while (true) { + forwardStage = (lastEntryEndOffset > 0 || offset == 0); + terrno = TSDB_CODE_SUCCESS; + if (forwardStage) { + candidate = (readSize - (haystack - buf)) > 0 ? haystack : NULL; + } else { + candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic)); + } + + if (candidate == NULL) break; pos = candidate - buf; // validate head @@ -137,13 +137,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { if (len < walCkHeadSz) { break; } + logContent = (SWalCkHead*)(buf + pos); if (walValidHeadCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -151,9 +152,9 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } // validate body - int64_t size = walCkHeadSz + logContent->head.bodyLen; - if (len < size) { - int64_t extraSize = size - len; + recordLen = walCkHeadSz + logContent->head.bodyLen; + if (len < recordLen) { + int64_t extraSize = recordLen - len; if (capacity < readSize + extraSize + sizeof(magic)) { capacity += extraSize; void* ptr = taosMemoryRealloc(buf, capacity); @@ -184,7 +185,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -194,21 +195,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // found one retVer = logContent->head.version; lastEntryBeginOffset = offset + pos; - lastEntryEndOffset = offset + pos + sizeof(SWalCkHead) + logContent->head.bodyLen; + lastEntryEndOffset = offset + pos + recordLen; // try next - haystack = buf + pos + 1; + haystack = buf + pos + recordLen; } - if (end == fileSize) firstTrial = false; - if (firstTrial) { - if (terrno == TSDB_CODE_SUCCESS) { - continue; - } else { - firstTrial = false; - } - } - if (retVer >= 0 || offset == 0) break; + offset = (lastEntryEndOffset > 0) ? lastEntryEndOffset : offset; + if (forwardStage && (terrno != TSDB_CODE_SUCCESS || end == fileSize)) break; } if (retVer < 0) {