other: merge main.
This commit is contained in:
Haojun Liao
commit
ad9f3cbc43
|
|
@ -0,0 +1,19 @@
|
|||
# apr-util
|
||||
ExternalProject_Add(aprutil-1
|
||||
URL https://dlcdn.apache.org//apr/apr-util-1.6.3.tar.gz
|
||||
URL_HASH SHA256=2b74d8932703826862ca305b094eef2983c27b39d5c9414442e9976a9acf1983
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||
#GIT_REPOSITORY https://github.com/apache/apr-util.git
|
||||
#GIT_TAG 1.5.4
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/apr-util"
|
||||
#BINARY_DIR ""
|
||||
BUILD_IN_SOURCE TRUE
|
||||
BUILD_ALWAYS 1
|
||||
#UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --with-apr=$ENV{HOME}/.cos-local.1
|
||||
#CONFIGURE_COMMAND ./configure --with-apr=/usr/local/apr
|
||||
BUILD_COMMAND make
|
||||
INSTALL_COMMAND make install
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# apr
|
||||
ExternalProject_Add(apr-1
|
||||
URL https://dlcdn.apache.org//apr/apr-1.7.4.tar.gz
|
||||
URL_HASH SHA256=a4137dd82a185076fa50ba54232d920a17c6469c30b0876569e1c2a05ff311d9
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||
#GIT_REPOSITORY https://github.com/apache/apr.git
|
||||
#GIT_TAG 1.5.2
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/apr"
|
||||
BUILD_IN_SOURCE TRUE
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
BUILD_ALWAYS 1
|
||||
#UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1/ --enable-shared=no
|
||||
#CONFIGURE_COMMAND ./configure
|
||||
BUILD_COMMAND make
|
||||
INSTALL_COMMAND make install
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
|
@ -77,6 +77,12 @@ ELSE ()
|
|||
SET(TD_TAOS_TOOLS TRUE)
|
||||
ENDIF ()
|
||||
|
||||
IF (${TD_WINDOWS})
|
||||
SET(TAOS_LIB taos_static)
|
||||
ELSE ()
|
||||
SET(TAOS_LIB taos)
|
||||
ENDIF ()
|
||||
|
||||
IF (TD_WINDOWS)
|
||||
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
|
||||
SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
|
||||
|
|
|
|||
|
|
@ -125,6 +125,16 @@ option(
|
|||
ON
|
||||
)
|
||||
|
||||
IF(${TD_LINUX})
|
||||
|
||||
option(
|
||||
BUILD_WITH_COS
|
||||
"If build with cos"
|
||||
ON
|
||||
)
|
||||
|
||||
ENDIF ()
|
||||
|
||||
option(
|
||||
BUILD_WITH_SQLITE
|
||||
"If build with sqlite"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
# cos
|
||||
ExternalProject_Add(cos
|
||||
GIT_REPOSITORY https://github.com/tencentyun/cos-c-sdk-v5.git
|
||||
GIT_TAG v5.0.16
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/cos-c-sdk-v5"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# curl
|
||||
ExternalProject_Add(curl
|
||||
URL https://curl.se/download/curl-8.2.1.tar.gz
|
||||
DOWNLOAD_NO_PROGRESS 1
|
||||
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
|
||||
#GIT_REPOSITORY https://github.com/curl/curl.git
|
||||
#GIT_TAG curl-7_88_1
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/curl"
|
||||
BUILD_IN_SOURCE TRUE
|
||||
BUILD_ALWAYS 1
|
||||
#UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --without-ssl --enable-shared=no --disable-ldap --disable-ldaps --without-brotli
|
||||
#CONFIGURE_COMMAND ./configure --without-ssl
|
||||
BUILD_COMMAND make
|
||||
INSTALL_COMMAND make install
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# cos
|
||||
ExternalProject_Add(mxml
|
||||
GIT_REPOSITORY https://github.com/michaelrsweet/mxml.git
|
||||
GIT_TAG release-2.10
|
||||
SOURCE_DIR "${TD_CONTRIB_DIR}/mxml"
|
||||
#BINARY_DIR ""
|
||||
BUILD_IN_SOURCE TRUE
|
||||
#UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.1 --enable-shared=no
|
||||
#CONFIGURE_COMMAND ./configure
|
||||
BUILD_COMMAND make
|
||||
INSTALL_COMMAND make install
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
|
@ -6,6 +6,39 @@ function(cat IN_FILE OUT_FILE)
|
|||
file(APPEND ${OUT_FILE} "${CONTENTS}")
|
||||
endfunction(cat IN_FILE OUT_FILE)
|
||||
|
||||
if(${TD_LINUX})
|
||||
|
||||
set(CONTRIB_TMP_FILE3 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in3")
|
||||
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
||||
|
||||
if(${BUILD_WITH_COS})
|
||||
file(MAKE_DIRECTORY $ENV{HOME}/.cos-local.1/)
|
||||
cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
||||
cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
||||
cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE3})
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
configure_file(${CONTRIB_TMP_FILE3} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
|
||||
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
|
||||
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
||||
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
|
||||
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
||||
|
||||
set(CONTRIB_TMP_FILE2 "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in2")
|
||||
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
|
||||
|
||||
if(${BUILD_WITH_COS})
|
||||
cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE2})
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
configure_file(${CONTRIB_TMP_FILE2} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
|
||||
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
|
||||
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
||||
execute_process(COMMAND "${CMAKE_COMMAND}" --build .
|
||||
WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
|
||||
|
||||
endif(${TD_LINUX})
|
||||
|
||||
set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
|
||||
configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
|
||||
|
|
@ -122,6 +155,16 @@ if(${BUILD_WITH_SQLITE})
|
|||
cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
endif(${BUILD_WITH_SQLITE})
|
||||
|
||||
# cos
|
||||
if(${BUILD_WITH_COS})
|
||||
#cat("${TD_SUPPORT_DIR}/mxml_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
#cat("${TD_SUPPORT_DIR}/apr_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
#cat("${TD_SUPPORT_DIR}/apr-util_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
#cat("${TD_SUPPORT_DIR}/curl_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
cat("${TD_SUPPORT_DIR}/cos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
add_definitions(-DUSE_COS)
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
# lucene
|
||||
if(${BUILD_WITH_LUCENE})
|
||||
cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
|
|
@ -347,6 +390,31 @@ if (${BUILD_WITH_ROCKSDB})
|
|||
endif()
|
||||
endif()
|
||||
|
||||
# cos
|
||||
if(${BUILD_WITH_COS})
|
||||
if(${TD_LINUX})
|
||||
set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.1)
|
||||
#ADD_DEFINITIONS(-DMINIXML_LIBRARY=${CMAKE_BINARY_DIR}/build/lib/libxml.a)
|
||||
option(ENABLE_TEST "Enable the tests" OFF)
|
||||
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.1/include)
|
||||
MESSAGE("$ENV{HOME}/.cos-local.1/include")
|
||||
|
||||
set(CMAKE_BUILD_TYPE debug)
|
||||
set(ORIG_CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME})
|
||||
set(CMAKE_PROJECT_NAME cos_c_sdk)
|
||||
|
||||
add_subdirectory(cos-c-sdk-v5 EXCLUDE_FROM_ALL)
|
||||
target_include_directories(
|
||||
cos_c_sdk
|
||||
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cos-c-sdk-v5/cos_c_sdk>
|
||||
)
|
||||
|
||||
set(CMAKE_PROJECT_NAME ${ORIG_CMAKE_PROJECT_NAME})
|
||||
else()
|
||||
|
||||
endif(${TD_LINUX})
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
# lucene
|
||||
# To support build on ubuntu: sudo apt-get install libboost-all-dev
|
||||
if(${BUILD_WITH_LUCENE})
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ if(${BUILD_WITH_ROCKSDB})
|
|||
add_subdirectory(rocksdb)
|
||||
endif(${BUILD_WITH_ROCKSDB})
|
||||
|
||||
# cos
|
||||
if(${BUILD_WITH_COS})
|
||||
add_subdirectory(cos)
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
if(${BUILD_WITH_LUCENE})
|
||||
add_subdirectory(lucene)
|
||||
endif(${BUILD_WITH_LUCENE})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
add_executable(cosTest "")
|
||||
target_sources(cosTest
|
||||
PRIVATE
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/main.c"
|
||||
)
|
||||
|
||||
#find_path(APR_INCLUDE_DIR apr-1/apr_time.h)
|
||||
#find_path(APR_UTIL_INCLUDE_DIR apr/include/apr-1/apr_md5.h)
|
||||
#find_path(MINIXML_INCLUDE_DIR mxml.h)
|
||||
#find_path(CURL_INCLUDE_DIR curl/curl.h)
|
||||
|
||||
#include_directories (${MINIXML_INCLUDE_DIR})
|
||||
#include_directories (${CURL_INCLUDE_DIR})
|
||||
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
|
||||
#FIND_PROGRAM(APU_CONFIG_BIN NAMES apu-config apu-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
|
||||
|
||||
IF (APR_CONFIG_BIN)
|
||||
EXECUTE_PROCESS(
|
||||
COMMAND ${APR_CONFIG_BIN} --includedir
|
||||
OUTPUT_VARIABLE APR_INCLUDE_DIR
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
ENDIF()
|
||||
#IF (APU_CONFIG_BIN)
|
||||
# EXECUTE_PROCESS(
|
||||
# COMMAND ${APU_CONFIG_BIN} --includedir
|
||||
# OUTPUT_VARIABLE APR_UTIL_INCLUDE_DIR
|
||||
# OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
# )
|
||||
#ENDIF()
|
||||
|
||||
include_directories (${APR_INCLUDE_DIR})
|
||||
#include_directories (${APR_UTIL_INCLUDE_DIR})
|
||||
|
||||
target_include_directories(
|
||||
cosTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
|
||||
)
|
||||
|
||||
#find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
|
||||
#find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
|
||||
#find_library(MINIXML_LIBRARY mxml)
|
||||
#find_library(CURL_LIBRARY curl)
|
||||
|
||||
target_link_libraries(cosTest cos_c_sdk)
|
||||
target_link_libraries(cosTest apr-1})
|
||||
target_link_libraries(cosTest aprutil-1})
|
||||
target_link_libraries(cosTest mxml)
|
||||
target_link_libraries(cosTest curl)
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -24,24 +24,19 @@ By subscribing to a topic, a consumer can obtain the latest data in that topic i
|
|||
To implement these features, TDengine indexes its write-ahead log (WAL) file for fast random access and provides configurable methods for replacing and retaining this file. You can define a retention period and size for this file. For information, see the CREATE DATABASE statement. In this way, the WAL file is transformed into a persistent storage engine that remembers the order in which events occur. However, note that configuring an overly long retention period for your WAL files makes database compression inefficient. TDengine then uses the WAL file instead of the time-series database as its storage engine for queries in the form of topics. TDengine reads the data from the WAL file; uses a unified query engine instance to perform filtering, transformations, and other operations; and finally pushes the data to consumers.
|
||||
|
||||
Tips:(c interface for example)
|
||||
- A consumption group consumes all data under the same topic, and different consumption groups are independent of each other;
|
||||
- A consumption group consumes all vgroups of the same topic, which can be composed of multiple consumers, but a vgroup is only consumed by one consumer. If the number of consumers exceeds the number of vgroups, the excess consumers do not consume data;
|
||||
- On the server side, only one offset is saved for each vgroup, and the offsets for each vgroup are monotonically increasing, but not necessarily continuous. There is no correlation between the offsets of various vgroups;
|
||||
- Each poll server will return a result block, which belongs to a vgroup and may contain data from multiple versions of wal. This block can be accessed through tmq_get_vgroup_offset. The offset interface obtains the offset of the first record in the block;
|
||||
- If a consumer group has never committed an offset, when its member consumers restart and pull data again, they start consuming from the set value of the parameter auto.offset.reset; In a consumer lifecycle, the client locally records the offset of the most recent pull data and will not pull duplicate data;
|
||||
- If a consumer terminates abnormally (without calling tmq_close), they need to wait for about 12 seconds to trigger their consumer group rebalance. The consumer's status on the server will change to LOST, and after about 1 day, the consumer will be automatically deleted; Exit normally, and after exiting, the consumer will be deleted; Add a new consumer, wait for about 2 seconds to trigger Rebalance, and the consumer's status on the server will change to ready;
|
||||
- The consumer group Rebalance will reassign Vgroups to all consumer members in the ready state of the group, and consumers can only assign/see/commit/poll operations to the Vgroups they are responsible for;
|
||||
- Consumers can tmq_position to obtain the offset of the current consumption, seek to the specified offset, and consume again;
|
||||
- Seek points the position to the specified offset without executing the commit operation. Once the seek is successful, it can poll the specified offset and subsequent data;
|
||||
- Position is to obtain the current consumption position, which is the position to be taken next time, not the current consumption position
|
||||
- Commit is the submission of the consumption location. Without parameters, it is the submission of the current consumption location (the location to be taken next time, not the current consumption location). With parameters, it is the location in the submission parameters (i.e. the location to be taken after the next exit and restart)
|
||||
- Seek is to set the consumer's consumption position. Wherever the seek goes, the position will be returned, all of which are the positions to be taken next time
|
||||
- Seek does not affect commit, commit does not affect seek, independent of each other, the two are different concepts
|
||||
- The begin interface is the offset of the first data in wal, and the end interface is the offset+1 of the last data in wal10.
|
||||
- Before the seek operation, tmq must be call tmq_get_topic_assignment, The assignment interface obtains the vgroup ID and offset range of the consumer. The seek operation will detect whether the vgroup ID and offset are legal, and if they are illegal, an error will be reported;
|
||||
- Due to the existence of a WAL expiration deletion mechanism, even if the seek operation is successful, it is possible that the offset has expired when polling data. If the offset of poll is less than the WAL minimum version number, it will be consumed from the WAL minimum version number;
|
||||
- The tmq_get_vgroup_offset interface obtains the offset of the first data in the result block where the record is located. When seeking to this offset, it will consume all the data in this block. Refer to point four;
|
||||
- Data subscription is to consume data from the wal. If some wal files are deleted according to WAL retention policy, the deleted data can't be consumed any more. So you need to set a reasonable value for parameter `WAL_RETENTION_PERIOD` or `WAL_RETENTION_SIZE` when creating the database and make sure your application consume the data in a timely way to make sure there is no data loss. This behavior is similar to Kafka and other widely used message queue products.
|
||||
1. A consumption group consumes all data under the same topic, and different consumption groups are independent of each other;
|
||||
2. A consumption group consumes all vgroups of the same topic, which can be composed of multiple consumers, but a vgroup is only consumed by one consumer. If the number of consumers exceeds the number of vgroups, the excess consumers do not consume data;
|
||||
3. On the server side, only one offset is saved for each vgroup, and the offsets for each vgroup are monotonically increasing, but not necessarily continuous. There is no correlation between the offsets of various vgroups;
|
||||
4. Each poll server will return a result block, which belongs to a vgroup and may contain data from multiple versions of wal. This block can be accessed through tmq_get_vgroup_offset. The offset interface obtains the offset of the first record in the block;
|
||||
5. If a consumer group has never committed an offset, when its member consumers restart and pull data again, they start consuming from the set value of the parameter auto.offset.reset; In a consumer lifecycle, the client locally records the offset of the most recent pull data and will not pull duplicate data;
|
||||
6. If a consumer terminates abnormally (without calling tmq_close), they need to wait for about 12 seconds to trigger their consumer group rebalance. The consumer's status on the server will change to LOST, and after about 1 day, the consumer will be automatically deleted; Exit normally, and after exiting, the consumer will be deleted; Add a new consumer, wait for about 2 seconds to trigger Rebalance, and the consumer's status on the server will change to ready;
|
||||
7. The consumer group Rebalance will reassign Vgroups to all consumer members in the ready state of the group, and consumers can only assign/see/commit/poll operations to the Vgroups they are responsible for;
|
||||
8. Consumers can tmq_position to obtain the offset of the current consumption, seek to the specified offset, and consume again;
|
||||
9. Seek points the position to the specified offset without executing the commit operation. Once the seek is successful, it can poll the specified offset and subsequent data;
|
||||
10. Before the seek operation, tmq must be call tmq_get_topic_assignment, The assignment interface obtains the vgroup ID and offset range of the consumer. The seek operation will detect whether the vgroup ID and offset are legal, and if they are illegal, an error will be reported;
|
||||
11. Due to the existence of a WAL expiration deletion mechanism, even if the seek operation is successful, it is possible that the offset has expired when polling data. If the offset of poll is less than the WAL minimum version number, it will be consumed from the WAL minimum version number;
|
||||
12. The tmq_get_vgroup_offset interface obtains the offset of the first data in the result block where the record is located. When seeking to this offset, it will consume all the data in this block. Refer to point four;
|
||||
13. Data subscription is to consume data from the wal. If some wal files are deleted according to WAL retention policy, the deleted data can't be consumed any more. So you need to set a reasonable value for parameter `WAL_RETENTION_PERIOD` or `WAL_RETENTION_SIZE` when creating the database and make sure your application consume the data in a timely way to make sure there is no data loss. This behavior is similar to Kafka and other widely used message queue products.
|
||||
|
||||
## Data Schema and API
|
||||
|
||||
|
|
@ -104,8 +99,6 @@ The related schemas and APIs in various languages are described as follows:
|
|||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
```
|
||||
|
||||
For more information, see [C/C++ Connector](/reference/connector/cpp).
|
||||
|
||||
The following example is based on the smart meter table described in Data Models. For complete sample code, see the C language section below.
|
||||
|
||||
</TabItem>
|
||||
|
|
|
|||
|
|
@ -63,11 +63,12 @@ serverPort 6030
|
|||
For all the dnodes in a TDengine cluster, the below parameters must be configured exactly the same, any node whose configuration is different from dnodes already in the cluster can't join the cluster.
|
||||
|
||||
| **#** | **Parameter** | **Definition** |
|
||||
| ----- | ------------------ | ------------------------------------------- |
|
||||
| ----- | ---------------- | ----------------------------------------------------------------------------- |
|
||||
| 1 | statusInterval | The interval by which dnode reports its status to mnode |
|
||||
| 2 | timezone | Timezone |
|
||||
| 3 | locale | System region and encoding |
|
||||
| 4 | charset | Character set |
|
||||
| 5 | ttlChangeOnWrite | Whether the ttl expiration time changes with the table modification operation |
|
||||
|
||||
## Start Cluster
|
||||
|
||||
|
|
|
|||
|
|
@ -698,7 +698,7 @@ ELAPSED(ts_primary_key [, time_unit])
|
|||
LEASTSQUARES(expr, start_val, step_val)
|
||||
```
|
||||
|
||||
**Description**: The linear regression function of the specified column and the timestamp column (primary key), `start_val` is the initial value and `step_val` is the step value.
|
||||
**Description**: The linear regression function of a specified column, `start_val` is the initial value and `step_val` is the step value.
|
||||
|
||||
**Return value type**: A string in the format of "(slope, intercept)"
|
||||
|
||||
|
|
|
|||
|
|
@ -1174,7 +1174,6 @@ public abstract class ConsumerLoop {
|
|||
config.setProperty("client.id", "1");
|
||||
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
|
||||
config.setProperty("value.deserializer.encoding", "UTF-8");
|
||||
config.setProperty("experimental.snapshot.enable", "true");
|
||||
|
||||
this.consumer = new TaosConsumer<>(config);
|
||||
this.topics = Collections.singletonList("topic_speed");
|
||||
|
|
|
|||
|
|
@ -26,24 +26,19 @@ import CDemo from "./_sub_c.mdx";
|
|||
本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。
|
||||
|
||||
说明（以c接口为例）：
|
||||
- 一个消费组消费同一个topic下的所有数据，不同消费组之间相互独立；
|
||||
- 一个消费组消费同一个topic所有的vgroup，消费组可由多个消费者组成，但一个vgroup仅被一个消费者消费，如果消费者数量超过了vgroup数量，多余的消费者不消费数据；
|
||||
- 在服务端每个vgroup仅保存一个offset，每个vgroup的offset是单调递增的，但不一定连续。各个vgroup的offset之间没有关联；
|
||||
- 每次poll服务端会返回一个结果block，该block属于一个vgroup，可能包含多个wal版本的数据，可以通过 tmq_get_vgroup_offset 接口获得是该block第一条记录的offset；
|
||||
- 一个消费组如果从未commit过offset，当其成员消费者重启重新拉取数据时，均从参数auto.offset.reset设定值开始消费；在一个消费者生命周期中，客户端本地记录了最近一次拉取数据的offset，不会拉取重复数据；
|
||||
- 消费者如果异常终止（没有调用tmq_close），需等约12秒后触发其所属消费组rebalance，该消费者在服务端状态变为LOST，约1天后该消费者自动被删除；正常退出，退出后就会删除消费者；新增消费者，需等约2秒触发rebalance，该消费者在服务端状态变为ready；
|
||||
- 消费组rebalance会对该组所有ready状态的消费者成员重新进行vgroup分配，消费者仅能对自己负责的vgroup进行assignment/seek/commit/poll操作；
|
||||
- 消费者可利用 tmq_position 获得当前消费的offset，并seek到指定offset，重新消费；
|
||||
- seek将position指向指定offset，不执行commit操作，一旦seek成功，可poll拉取指定offset及以后的数据；
|
||||
- seek 操作之前须调用 tmq_get_topic_assignment 接口获取该consumer的vgroup ID和offset范围。seek 操作会检测vgroup ID 和 offset是否合法，如非法将报错；
|
||||
- position是获取当前的消费位置，是下次要取的位置，不是当前消费到的位置
|
||||
- commit是提交消费位置，不带参数的话，是提交当前消费位置（下次要取的位置，不是当前消费到的位置），带参数的话，是提交参数里的位置（也即下次退出重启后要取的位置）
|
||||
- seek是设置consumer消费位置，seek到哪，position就返回哪，都是下次要取的位置
|
||||
- seek不会影响commit，commit不影响seek，相互独立，两个是不同的概念
|
||||
- begin接口为wal 第一条数据的offset，end 接口为wal 最后一条数据的offset + 1
|
||||
- tmq_get_vgroup_offset接口获取的是记录所在结果block块里的第一条数据的offset，当seek至该offset时，将消费到这个block里的全部数据。参见第四点；
|
||||
- 由于存在 WAL 过期删除机制，即使seek 操作成功，poll数据时有可能offset已失效。如果poll 的offset 小于 WAL 最小版本号，将会从WAL最小版本号消费；
|
||||
- 数据订阅是从 WAL 消费数据，如果一些 WAL 文件被基于 WAL 保留策略删除，则已经删除的 WAL 文件中的数据就无法再消费到。需要根据业务需要在创建数据库时合理设置 `WAL_RETENTION_PERIOD` 或 `WAL_RETENTION_SIZE` ，并确保应用及时消费数据，这样才不会产生数据丢失的现象。数据订阅的行为与 Kafka 等广泛使用的消息队列类产品的行为相似；
|
||||
1. 一个消费组消费同一个topic下的所有数据，不同消费组之间相互独立；
|
||||
2. 一个消费组消费同一个topic所有的vgroup，消费组可由多个消费者组成，但一个vgroup仅被一个消费者消费，如果消费者数量超过了vgroup数量，多余的消费者不消费数据；
|
||||
3. 在服务端每个vgroup仅保存一个offset，每个vgroup的offset是单调递增的，但不一定连续。各个vgroup的offset之间没有关联；
|
||||
4. 每次poll服务端会返回一个结果block，该block属于一个vgroup，可能包含多个wal版本的数据，可以通过 tmq_get_vgroup_offset 接口获得是该block第一条记录的offset；
|
||||
5. 一个消费组如果从未commit过offset，当其成员消费者重启重新拉取数据时，均从参数auto.offset.reset设定值开始消费；在一个消费者生命周期中，客户端本地记录了最近一次拉取数据的offset，不会拉取重复数据；
|
||||
6. 消费者如果异常终止（没有调用tmq_close），需等约12秒后触发其所属消费组rebalance，该消费者在服务端状态变为LOST，约1天后该消费者自动被删除；正常退出，退出后就会删除消费者；新增消费者，需等约2秒触发rebalance，该消费者在服务端状态变为ready；
|
||||
7. 消费组rebalance会对该组所有ready状态的消费者成员重新进行vgroup分配，消费者仅能对自己负责的vgroup进行assignment/seek/commit/poll操作；
|
||||
8. 消费者可利用 tmq_position 获得当前消费的offset，并seek到指定offset，重新消费；
|
||||
9. seek将position指向指定offset，不执行commit操作，一旦seek成功，可poll拉取指定offset及以后的数据；
|
||||
10. seek 操作之前须调用 tmq_get_topic_assignment 接口获取该consumer的vgroup ID和offset范围。seek 操作会检测vgroup ID 和 offset是否合法，如非法将报错；
|
||||
11. tmq_get_vgroup_offset接口获取的是记录所在结果block块里的第一条数据的offset，当seek至该offset时，将消费到这个block里的全部数据。参见第四点；
|
||||
12. 由于存在 WAL 过期删除机制，即使seek 操作成功，poll数据时有可能offset已失效。如果poll 的offset 小于 WAL 最小版本号，将会从WAL最小版本号消费；
|
||||
13. 数据订阅是从 WAL 消费数据，如果一些 WAL 文件被基于 WAL 保留策略删除，则已经删除的 WAL 文件中的数据就无法再消费到。需要根据业务需要在创建数据库时合理设置 `WAL_RETENTION_PERIOD` 或 `WAL_RETENTION_SIZE` ，并确保应用及时消费数据，这样才不会产生数据丢失的现象。数据订阅的行为与 Kafka 等广泛使用的消息队列类产品的行为相似；
|
||||
|
||||
## 主要数据结构和 API
|
||||
|
||||
|
|
@ -69,7 +64,7 @@ import CDemo from "./_sub_c.mdx";
|
|||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end; // The last version of wal + 1
|
||||
int64_t end;
|
||||
} tmq_topic_assignment;
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
|
|
@ -89,24 +84,24 @@ import CDemo from "./_sub_c.mdx";
|
|||
DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
|
||||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
|
||||
DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); //Commit the msg’s offset + 1
|
||||
DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg);
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment);
|
||||
DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment);
|
||||
DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); // The current offset is the offset of the last consumed message + 1
|
||||
DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId);
|
||||
|
||||
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
|
||||
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
|
||||
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); // Get current offset of the result
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res);
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
```
|
||||
|
||||
这些 API 的文档请见 [C/C++ Connector](../../connector/cpp)，下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码请见下面 C 语言的示例代码。
|
||||
下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码请见下面 C 语言的示例代码。
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="java" label="Java">
|
||||
|
|
|
|||
|
|
@ -63,11 +63,12 @@ serverPort 6030
|
|||
加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。
|
||||
|
||||
| **#** | **配置参数名称** | **含义** |
|
||||
| ----- | ------------------ | ------------------------------------------- |
|
||||
| ----- | ---------------- | ------------------------------------ |
|
||||
| 1 | statusInterval | dnode 向 mnode 报告状态时长 |
|
||||
| 2 | timezone | 时区 |
|
||||
| 3 | locale | 系统区位信息及编码格式 |
|
||||
| 4 | charset | 字符集编码 |
|
||||
| 5 | ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变 |
|
||||
|
||||
## 启动集群
|
||||
|
||||
|
|
|
|||
|
|
@ -700,7 +700,7 @@ ELAPSED(ts_primary_key [, time_unit])
|
|||
LEASTSQUARES(expr, start_val, step_val)
|
||||
```
|
||||
|
||||
**功能说明**：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。
|
||||
**功能说明**：统计表中某列的值的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。
|
||||
|
||||
**返回数据类型**：字符串表达式（斜率, 截距）。
|
||||
|
||||
|
|
|
|||
|
|
@ -272,7 +272,7 @@ typedef struct tmq_topic_assignment {
|
|||
int32_t vgId;
|
||||
int64_t currentOffset;
|
||||
int64_t begin;
|
||||
int64_t end; // The last invalidate version of wal + 1
|
||||
int64_t end;
|
||||
} tmq_topic_assignment;
|
||||
|
||||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
|
|
@ -305,7 +305,7 @@ DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t v
|
|||
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
|
||||
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
|
||||
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); // Get current offset of the result
|
||||
DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res);
|
||||
DLL_EXPORT const char *tmq_err2str(int32_t code);
|
||||
|
||||
/* ------------------------------ TAOSX -----------------------------------*/
|
||||
|
|
|
|||
|
|
@ -98,6 +98,16 @@ typedef struct SMTbCursor {
|
|||
int8_t paused;
|
||||
} SMTbCursor;
|
||||
|
||||
typedef struct SMCtbCursor {
|
||||
SMeta *pMeta;
|
||||
void *pCur;
|
||||
tb_uid_t suid;
|
||||
void *pKey;
|
||||
void *pVal;
|
||||
int kLen;
|
||||
int vLen;
|
||||
} SMCtbCursor;
|
||||
|
||||
typedef struct SRowBuffPos {
|
||||
void* pRowBuff;
|
||||
void* pKey;
|
||||
|
|
@ -278,13 +288,15 @@ typedef struct SStoreMeta {
|
|||
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
|
||||
int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) &
|
||||
// metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
|
||||
|
||||
int64_t (*getNumOfRowsInMem)(void* pVnode);
|
||||
/**
|
||||
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
|
||||
int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
|
||||
int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
|
||||
*/
|
||||
SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock);
|
||||
void (*closeCtbCursor)(SMCtbCursor *pCtbCur, int lock);
|
||||
tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
|
||||
} SStoreMeta;
|
||||
|
||||
typedef struct SStoreMetaReader {
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_REVOKE_STMT,
|
||||
QUERY_NODE_SHOW_DNODES_STMT,
|
||||
QUERY_NODE_SHOW_MNODES_STMT,
|
||||
QUERY_NODE_SHOW_MODULES_STMT,
|
||||
// QUERY_NODE_SHOW_MODULES_STMT,
|
||||
QUERY_NODE_SHOW_QNODES_STMT,
|
||||
QUERY_NODE_SHOW_SNODES_STMT,
|
||||
QUERY_NODE_SHOW_BNODES_STMT,
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ typedef struct SScanLogicNode {
|
|||
bool sortPrimaryKey;
|
||||
bool igLastNull;
|
||||
bool groupOrderScan;
|
||||
bool onlyMetaCtbIdx; // for tag scan with no tbname
|
||||
} SScanLogicNode;
|
||||
|
||||
typedef struct SJoinLogicNode {
|
||||
|
|
@ -334,7 +335,11 @@ typedef struct SScanPhysiNode {
|
|||
bool groupOrderScan;
|
||||
} SScanPhysiNode;
|
||||
|
||||
typedef SScanPhysiNode STagScanPhysiNode;
|
||||
typedef struct STagScanPhysiNode {
|
||||
SScanPhysiNode scan;
|
||||
bool onlyMetaCtbIdx; //no tbname, tag index not used.
|
||||
} STagScanPhysiNode;
|
||||
|
||||
typedef SScanPhysiNode SBlockDistScanPhysiNode;
|
||||
|
||||
typedef struct SLastRowScanPhysiNode {
|
||||
|
|
|
|||
|
|
@ -300,6 +300,25 @@ void tfsClosedir(STfsDir *pDir);
|
|||
*/
|
||||
int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo);
|
||||
|
||||
/**
|
||||
* @brief Check if disk space available at level
|
||||
*
|
||||
* @param pTfs The fs object.
|
||||
* #param level the level
|
||||
* @return bool
|
||||
*/
|
||||
bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level);
|
||||
|
||||
/**
|
||||
* @brief Check if disk space sufficient at disk of level
|
||||
*
|
||||
* @param pTfs The fs object.
|
||||
* @param level the level
|
||||
* @param disk the disk
|
||||
* @return bool
|
||||
*/
|
||||
bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
|
|||
|
||||
int32_t taosUmaskFile(int32_t maskVal);
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
|
||||
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
|
||||
bool taosCheckExistFile(const char *pathname);
|
||||
|
|
|
|||
|
|
@ -95,6 +95,8 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf);
|
|||
struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);
|
||||
time_t taosTime(time_t *t);
|
||||
time_t taosMktime(struct tm *timep);
|
||||
int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour,
|
||||
const uint32_t min, const uint32_t sec, int64_t time_zone);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ target_include_directories(
|
|||
target_link_libraries(
|
||||
taos
|
||||
INTERFACE api
|
||||
PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
|
||||
PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
|
||||
)
|
||||
|
||||
if(TD_DARWIN_ARM64)
|
||||
|
|
@ -57,7 +57,7 @@ target_include_directories(
|
|||
target_link_libraries(
|
||||
taos_static
|
||||
INTERFACE api
|
||||
PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
|
||||
PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
|
||||
)
|
||||
|
||||
if(${BUILD_TEST})
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ extern "C" {
|
|||
#include "ttime.h"
|
||||
#include "ttypes.h"
|
||||
#include "cJSON.h"
|
||||
#include "geosWrapper.h"
|
||||
|
||||
#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
|
||||
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
|
||||
|
|
@ -192,7 +193,7 @@ typedef struct {
|
|||
//
|
||||
SArray *preLineTagKV;
|
||||
SArray *maxTagKVs;
|
||||
SArray *masColKVs;
|
||||
SArray *maxColKVs;
|
||||
|
||||
SSmlLineInfo preLine;
|
||||
STableMeta *currSTableMeta;
|
||||
|
|
|
|||
|
|
@ -1073,6 +1073,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
return 0;
|
||||
|
||||
end:
|
||||
taosHashCancelIterate(info->superTables, tmp);
|
||||
taosHashCleanup(hashTmp);
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
|
||||
|
|
@ -1191,6 +1192,7 @@ void freeSSmlKv(void *data) {
|
|||
SSmlKv *kv = (SSmlKv *)data;
|
||||
if (kv->keyEscaped) taosMemoryFree((void *)(kv->key));
|
||||
if (kv->valueEscaped) taosMemoryFree((void *)(kv->value));
|
||||
if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value));
|
||||
}
|
||||
|
||||
void smlDestroyInfo(SSmlHandle *info) {
|
||||
|
|
@ -1433,6 +1435,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
|
||||
if(code != TSDB_CODE_SUCCESS){
|
||||
taosMemoryFree(measure);
|
||||
taosHashCancelIterate(info->childTables, oneTable);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -1441,6 +1444,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName);
|
||||
taosMemoryFree(measure);
|
||||
taosHashCancelIterate(info->childTables, oneTable);
|
||||
return code;
|
||||
}
|
||||
taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg));
|
||||
|
|
@ -1450,6 +1454,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) {
|
||||
uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName);
|
||||
taosMemoryFree(measure);
|
||||
taosHashCancelIterate(info->childTables, oneTable);
|
||||
return TSDB_CODE_SML_INTERNAL_ERROR;
|
||||
}
|
||||
|
||||
|
|
@ -1465,6 +1470,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
taosMemoryFree(measure);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
|
||||
taosHashCancelIterate(info->childTables, oneTable);
|
||||
return code;
|
||||
}
|
||||
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
|
||||
|
|
|
|||
|
|
@ -102,6 +102,30 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
|
|||
return TSDB_CODE_TSC_INVALID_VALUE;
|
||||
}
|
||||
|
||||
if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry
|
||||
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= sizeof("POINT")+3) {
|
||||
int32_t code = initCtxGeomFromText();
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
char* tmp = taosMemoryCalloc(pVal->length, 1);
|
||||
memcpy(tmp, pVal->value + 2, pVal->length - 3);
|
||||
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
|
||||
taosMemoryFree(tmp);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
pVal->type = TSDB_DATA_TYPE_GEOMETRY;
|
||||
if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
|
||||
geosFreeBuffer((void*)(pVal->value));
|
||||
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
return TSDB_CODE_TSC_INVALID_VALUE;
|
||||
}
|
||||
|
||||
if (pVal->value[0] == 't' || pVal->value[0] == 'T') {
|
||||
if (pVal->length == 1 ||
|
||||
(pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') &&
|
||||
|
|
@ -390,14 +414,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
|
|||
SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
|
||||
if (tag->type == TSDB_DATA_TYPE_NCHAR) {
|
||||
kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
|
||||
} else if (tag->type == TSDB_DATA_TYPE_BINARY) {
|
||||
} else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
kv.length = tag->bytes - VARSTR_HEADER_SIZE;
|
||||
}
|
||||
taosArrayPush((*tmp)->cols, &kv);
|
||||
}
|
||||
}
|
||||
info->currSTableMeta = (*tmp)->tableMeta;
|
||||
info->masColKVs = (*tmp)->cols;
|
||||
info->maxColKVs = (*tmp)->cols;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -512,13 +536,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
|
|||
freeSSmlKv(&kv);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
if (cnt >= taosArrayGetSize(info->masColKVs)) {
|
||||
if (cnt >= taosArrayGetSize(info->maxColKVs)) {
|
||||
info->dataFormat = false;
|
||||
info->reRun = true;
|
||||
freeSSmlKv(&kv);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
|
||||
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
|
||||
if (kv.type != maxKV->type) {
|
||||
info->dataFormat = false;
|
||||
info->reRun = true;
|
||||
|
|
@ -663,14 +687,15 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
|
|||
if (info->dataFormat) {
|
||||
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
|
||||
ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
return ret;
|
||||
}
|
||||
if (ret == TSDB_CODE_SUCCESS) {
|
||||
ret = smlBuildRow(info->currTableDataCtx);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
}
|
||||
|
||||
clearColValArray(info->currTableDataCtx->pValues);
|
||||
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
|
||||
smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
|
||||
return ret;
|
||||
}
|
||||
clearColValArray(info->currTableDataCtx->pValues);
|
||||
} else {
|
||||
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
|
||||
taosArraySet(elements->colArray, 0, &kv);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ TARGET_LINK_LIBRARIES(
|
|||
ADD_EXECUTABLE(smlTest smlTest.cpp)
|
||||
TARGET_LINK_LIBRARIES(
|
||||
smlTest
|
||||
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
|
||||
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
|
||||
)
|
||||
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,14 @@ ENDIF ()
|
|||
IF (TD_STORAGE)
|
||||
ADD_DEFINITIONS(-D_STORAGE)
|
||||
TARGET_LINK_LIBRARIES(common PRIVATE storage)
|
||||
|
||||
IF(${TD_LINUX})
|
||||
IF(${BUILD_WITH_COS})
|
||||
add_definitions(-DUSE_COS)
|
||||
ENDIF(${BUILD_WITH_COS})
|
||||
|
||||
ENDIF(${TD_LINUX})
|
||||
|
||||
ENDIF ()
|
||||
|
||||
target_include_directories(
|
||||
|
|
|
|||
|
|
@ -314,7 +314,7 @@ static const SSysDbTableSchema userUserPrivilegesSchema[] = {
|
|||
static const SSysTableMeta infosMeta[] = {
|
||||
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
|
||||
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
|
||||
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true},
|
||||
// {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true},
|
||||
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true},
|
||||
{TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema), true},
|
||||
{TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true},
|
||||
|
|
|
|||
|
|
@ -238,6 +238,14 @@ bool tsFilterScalarMode = false;
|
|||
int32_t tsKeepTimeOffset = 0; // latency of data migration
|
||||
int tsResolveFQDNRetryTime = 100; //seconds
|
||||
|
||||
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
|
||||
char tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
|
||||
char tsS3AccessKeyId[TSDB_FQDN_LEN] = "<accesskeyid>";
|
||||
char tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
|
||||
char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
|
||||
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
|
||||
int8_t tsS3Enabled = false;
|
||||
|
||||
#ifndef _STORAGE
|
||||
int32_t taosSetTfsCfg(SConfig *pCfg) {
|
||||
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
|
||||
|
|
@ -259,7 +267,43 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
|
|||
int32_t taosSetTfsCfg(SConfig *pCfg);
|
||||
#endif
|
||||
|
||||
struct SConfig *taosGetCfg() { return tsCfg; }
|
||||
int32_t taosSetS3Cfg(SConfig *pCfg) {
|
||||
tstrncpy(tsS3AccessKey, cfgGetItem(pCfg, "s3Accesskey")->str, TSDB_FQDN_LEN);
|
||||
if (tsS3AccessKey[0] == '<') {
|
||||
return 0;
|
||||
}
|
||||
char *colon = strchr(tsS3AccessKey, ':');
|
||||
if (!colon) {
|
||||
uError("invalid access key:%s", tsS3AccessKey);
|
||||
return -1;
|
||||
}
|
||||
*colon = '\0';
|
||||
tstrncpy(tsS3AccessKeyId, tsS3AccessKey, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3AccessKeySecret, colon + 1, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3Endpoint, cfgGetItem(pCfg, "s3Endpoint")->str, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, cfgGetItem(pCfg, "s3BucketName")->str, TSDB_FQDN_LEN);
|
||||
char *cos = strstr(tsS3Endpoint, "cos.");
|
||||
if (cos) {
|
||||
char *appid = strrchr(tsS3BucketName, '-');
|
||||
if (!appid) {
|
||||
uError("failed to locate appid in bucket:%s", tsS3BucketName);
|
||||
return -1;
|
||||
} else {
|
||||
tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
|
||||
}
|
||||
}
|
||||
if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
|
||||
#ifdef USE_COS
|
||||
tsS3Enabled = true;
|
||||
#endif
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct SConfig *taosGetCfg() {
|
||||
return tsCfg;
|
||||
}
|
||||
|
||||
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
|
||||
char *apolloUrl) {
|
||||
|
|
@ -583,6 +627,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, 0) != 0) return -1;
|
||||
|
||||
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1;
|
||||
|
||||
GRANT_CFG_ADD;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1505,6 +1553,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
|
|||
if (taosSetServerCfg(tsCfg)) return -1;
|
||||
if (taosSetReleaseCfg(tsCfg)) return -1;
|
||||
if (taosSetTfsCfg(tsCfg) != 0) return -1;
|
||||
if (taosSetS3Cfg(tsCfg) != 0) return -1;
|
||||
}
|
||||
taosSetSystemCfg(tsCfg);
|
||||
|
||||
|
|
|
|||
|
|
@ -25,46 +25,6 @@
|
|||
|
||||
#include "tlog.h"
|
||||
|
||||
/*
|
||||
* mktime64 - Converts date to seconds.
|
||||
* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
|
||||
* Assumes input in normal date format, i.e. 1980-12-31 23:59:59
|
||||
* => year=1980, mon=12, day=31, hour=23, min=59, sec=59.
|
||||
*
|
||||
* [For the Julian calendar (which was used in Russia before 1917,
|
||||
* Britain & colonies before 1752, anywhere else before 1582,
|
||||
* and is still in use by some communities) leave out the
|
||||
* -year/100+year/400 terms, and add 10.]
|
||||
*
|
||||
* This algorithm was first published by Gauss (I think).
|
||||
*
|
||||
* A leap second can be indicated by calling this function with sec as
|
||||
* 60 (allowable under ISO 8601). The leap second is treated the same
|
||||
* as the following second since they don't exist in UNIX time.
|
||||
*
|
||||
* An encoding of midnight at the end of the day as 24:00:00 - ie. midnight
|
||||
* tomorrow - (allowable under ISO 8601) is supported.
|
||||
*/
|
||||
static int64_t user_mktime64(const uint32_t year0, const uint32_t mon0, const uint32_t day, const uint32_t hour,
|
||||
const uint32_t min, const uint32_t sec, int64_t time_zone) {
|
||||
uint32_t mon = mon0, year = year0;
|
||||
|
||||
/* 1..12 -> 11,12,1..10 */
|
||||
if (0 >= (int32_t)(mon -= 2)) {
|
||||
mon += 12; /* Puts Feb last since it has leap day */
|
||||
year -= 1;
|
||||
}
|
||||
|
||||
// int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
|
||||
// year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
|
||||
int64_t res;
|
||||
res = 367 * ((int64_t)mon) / 12;
|
||||
res += year / 4 - year / 100 + year / 400 + day + ((int64_t)year) * 365 - 719499;
|
||||
res = res * 24;
|
||||
res = ((res + hour) * 60 + min) * 60 + sec;
|
||||
|
||||
return (res + time_zone);
|
||||
}
|
||||
|
||||
// ==== mktime() kernel code =================//
|
||||
static int64_t m_deltaUtc = 0;
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
|
|||
char file[PATH_MAX] = {0};
|
||||
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
dInfo("mnode file:%s not exist", file);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
|
|||
SWrapperCfg *pCfgs = NULL;
|
||||
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
dInfo("vnode file:%s not exist", file);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -460,7 +460,6 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
|
|||
vmCloseVnodes(pMgmt);
|
||||
vmStopWorker(pMgmt);
|
||||
vnodeCleanup();
|
||||
tfsClose(pMgmt->pTfs);
|
||||
taosThreadRwlockDestroy(&pMgmt->lock);
|
||||
taosMemoryFree(pMgmt);
|
||||
}
|
||||
|
|
@ -535,20 +534,9 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
|
|||
pMgmt->msgCb.mgmt = pMgmt;
|
||||
taosThreadRwlockInit(&pMgmt->lock, NULL);
|
||||
|
||||
SDiskCfg dCfg = {0};
|
||||
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
|
||||
dCfg.level = 0;
|
||||
dCfg.primary = 1;
|
||||
SDiskCfg *pDisks = tsDiskCfg;
|
||||
int32_t numOfDisks = tsDiskCfgNum;
|
||||
if (numOfDisks <= 0 || pDisks == NULL) {
|
||||
pDisks = &dCfg;
|
||||
numOfDisks = 1;
|
||||
}
|
||||
|
||||
pMgmt->pTfs = tfsOpen(pDisks, numOfDisks);
|
||||
pMgmt->pTfs = pInput->pTfs;
|
||||
if (pMgmt->pTfs == NULL) {
|
||||
dError("failed to init tfs since %s", terrstr());
|
||||
dError("tfs is null.");
|
||||
goto _OVER;
|
||||
}
|
||||
tmsgReportStartup("vnode-tfs", "initialized");
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "vmInt.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
|
||||
if (pMsg->info.handle == NULL) return;
|
||||
|
|
@ -114,9 +115,10 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
|
|||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg);
|
||||
|
||||
terrno = 0;
|
||||
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
|
||||
if (code != 0) {
|
||||
if (terrno != 0) {
|
||||
if (code == -1 && terrno != 0) {
|
||||
code = terrno;
|
||||
}
|
||||
|
||||
|
|
@ -158,6 +160,15 @@ static void vmSendResponse(SRpcMsg *pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
|
||||
STfs *pTfs = pVnode->pImpl->pTfs;
|
||||
if (pTfs) {
|
||||
return tfsDiskSpaceSufficient(pTfs, 0, pVnode->diskPrimary);
|
||||
} else {
|
||||
return osDataSpaceSufficient();
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
if (pMsg->contLen < sizeof(SMsgHead)) {
|
||||
|
|
@ -203,7 +214,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
taosWriteQitem(pVnode->pFetchQ, pMsg);
|
||||
break;
|
||||
case WRITE_QUEUE:
|
||||
if (!osDataSpaceSufficient()) {
|
||||
if (!vmDataSpaceSufficient(pVnode)) {
|
||||
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
|
||||
code = terrno;
|
||||
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include "uv.h"
|
||||
|
||||
#include "dmInt.h"
|
||||
#include "tfs.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -79,6 +80,7 @@ typedef struct SDnode {
|
|||
TdThreadMutex mutex;
|
||||
TdFilePtr lockfile;
|
||||
SDnodeData data;
|
||||
STfs *pTfs;
|
||||
SMgmtWrapper wrappers[NODE_END];
|
||||
} SDnode;
|
||||
|
||||
|
|
|
|||
|
|
@ -96,28 +96,23 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
static bool dmCheckDiskSpace() {
|
||||
osUpdate();
|
||||
// sufficiency
|
||||
if (!osDataSpaceSufficient()) {
|
||||
dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least",
|
||||
(double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
|
||||
(double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
|
||||
static bool dmDataSpaceAvailable() {
|
||||
SDnode *pDnode = dmInstance();
|
||||
if (pDnode->pTfs) {
|
||||
return tfsDiskSpaceAvailable(pDnode->pTfs, 0);
|
||||
}
|
||||
if (!osLogSpaceSufficient()) {
|
||||
dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least",
|
||||
(double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
|
||||
(double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
|
||||
}
|
||||
if (!osTempSpaceSufficient()) {
|
||||
dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least",
|
||||
(double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
|
||||
(double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
|
||||
}
|
||||
// availability
|
||||
bool ret = true;
|
||||
if (!osDataSpaceAvailable()) {
|
||||
dError("data disk space unavailable, i.e. %s", tsDataDir);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool dmCheckDiskSpace() {
|
||||
osUpdate();
|
||||
// availability
|
||||
bool ret = true;
|
||||
if (!dmDataSpaceAvailable()) {
|
||||
terrno = TSDB_CODE_NO_DISKSPACE;
|
||||
ret = false;
|
||||
}
|
||||
|
|
@ -134,6 +129,34 @@ static bool dmCheckDiskSpace() {
|
|||
return ret;
|
||||
}
|
||||
|
||||
int32_t dmDiskInit() {
|
||||
SDnode *pDnode = dmInstance();
|
||||
SDiskCfg dCfg = {0};
|
||||
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
|
||||
dCfg.level = 0;
|
||||
dCfg.primary = 1;
|
||||
SDiskCfg *pDisks = tsDiskCfg;
|
||||
int32_t numOfDisks = tsDiskCfgNum;
|
||||
if (numOfDisks <= 0 || pDisks == NULL) {
|
||||
pDisks = &dCfg;
|
||||
numOfDisks = 1;
|
||||
}
|
||||
|
||||
pDnode->pTfs = tfsOpen(pDisks, numOfDisks);
|
||||
if (pDnode->pTfs == NULL) {
|
||||
dError("failed to init tfs since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t dmDiskClose() {
|
||||
SDnode *pDnode = dmInstance();
|
||||
tfsClose(pDnode->pTfs);
|
||||
pDnode->pTfs = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool dmCheckDataDirVersion() {
|
||||
char checkDataDirJsonFileName[PATH_MAX] = {0};
|
||||
snprintf(checkDataDirJsonFileName, PATH_MAX, "%s/dnode/dnodeCfg.json", tsDataDir);
|
||||
|
|
@ -147,6 +170,7 @@ static bool dmCheckDataDirVersion() {
|
|||
|
||||
int32_t dmInit() {
|
||||
dInfo("start to init dnode env");
|
||||
if (dmDiskInit() != 0) return -1;
|
||||
if (!dmCheckDataDirVersion()) return -1;
|
||||
if (!dmCheckDiskSpace()) return -1;
|
||||
if (dmCheckRepeatInit(dmInstance()) != 0) return -1;
|
||||
|
|
@ -177,6 +201,7 @@ void dmCleanup() {
|
|||
udfcClose();
|
||||
udfStopUdfd();
|
||||
taosStopCacheRefreshWorker();
|
||||
dmDiskClose();
|
||||
dInfo("dnode env is cleaned up");
|
||||
|
||||
taosCleanupCfg();
|
||||
|
|
@ -367,6 +392,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
|
|||
SMgmtInputOpt opt = {
|
||||
.path = pWrapper->path,
|
||||
.name = pWrapper->name,
|
||||
.pTfs = pWrapper->pDnode->pTfs,
|
||||
.pData = &pWrapper->pDnode->data,
|
||||
.processCreateNodeFp = dmProcessCreateNodeReq,
|
||||
.processAlterNodeTypeFp = dmProcessAlterNodeTypeReq,
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
#include "monitor.h"
|
||||
#include "qnode.h"
|
||||
#include "sync.h"
|
||||
#include "tfs.h"
|
||||
#include "wal.h"
|
||||
|
||||
#include "libs/function/tudf.h"
|
||||
|
|
@ -111,6 +112,7 @@ typedef struct {
|
|||
typedef struct {
|
||||
const char *path;
|
||||
const char *name;
|
||||
STfs *pTfs;
|
||||
SDnodeData *pData;
|
||||
SMsgCb msgCb;
|
||||
ProcessCreateNodeFp processCreateNodeFp;
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ int32_t dmReadEps(SDnodeData *pData) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
dInfo("dnode file:%s not exist", file);
|
||||
code = 0;
|
||||
goto _OVER;
|
||||
|
|
@ -469,7 +469,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
|
|||
char file[PATH_MAX] = {0};
|
||||
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
dDebug("dnode file:%s not exist", file);
|
||||
code = 0;
|
||||
goto _OVER;
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
|
|||
char file[PATH_MAX] = {0};
|
||||
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
dInfo("file:%s not exist", file);
|
||||
code = 0;
|
||||
goto _OVER;
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ static const char *offlineReason[] = {
|
|||
"timezone not match",
|
||||
"locale not match",
|
||||
"charset not match",
|
||||
"ttl change on write not match"
|
||||
"ttlChangeOnWrite not match",
|
||||
"unknown",
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -58,8 +58,10 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
|
|||
type = TSDB_MGMT_TABLE_DNODE;
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) {
|
||||
type = TSDB_MGMT_TABLE_MNODE;
|
||||
/*
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) {
|
||||
type = TSDB_MGMT_TABLE_MODULE;
|
||||
*/
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) {
|
||||
type = TSDB_MGMT_TABLE_QNODE;
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
|
||||
|
|
|
|||
|
|
@ -489,7 +489,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i);
|
||||
if(pVgEp->vgId == d1->vgId){
|
||||
jump = true;
|
||||
mInfo("pSub->offsetRows jump, because consumer id:%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId);
|
||||
mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -863,6 +863,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
} else {
|
||||
while (1) {
|
||||
SDbObj *pDb = NULL;
|
||||
|
|
@ -887,6 +888,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
} else {
|
||||
while (1) {
|
||||
SDbObj *pDb = NULL;
|
||||
|
|
@ -908,6 +910,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
goto _OVER;
|
||||
}
|
||||
taosHashRemove(newUser.readDbs, alterReq.objname, len);
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
} else {
|
||||
taosHashClear(newUser.readDbs);
|
||||
}
|
||||
|
|
@ -922,6 +925,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
goto _OVER;
|
||||
}
|
||||
taosHashRemove(newUser.writeDbs, alterReq.objname, len);
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
} else {
|
||||
taosHashClear(newUser.writeDbs);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ set(
|
|||
"src/vnd/vnodeCommit.c"
|
||||
"src/vnd/vnodeQuery.c"
|
||||
"src/vnd/vnodeModule.c"
|
||||
"src/vnd/vnodeCos.c"
|
||||
"src/vnd/vnodeSvr.c"
|
||||
"src/vnd/vnodeSync.c"
|
||||
"src/vnd/vnodeSnapshot.c"
|
||||
|
|
@ -155,6 +156,45 @@ target_link_libraries(
|
|||
PUBLIC index
|
||||
)
|
||||
|
||||
if(${TD_LINUX})
|
||||
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
|
||||
find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
|
||||
find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
|
||||
find_library(MINIXML_LIBRARY mxml)
|
||||
find_library(CURL_LIBRARY curl)
|
||||
target_link_libraries(
|
||||
vnode
|
||||
|
||||
# s3
|
||||
PUBLIC cos_c_sdk_static
|
||||
PUBLIC ${APR_UTIL_LIBRARY}
|
||||
PUBLIC ${APR_LIBRARY}
|
||||
PUBLIC ${MINIXML_LIBRARY}
|
||||
PUBLIC ${CURL_LIBRARY}
|
||||
)
|
||||
|
||||
# s3
|
||||
FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
|
||||
IF (APR_CONFIG_BIN)
|
||||
EXECUTE_PROCESS(
|
||||
COMMAND ${APR_CONFIG_BIN} --includedir
|
||||
OUTPUT_VARIABLE APR_INCLUDE_DIR
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
ENDIF()
|
||||
include_directories (${APR_INCLUDE_DIR})
|
||||
target_include_directories(
|
||||
vnode
|
||||
PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
|
||||
PUBLIC "$ENV{HOME}/.cos-local.1/include"
|
||||
)
|
||||
|
||||
if(${BUILD_WITH_COS})
|
||||
add_definitions(-DUSE_COS)
|
||||
endif(${BUILD_WITH_COS})
|
||||
|
||||
endif(${TD_LINUX})
|
||||
|
||||
IF (TD_GRANT)
|
||||
TARGET_LINK_LIBRARIES(vnode PUBLIC grant)
|
||||
ENDIF ()
|
||||
|
|
@ -169,8 +209,6 @@ if(${BUILD_WITH_ROCKSDB})
|
|||
add_definitions(-DUSE_ROCKSDB)
|
||||
endif(${BUILD_WITH_ROCKSDB})
|
||||
|
||||
|
||||
|
||||
if(${BUILD_TEST})
|
||||
add_subdirectory(test)
|
||||
endif(${BUILD_TEST})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_VND_COS_H_
|
||||
#define _TD_VND_COS_H_
|
||||
|
||||
#include "vnd.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
|
||||
int32_t s3Init();
|
||||
void s3CleanUp();
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object);
|
||||
void s3DeleteObjects(const char *object_name[], int nobject);
|
||||
bool s3Exists(const char *object_name);
|
||||
bool s3Get(const char *object_name, const char *path);
|
||||
void s3EvictCache(const char *path, long object_size);
|
||||
long s3Size(const char *object_name);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_VND_COS_H_*/
|
||||
|
|
@ -167,7 +167,7 @@ int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
|
|||
int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq);
|
||||
|
||||
int64_t metaGetTimeSeriesNum(SMeta* pMeta);
|
||||
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock);
|
||||
SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock);
|
||||
void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock);
|
||||
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
|
||||
SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid);
|
||||
|
|
|
|||
|
|
@ -408,17 +408,9 @@ _err:
|
|||
return NULL;
|
||||
}
|
||||
|
||||
struct SMCtbCursor {
|
||||
SMeta *pMeta;
|
||||
TBC *pCur;
|
||||
tb_uid_t suid;
|
||||
void *pKey;
|
||||
void *pVal;
|
||||
int kLen;
|
||||
int vLen;
|
||||
};
|
||||
|
||||
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) {
|
||||
SMCtbCursor *metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock) {
|
||||
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
|
||||
SMCtbCursor *pCtbCur = NULL;
|
||||
SCtbIdxKey ctbIdxKey;
|
||||
int ret = 0;
|
||||
|
|
@ -435,7 +427,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid, int lock) {
|
|||
metaRLock(pMeta);
|
||||
}
|
||||
|
||||
ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL);
|
||||
ret = tdbTbcOpen(pMeta->pCtbIdx, (TBC**)&pCtbCur->pCur, NULL);
|
||||
if (ret < 0) {
|
||||
metaULock(pMeta);
|
||||
taosMemoryFree(pCtbCur);
|
||||
|
|
@ -1373,7 +1365,7 @@ int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) {
|
|||
}
|
||||
|
||||
int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) {
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1);
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 1);
|
||||
|
||||
// If len > 0 means there already have uids, and we only want the
|
||||
// tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept
|
||||
|
|
@ -1434,36 +1426,37 @@ int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo, SMetaReader *pR
|
|||
int nData = 0;
|
||||
int lock = 0;
|
||||
|
||||
metaRLock(pMeta);
|
||||
if (pReader && !(pReader->flags & META_READER_NOLOCK)) {
|
||||
lock = 1;
|
||||
}
|
||||
|
||||
if(!lock) metaRLock(pMeta);
|
||||
|
||||
// search cache
|
||||
if (metaCacheGet(pMeta, uid, pInfo) == 0) {
|
||||
metaULock(pMeta);
|
||||
if(!lock) metaULock(pMeta);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
// search TDB
|
||||
if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) {
|
||||
// not found
|
||||
metaULock(pMeta);
|
||||
if(!lock) metaULock(pMeta);
|
||||
code = TSDB_CODE_NOT_FOUND;
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
metaULock(pMeta);
|
||||
if(!lock) metaULock(pMeta);
|
||||
|
||||
pInfo->uid = uid;
|
||||
pInfo->suid = ((SUidIdxVal *)pData)->suid;
|
||||
pInfo->version = ((SUidIdxVal *)pData)->version;
|
||||
pInfo->skmVer = ((SUidIdxVal *)pData)->skmVer;
|
||||
|
||||
if (pReader != NULL) {
|
||||
lock = !(pReader->flags & META_READER_NOLOCK);
|
||||
if (lock) {
|
||||
metaULock(pReader->pMeta);
|
||||
// metaReaderReleaseLock(pReader);
|
||||
}
|
||||
}
|
||||
// upsert the cache
|
||||
metaWLock(pMeta);
|
||||
metaCacheUpsert(pMeta, pInfo);
|
||||
|
|
|
|||
|
|
@ -905,6 +905,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg,
|
|||
tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
|
||||
if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) {
|
||||
smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr());
|
||||
taosHashCancelIterate(uidStore.uidHash, pIter);
|
||||
goto _err;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -644,7 +644,8 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, (uint8_t*)data, len);
|
||||
if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
|
|
@ -652,19 +653,22 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
STqOffset* pOffset = &vgOffset.offset;
|
||||
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey);
|
||||
if (pSavedOffset == NULL) {
|
||||
return TSDB_CODE_TMQ_NO_COMMITTED;
|
||||
terrno = TSDB_CODE_TMQ_NO_COMMITTED;
|
||||
return terrno;
|
||||
}
|
||||
vgOffset.offset = *pSavedOffset;
|
||||
|
||||
int32_t code = 0;
|
||||
tEncodeSize(tEncodeMqVgOffset, &vgOffset, len, code);
|
||||
if (code < 0) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
void* buf = rpcMallocCont(len);
|
||||
if (buf == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return terrno;
|
||||
}
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, buf, len);
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
|
|||
}
|
||||
|
||||
int64_t sz = 0;
|
||||
if (taosStatFile(fname, &sz, NULL) < 0) {
|
||||
if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
|
||||
taosCloseFile(&pFile);
|
||||
taosMemoryFree(fname);
|
||||
return -1;
|
||||
|
|
|
|||
|
|
@ -1092,6 +1092,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
|
|||
if(ret != TDB_CODE_SUCCESS) {
|
||||
tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId);
|
||||
taosArrayDestroy(list);
|
||||
taosHashCancelIterate(pTq->pHandle, pIter);
|
||||
return ret;
|
||||
}
|
||||
tqReaderSetTbUidList(pTqHandle->execHandle.pTqReader, list, NULL);
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
|
|||
// SDelFile
|
||||
if (pTsdb->fs.pDelFile) {
|
||||
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
|
||||
if (taosStatFile(fname, &size, NULL)) {
|
||||
if (taosStatFile(fname, &size, NULL, NULL)) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
|
@ -195,7 +195,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
|
|||
|
||||
// head =========
|
||||
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
|
||||
if (taosStatFile(fname, &size, NULL)) {
|
||||
if (taosStatFile(fname, &size, NULL, NULL)) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
|
@ -206,7 +206,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
|
|||
|
||||
// data =========
|
||||
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
|
||||
if (taosStatFile(fname, &size, NULL)) {
|
||||
if (taosStatFile(fname, &size, NULL, NULL)) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
|
@ -221,7 +221,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
|
|||
|
||||
// sma =============
|
||||
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
|
||||
if (taosStatFile(fname, &size, NULL)) {
|
||||
if (taosStatFile(fname, &size, NULL, NULL)) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
|
@ -237,7 +237,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
|
|||
// stt ===========
|
||||
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
|
||||
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
|
||||
if (taosStatFile(fname, &size, NULL)) {
|
||||
if (taosStatFile(fname, &size, NULL, NULL)) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
#include "tsdb.h"
|
||||
#include "vndCos.h"
|
||||
|
||||
// =============== PAGE-WISE FILE ===============
|
||||
int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
|
||||
|
|
@ -33,11 +34,26 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
|
|||
pFD->szPage = szPage;
|
||||
pFD->flag = flag;
|
||||
pFD->pFD = taosOpenFile(path, flag);
|
||||
if (pFD->pFD == NULL) {
|
||||
const char *object_name = taosDirEntryBaseName((char *)path);
|
||||
long s3_size = s3Size(object_name);
|
||||
if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
|
||||
s3EvictCache(path, s3_size);
|
||||
s3Get(object_name, path);
|
||||
|
||||
pFD->pFD = taosOpenFile(path, flag);
|
||||
|
||||
if (pFD->pFD == NULL) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
} else {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
}
|
||||
pFD->szPage = szPage;
|
||||
pFD->pgno = 0;
|
||||
pFD->pBuf = taosMemoryCalloc(1, szPage);
|
||||
|
|
@ -50,7 +66,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
|
|||
|
||||
// not check file size when reading data files.
|
||||
if (flag != TD_FILE_READ) {
|
||||
if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
|
||||
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(pFD->pBuf);
|
||||
taosCloseFile(&pFD->pFD);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include "tsdb.h"
|
||||
#include "tsdbFS2.h"
|
||||
#include "vndCos.h"
|
||||
|
||||
typedef struct {
|
||||
STsdb *tsdb;
|
||||
|
|
@ -41,6 +42,28 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
|
|||
return TARRAY2_APPEND(rtner->fopArr, op);
|
||||
}
|
||||
|
||||
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
STFileOp op = {
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
s3DeleteObjects(&object_name, 1);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
|
@ -76,6 +99,34 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
TdFilePtr fdFrom = NULL;
|
||||
TdFilePtr fdTo = NULL;
|
||||
|
||||
tsdbTFileName(rtner->tsdb, to, fname);
|
||||
|
||||
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
code = s3PutObjectFromFile(from->fname, object_name);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
taosCloseFile(&fdFrom);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
taosCloseFile(&fdFrom);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
|
@ -123,6 +174,53 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
STFileOp op = {0};
|
||||
|
||||
// remove old
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// create new
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_CREATE,
|
||||
.fid = fobj->f->fid,
|
||||
.nf =
|
||||
{
|
||||
.type = fobj->f->type,
|
||||
.did = did[0],
|
||||
.fid = fobj->f->fid,
|
||||
.cid = fobj->f->cid,
|
||||
.size = fobj->f->size,
|
||||
.stt[0] =
|
||||
{
|
||||
.level = fobj->f->stt[0].level,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// do copy the file
|
||||
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
STsdb *tsdb;
|
||||
int32_t sync;
|
||||
|
|
@ -201,9 +299,15 @@ static int32_t tsdbDoRetention2(void *arg) {
|
|||
for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) {
|
||||
if (fobj == NULL) continue;
|
||||
|
||||
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
|
||||
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
|
||||
code = tsdbRemoveFileObjectS3(rtner, fobj);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
} else {
|
||||
code = tsdbDoRemoveFileObject(rtner, fobj);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
}
|
||||
|
||||
SSttLvl *lvl;
|
||||
TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) {
|
||||
|
|
@ -228,9 +332,16 @@ static int32_t tsdbDoRetention2(void *arg) {
|
|||
if (fobj == NULL) continue;
|
||||
|
||||
if (fobj->f->did.level == did.level) continue;
|
||||
|
||||
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
|
||||
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
|
||||
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
} else {
|
||||
code = tsdbDoMigrateFileObj(rtner, fobj, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
}
|
||||
|
||||
// stt
|
||||
SSttLvl *lvl;
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
|
|||
int32_t code = 0;
|
||||
STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
|
||||
TSKEY now = taosGetTimestamp(pCfg->precision);
|
||||
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
|
||||
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
|
||||
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
|
||||
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,323 @@
|
|||
#define ALLOW_FORBID_FUNC
|
||||
|
||||
#include "vndCos.h"
|
||||
|
||||
extern char tsS3Endpoint[];
|
||||
extern char tsS3AccessKeyId[];
|
||||
extern char tsS3AccessKeySecret[];
|
||||
extern char tsS3BucketName[];
|
||||
extern char tsS3AppId[];
|
||||
|
||||
#ifdef USE_COS
|
||||
#include "cos_api.h"
|
||||
#include "cos_http_io.h"
|
||||
#include "cos_log.h"
|
||||
|
||||
int32_t s3Init() {
|
||||
if (cos_http_io_initialize(NULL, 0) != COSE_OK) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// set log level, default COS_LOG_WARN
|
||||
cos_log_set_level(COS_LOG_WARN);
|
||||
|
||||
// set log output, default stderr
|
||||
cos_log_set_output(NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void s3CleanUp() { cos_http_io_deinitialize(); }
|
||||
|
||||
static void log_status(cos_status_t *s) {
|
||||
cos_warn_log("status->code: %d", s->code);
|
||||
if (s->error_code) cos_warn_log("status->error_code: %s", s->error_code);
|
||||
if (s->error_msg) cos_warn_log("status->error_msg: %s", s->error_msg);
|
||||
if (s->req_id) cos_warn_log("status->req_id: %s", s->req_id);
|
||||
}
|
||||
|
||||
static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
|
||||
options->config = cos_config_create(options->pool);
|
||||
|
||||
cos_config_t *config = options->config;
|
||||
|
||||
cos_str_set(&config->endpoint, tsS3Endpoint);
|
||||
cos_str_set(&config->access_key_id, tsS3AccessKeyId);
|
||||
cos_str_set(&config->access_key_secret, tsS3AccessKeySecret);
|
||||
cos_str_set(&config->appid, tsS3AppId);
|
||||
|
||||
config->is_cname = is_cname;
|
||||
|
||||
options->ctl = cos_http_controller_create(options->pool, 0);
|
||||
}
|
||||
|
||||
int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
|
||||
int32_t code = 0;
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_status_t *s = NULL;
|
||||
cos_request_options_t *options = NULL;
|
||||
cos_string_t bucket, object, file;
|
||||
cos_table_t *resp_headers;
|
||||
int traffic_limit = 0;
|
||||
|
||||
cos_pool_create(&p, NULL);
|
||||
options = cos_request_options_create(p);
|
||||
s3InitRequestOptions(options, is_cname);
|
||||
cos_table_t *headers = NULL;
|
||||
if (traffic_limit) {
|
||||
// 限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
|
||||
headers = cos_table_make(p, 1);
|
||||
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
|
||||
}
|
||||
cos_str_set(&bucket, tsS3BucketName);
|
||||
cos_str_set(&file, file_str);
|
||||
cos_str_set(&object, object_str);
|
||||
s = cos_put_object_from_file(options, &bucket, &object, &file, headers, &resp_headers);
|
||||
log_status(s);
|
||||
|
||||
cos_pool_destroy(p);
|
||||
|
||||
if (s->code != 200) {
|
||||
return code = s->code;
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
void s3DeleteObjects(const char *object_name[], int nobject) {
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_string_t bucket;
|
||||
cos_table_t *resp_headers = NULL;
|
||||
cos_request_options_t *options = NULL;
|
||||
cos_list_t object_list;
|
||||
cos_list_t deleted_object_list;
|
||||
int is_quiet = COS_TRUE;
|
||||
|
||||
cos_pool_create(&p, NULL);
|
||||
options = cos_request_options_create(p);
|
||||
s3InitRequestOptions(options, is_cname);
|
||||
cos_str_set(&bucket, tsS3BucketName);
|
||||
|
||||
cos_list_init(&object_list);
|
||||
cos_list_init(&deleted_object_list);
|
||||
|
||||
for (int i = 0; i < nobject; ++i) {
|
||||
cos_object_key_t *content = cos_create_cos_object_key(p);
|
||||
cos_str_set(&content->key, object_name[i]);
|
||||
cos_list_add_tail(&content->node, &object_list);
|
||||
}
|
||||
|
||||
cos_status_t *s = cos_delete_objects(options, &bucket, &object_list, is_quiet, &resp_headers, &deleted_object_list);
|
||||
log_status(s);
|
||||
|
||||
cos_pool_destroy(p);
|
||||
|
||||
if (cos_status_is_ok(s)) {
|
||||
cos_warn_log("delete objects succeeded\n");
|
||||
} else {
|
||||
cos_warn_log("delete objects failed\n");
|
||||
}
|
||||
}
|
||||
|
||||
bool s3Exists(const char *object_name) {
|
||||
bool ret = false;
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_status_t *s = NULL;
|
||||
cos_request_options_t *options = NULL;
|
||||
cos_string_t bucket;
|
||||
cos_string_t object;
|
||||
cos_table_t *resp_headers;
|
||||
cos_table_t *headers = NULL;
|
||||
cos_object_exist_status_e object_exist;
|
||||
|
||||
cos_pool_create(&p, NULL);
|
||||
options = cos_request_options_create(p);
|
||||
s3InitRequestOptions(options, is_cname);
|
||||
cos_str_set(&bucket, tsS3BucketName);
|
||||
cos_str_set(&object, object_name);
|
||||
|
||||
s = cos_check_object_exist(options, &bucket, &object, headers, &object_exist, &resp_headers);
|
||||
if (object_exist == COS_OBJECT_NON_EXIST) {
|
||||
cos_warn_log("object: %.*s non exist.\n", object.len, object.data);
|
||||
} else if (object_exist == COS_OBJECT_EXIST) {
|
||||
ret = true;
|
||||
cos_warn_log("object: %.*s exist.\n", object.len, object.data);
|
||||
} else {
|
||||
cos_warn_log("object: %.*s unknown status.\n", object.len, object.data);
|
||||
log_status(s);
|
||||
}
|
||||
|
||||
cos_pool_destroy(p);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool s3Get(const char *object_name, const char *path) {
|
||||
bool ret = false;
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_status_t *s = NULL;
|
||||
cos_request_options_t *options = NULL;
|
||||
cos_string_t bucket;
|
||||
cos_string_t object;
|
||||
cos_string_t file;
|
||||
cos_table_t *resp_headers = NULL;
|
||||
cos_table_t *headers = NULL;
|
||||
int traffic_limit = 0;
|
||||
|
||||
//创建内存池
|
||||
cos_pool_create(&p, NULL);
|
||||
|
||||
//初始化请求选项
|
||||
options = cos_request_options_create(p);
|
||||
s3InitRequestOptions(options, is_cname);
|
||||
cos_str_set(&bucket, tsS3BucketName);
|
||||
if (traffic_limit) {
|
||||
//限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误
|
||||
headers = cos_table_make(p, 1);
|
||||
cos_table_add_int(headers, "x-cos-traffic-limit", 819200);
|
||||
}
|
||||
|
||||
//下载对象
|
||||
cos_str_set(&file, path);
|
||||
cos_str_set(&object, object_name);
|
||||
s = cos_get_object_to_file(options, &bucket, &object, headers, NULL, &file, &resp_headers);
|
||||
if (cos_status_is_ok(s)) {
|
||||
ret = true;
|
||||
cos_warn_log("get object succeeded\n");
|
||||
} else {
|
||||
cos_warn_log("get object failed\n");
|
||||
}
|
||||
|
||||
//销毁内存池
|
||||
cos_pool_destroy(p);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int64_t size;
|
||||
int32_t atime;
|
||||
char name[TSDB_FILENAME_LEN];
|
||||
} SEvictFile;
|
||||
|
||||
static int32_t evictFileCompareAsce(const void *pLeft, const void *pRight) {
|
||||
SEvictFile *lhs = (SEvictFile *)pLeft;
|
||||
SEvictFile *rhs = (SEvictFile *)pRight;
|
||||
return lhs->atime < rhs->atime ? -1 : 1;
|
||||
}
|
||||
|
||||
void s3EvictCache(const char *path, long object_size) {
|
||||
SDiskSize disk_size = {0};
|
||||
char dir_name[TSDB_FILENAME_LEN] = "\0";
|
||||
|
||||
tstrncpy(dir_name, path, TSDB_FILENAME_LEN);
|
||||
taosDirName(dir_name);
|
||||
|
||||
if (taosGetDiskSize((char *)dir_name, &disk_size) < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
vError("failed to get disk:%s size since %s", path, terrstr());
|
||||
return;
|
||||
}
|
||||
|
||||
if (object_size >= disk_size.avail - (1 << 30)) {
|
||||
// evict too old files
|
||||
// 1, list data files' atime under dir(path)
|
||||
tdbDirPtr pDir = taosOpenDir(dir_name);
|
||||
if (pDir == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
vError("failed to open %s since %s", dir_name, terrstr());
|
||||
}
|
||||
SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
|
||||
tdbDirEntryPtr pDirEntry;
|
||||
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
|
||||
char *name = taosGetDirEntryName(pDirEntry);
|
||||
if (!strncmp(name + strlen(name) - 5, ".data", 5)) {
|
||||
SEvictFile e_file = {0};
|
||||
char entry_name[TSDB_FILENAME_LEN] = "\0";
|
||||
int dir_len = strlen(dir_name);
|
||||
|
||||
memcpy(e_file.name, dir_name, dir_len);
|
||||
e_file.name[dir_len] = '/';
|
||||
memcpy(e_file.name + dir_len + 1, name, strlen(name));
|
||||
|
||||
taosStatFile(e_file.name, &e_file.size, NULL, &e_file.atime);
|
||||
|
||||
taosArrayPush(evict_files, &e_file);
|
||||
}
|
||||
}
|
||||
taosCloseDir(&pDir);
|
||||
|
||||
// 2, sort by atime
|
||||
taosArraySort(evict_files, evictFileCompareAsce);
|
||||
|
||||
// 3, remove files ascendingly until we get enough object_size space
|
||||
long evict_size = 0;
|
||||
size_t ef_size = TARRAY_SIZE(evict_files);
|
||||
for (size_t i = 0; i < ef_size; ++i) {
|
||||
SEvictFile *evict_file = taosArrayGet(evict_files, i);
|
||||
taosRemoveFile(evict_file->name);
|
||||
evict_size += evict_file->size;
|
||||
if (evict_size >= object_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
taosArrayDestroy(evict_files);
|
||||
}
|
||||
}
|
||||
|
||||
long s3Size(const char *object_name) {
|
||||
long size = 0;
|
||||
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_status_t *s = NULL;
|
||||
cos_request_options_t *options = NULL;
|
||||
cos_string_t bucket;
|
||||
cos_string_t object;
|
||||
cos_table_t *resp_headers = NULL;
|
||||
|
||||
//创建内存池
|
||||
cos_pool_create(&p, NULL);
|
||||
|
||||
//初始化请求选项
|
||||
options = cos_request_options_create(p);
|
||||
s3InitRequestOptions(options, is_cname);
|
||||
cos_str_set(&bucket, tsS3BucketName);
|
||||
|
||||
//获取对象元数据
|
||||
cos_str_set(&object, object_name);
|
||||
s = cos_head_object(options, &bucket, &object, NULL, &resp_headers);
|
||||
// print_headers(resp_headers);
|
||||
if (cos_status_is_ok(s)) {
|
||||
char *content_length_str = (char *)apr_table_get(resp_headers, COS_CONTENT_LENGTH);
|
||||
if (content_length_str != NULL) {
|
||||
size = atol(content_length_str);
|
||||
}
|
||||
cos_warn_log("head object succeeded: %ld\n", size);
|
||||
} else {
|
||||
cos_warn_log("head object failed\n");
|
||||
}
|
||||
|
||||
//销毁内存池
|
||||
cos_pool_destroy(p);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int32_t s3Init() { return 0; }
|
||||
void s3CleanUp() {}
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
|
||||
void s3DeleteObjects(const char *object_name[], int nobject) {}
|
||||
bool s3Exists(const char *object_name) { return false; }
|
||||
bool s3Get(const char *object_name, const char *path) { return false; }
|
||||
void s3EvictCache(const char *path, long object_size) {}
|
||||
long s3Size(const char *object_name) { return 0; }
|
||||
|
||||
#endif
|
||||
|
|
@ -96,6 +96,10 @@ void initMetadataAPI(SStoreMeta* pMeta) {
|
|||
|
||||
pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup;
|
||||
pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache;
|
||||
|
||||
pMeta->openCtbCursor = metaOpenCtbCursor;
|
||||
pMeta->closeCtbCursor = metaCloseCtbCursor;
|
||||
pMeta->ctbCursorNext = metaCtbCursorNext;
|
||||
}
|
||||
|
||||
void initTqAPI(SStoreTqReader* pTq) {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
#include "vnd.h"
|
||||
#include "vndCos.h"
|
||||
|
||||
typedef struct SVnodeTask SVnodeTask;
|
||||
struct SVnodeTask {
|
||||
|
|
@ -81,6 +82,9 @@ int vnodeInit(int nthreads) {
|
|||
if (tqInit() < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (s3Init() < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -112,6 +116,7 @@ void vnodeCleanup() {
|
|||
walCleanUp();
|
||||
tqCleanUp();
|
||||
smaCleanUp();
|
||||
s3CleanUp();
|
||||
}
|
||||
|
||||
int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) {
|
||||
|
|
|
|||
|
|
@ -440,7 +440,7 @@ int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
|
|||
}
|
||||
|
||||
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid, 1);
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
|
||||
|
||||
while (1) {
|
||||
tb_uid_t id = metaCtbCursorNext(pCur);
|
||||
|
|
@ -462,7 +462,7 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
|
|||
|
||||
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
|
||||
SVnode *pVnodeObj = pVnode;
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1);
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
|
||||
|
||||
while (1) {
|
||||
tb_uid_t id = metaCtbCursorNext(pCur);
|
||||
|
|
@ -521,7 +521,7 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
|
|||
}
|
||||
|
||||
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0);
|
||||
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, suid, 0);
|
||||
if (!pCur) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -291,17 +291,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
switch (pNode->type) {
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
|
||||
STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
|
||||
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname);
|
||||
EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->scan.tableName.tname);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
|
||||
if (pResNode->pExecInfo) {
|
||||
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
|
||||
}
|
||||
if (pTagScanNode->pScanPseudoCols) {
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->pScanPseudoCols->length);
|
||||
if (pTagScanNode->scan.pScanPseudoCols) {
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->scan.pScanPseudoCols->length);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
|
||||
}
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
|
||||
EXPLAIN_ROW_END();
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
|
||||
|
|
@ -309,11 +309,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
|
|||
if (verbose) {
|
||||
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
|
||||
nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots));
|
||||
nodesGetOutputNumFromSlotList(pTagScanNode->scan.node.pOutputDataBlockDesc->pSlots));
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize);
|
||||
EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->node.pLimit);
|
||||
EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->node.pSlimit);
|
||||
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
|
||||
EXPLAIN_ROW_APPEND_LIMIT(pTagScanNode->scan.node.pLimit);
|
||||
EXPLAIN_ROW_APPEND_SLIMIT(pTagScanNode->scan.node.pSlimit);
|
||||
EXPLAIN_ROW_END();
|
||||
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
|
||||
|
||||
|
|
|
|||
|
|
@ -190,4 +190,6 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag);
|
|||
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
|
||||
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
|
||||
|
||||
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
|
||||
SStorageAPI* pStorageAPI);
|
||||
#endif // TDENGINE_EXECUTIL_H
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ extern "C" {
|
|||
#include "tsort.h"
|
||||
#include "ttszip.h"
|
||||
#include "tvariant.h"
|
||||
#include "theap.h"
|
||||
|
||||
#include "dataSinkMgt.h"
|
||||
#include "executil.h"
|
||||
|
|
@ -250,6 +251,12 @@ typedef struct STableMergeScanInfo {
|
|||
SSortExecInfo sortExecInfo;
|
||||
} STableMergeScanInfo;
|
||||
|
||||
typedef struct STagScanFilterContext {
|
||||
SHashObj* colHash;
|
||||
int32_t index;
|
||||
SArray* cInfoList;
|
||||
} STagScanFilterContext;
|
||||
|
||||
typedef struct STagScanInfo {
|
||||
SColumnInfo* pCols;
|
||||
SSDataBlock* pRes;
|
||||
|
|
@ -258,6 +265,14 @@ typedef struct STagScanInfo {
|
|||
SLimitNode* pSlimit;
|
||||
SReadHandle readHandle;
|
||||
STableListInfo* pTableListInfo;
|
||||
uint64_t suid;
|
||||
void* pCtbCursor;
|
||||
SNode* pTagCond;
|
||||
SNode* pTagIndexCond;
|
||||
STagScanFilterContext filterCtx;
|
||||
SArray* aUidTags; // SArray<STUidTagInfo>
|
||||
SArray* aFilterIdxs; // SArray<int32_t>
|
||||
SStorageAPI* pStorageAPI;
|
||||
} STagScanInfo;
|
||||
|
||||
typedef enum EStreamScanMode {
|
||||
|
|
@ -417,6 +432,14 @@ typedef struct SIntervalAggOperatorInfo {
|
|||
EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model]
|
||||
STimeWindowAggSupp twAggSup;
|
||||
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
|
||||
// for limit optimization
|
||||
bool limited;
|
||||
int64_t limit;
|
||||
bool slimited;
|
||||
int64_t slimit;
|
||||
uint64_t curGroupId; // initialize to UINT64_MAX
|
||||
uint64_t handledGroupNum;
|
||||
BoundedQueue* pBQ;
|
||||
} SIntervalAggOperatorInfo;
|
||||
|
||||
typedef struct SMergeAlignedIntervalAggOperatorInfo {
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
|
|||
|
||||
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ struct SExecTaskInfo {
|
|||
void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);
|
||||
SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI);
|
||||
void doDestroyTask(SExecTaskInfo* pTaskInfo);
|
||||
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
|
||||
bool isTaskKilled(void* pTaskInfo);
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
|
||||
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
|
||||
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
|
||||
|
|
|
|||
|
|
@ -191,6 +191,9 @@ int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols);
|
|||
bool tsortIsClosed(SSortHandle* pHandle);
|
||||
void tsortSetClosed(SSortHandle* pHandle);
|
||||
|
||||
void tsortSetSingleTableMerge(SSortHandle* pHandle);
|
||||
void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -47,8 +47,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* p
|
|||
|
||||
static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
|
||||
STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
|
||||
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
|
||||
SStorageAPI* pStorageAPI);
|
||||
|
||||
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
|
||||
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
|
||||
|
|
@ -846,7 +844,7 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S
|
|||
return -1;
|
||||
}
|
||||
|
||||
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
|
||||
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
|
||||
SStorageAPI* pStorageAPI) {
|
||||
SSDataBlock* pResBlock = createDataBlock();
|
||||
if (pResBlock == NULL) {
|
||||
|
|
@ -2118,8 +2116,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList);
|
||||
|
||||
if (groupSort) {
|
||||
if (groupSort || pScanNode->groupOrderScan) {
|
||||
code = sortTableGroup(pTableListInfo);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -275,7 +275,6 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
|
|||
SNode* pTagIndexCond, const char* pUser, const char* dbname) {
|
||||
int32_t type = nodeType(pPhyNode);
|
||||
const char* idstr = GET_TASKID(pTaskInfo);
|
||||
|
||||
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
|
||||
SOperatorInfo* pOperator = NULL;
|
||||
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
|
||||
|
|
@ -371,17 +370,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
|
|||
STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode;
|
||||
pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo);
|
||||
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
|
||||
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
|
||||
STagScanPhysiNode* pTagScanPhyNode = (STagScanPhysiNode*)pPhyNode;
|
||||
STableListInfo* pTableListInfo = tableListCreate();
|
||||
int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
|
||||
if (!pTagScanPhyNode->onlyMetaCtbIdx) {
|
||||
int32_t code = createScanTableListInfo((SScanPhysiNode*)pTagScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond,
|
||||
pTagIndexCond, pTaskInfo);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
pTaskInfo->code = code;
|
||||
qError("failed to getTableList, code: %s", tstrerror(code));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
|
||||
}
|
||||
pOperator = createTagScanOperatorInfo(pHandle, pTagScanPhyNode, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo);
|
||||
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
|
||||
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
|
||||
STableListInfo* pTableListInfo = tableListCreate();
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
|
|||
return pTaskInfo;
|
||||
}
|
||||
|
||||
bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
|
||||
bool isTaskKilled(void* pTaskInfo) { return (0 != ((SExecTaskInfo*)pTaskInfo)->code); }
|
||||
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) {
|
||||
pTaskInfo->code = rspCode;
|
||||
|
|
|
|||
|
|
@ -504,7 +504,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
|
|||
|
||||
// 1. check if it is existed in meta cache
|
||||
if (pCache == NULL) {
|
||||
pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, META_READER_NOLOCK, &pHandle->api.metaFn);
|
||||
pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn);
|
||||
code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
// when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed.
|
||||
|
|
@ -848,6 +848,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
|
|||
return result;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
|
||||
setOperatorCompleted(pOperator);
|
||||
return NULL;
|
||||
|
|
@ -869,9 +870,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
|
|||
if (result != NULL) {
|
||||
return result;
|
||||
}
|
||||
|
||||
setOperatorCompleted(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2689,7 +2688,236 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
|
|||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
|
||||
static void tagScanFreeUidTag(void* p) {
|
||||
STUidTagInfo* pInfo = p;
|
||||
if (pInfo->pTagVal != NULL) {
|
||||
taosMemoryFree(pInfo->pTagVal);
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) {
|
||||
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
|
||||
if (pColumnData == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
pColumnData->info.type = pType->type;
|
||||
pColumnData->info.bytes = pType->bytes;
|
||||
pColumnData->info.scale = pType->scale;
|
||||
pColumnData->info.precision = pType->precision;
|
||||
|
||||
int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
taosMemoryFree(pColumnData);
|
||||
return terrno;
|
||||
}
|
||||
|
||||
pParam->columnData = pColumnData;
|
||||
pParam->colAlloced = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) {
|
||||
SColumnNode* pSColumnNode = NULL;
|
||||
if (QUERY_NODE_COLUMN == nodeType((*pNode))) {
|
||||
pSColumnNode = *(SColumnNode**)pNode;
|
||||
} else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) {
|
||||
SFunctionNode* pFuncNode = *(SFunctionNode**)(pNode);
|
||||
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
|
||||
pSColumnNode = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
|
||||
if (NULL == pSColumnNode) {
|
||||
return DEAL_RES_ERROR;
|
||||
}
|
||||
pSColumnNode->colId = -1;
|
||||
pSColumnNode->colType = COLUMN_TYPE_TBNAME;
|
||||
pSColumnNode->node.resType.type = TSDB_DATA_TYPE_VARCHAR;
|
||||
pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
|
||||
nodesDestroyNode(*pNode);
|
||||
*pNode = (SNode*)pSColumnNode;
|
||||
} else {
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
} else {
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
STagScanFilterContext* pCtx = (STagScanFilterContext*)pContext;
|
||||
void* data = taosHashGet(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId));
|
||||
if (!data) {
|
||||
taosHashPut(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
|
||||
pSColumnNode->slotId = pCtx->index++;
|
||||
SColumnInfo cInfo = {.colId = pSColumnNode->colId,
|
||||
.type = pSColumnNode->node.resType.type,
|
||||
.bytes = pSColumnNode->node.resType.bytes};
|
||||
taosArrayPush(pCtx->cInfoList, &cInfo);
|
||||
} else {
|
||||
SColumnNode* col = *(SColumnNode**)data;
|
||||
pSColumnNode->slotId = col->slotId;
|
||||
}
|
||||
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
|
||||
static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode, SStorageAPI* pAPI, STagScanInfo* pInfo) {
|
||||
int32_t code = 0;
|
||||
int32_t numOfTables = taosArrayGetSize(aUidTags);
|
||||
|
||||
SSDataBlock* pResBlock = createTagValBlockForFilter(pInfo->filterCtx.cInfoList, numOfTables, aUidTags, pVnode, pAPI);
|
||||
|
||||
SArray* pBlockList = taosArrayInit(1, POINTER_BYTES);
|
||||
taosArrayPush(pBlockList, &pResBlock);
|
||||
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
|
||||
|
||||
SScalarParam output = {0};
|
||||
tagScanCreateResultData(&type, numOfTables, &output);
|
||||
|
||||
scalarCalculate(pTagCond, pBlockList, &output);
|
||||
|
||||
bool* result = (bool*)output.columnData->pData;
|
||||
for (int32_t i = 0 ; i < numOfTables; ++i) {
|
||||
if (result[i]) {
|
||||
taosArrayPush(aFilterIdxs, &i);
|
||||
}
|
||||
}
|
||||
|
||||
colDataDestroy(output.columnData);
|
||||
taosMemoryFreeClear(output.columnData);
|
||||
|
||||
blockDataDestroy(pResBlock);
|
||||
taosArrayDestroy(pBlockList);
|
||||
|
||||
|
||||
}
|
||||
|
||||
static void tagScanFillOneCellWithTag(const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) {
|
||||
if (fmIsScanPseudoColumnFunc(pExprInfo->pExpr->_function.functionId)) { // tbname
|
||||
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
// if (pUidTagInfo->name != NULL) {
|
||||
// STR_TO_VARSTR(str, pUidTagInfo->name);
|
||||
// } else { // name is not retrieved during filter
|
||||
// pAPI->metaFn.getTableNameByUid(pVnode, pUidTagInfo->uid, str);
|
||||
// }
|
||||
STR_TO_VARSTR(str, "ctbidx");
|
||||
|
||||
colDataSetVal(pColInfo, rowIndex, str, false);
|
||||
} else {
|
||||
STagVal tagVal = {0};
|
||||
tagVal.cid = pExprInfo->base.pParam[0].pCol->colId;
|
||||
if (pUidTagInfo->pTagVal == NULL) {
|
||||
colDataSetNULL(pColInfo, rowIndex);
|
||||
} else {
|
||||
const char* p = pAPI->metaFn.extractTagVal(pUidTagInfo->pTagVal, pColInfo->info.type, &tagVal);
|
||||
|
||||
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
|
||||
colDataSetNULL(pColInfo, rowIndex);
|
||||
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
|
||||
colDataSetVal(pColInfo, rowIndex, p, false);
|
||||
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
|
||||
char* tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1);
|
||||
varDataSetLen(tmp, tagVal.nData);
|
||||
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
|
||||
colDataSetVal(pColInfo, rowIndex, tmp, false);
|
||||
taosMemoryFree(tmp);
|
||||
} else {
|
||||
colDataSetVal(pColInfo, rowIndex, (const char*)&tagVal.i64, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRes, SArray* aUidTags, SArray* aFilterIdxs, bool ignoreFilterIdx,
|
||||
SStorageAPI* pAPI) {
|
||||
STagScanInfo* pInfo = pOperator->info;
|
||||
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
|
||||
if (!ignoreFilterIdx) {
|
||||
size_t szTables = taosArrayGetSize(aFilterIdxs);
|
||||
for (int i = 0; i < szTables; ++i) {
|
||||
int32_t idx = *(int32_t*)taosArrayGet(aFilterIdxs, i);
|
||||
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx);
|
||||
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
|
||||
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
|
||||
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
size_t szTables = taosArrayGetSize(aUidTags);
|
||||
for (int i = 0; i < szTables; ++i) {
|
||||
STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i);
|
||||
for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
|
||||
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
|
||||
tagScanFillOneCellWithTag(pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static SSDataBlock* doTagScanFromCtbIdx(SOperatorInfo* pOperator) {
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
|
||||
|
||||
STagScanInfo* pInfo = pOperator->info;
|
||||
SSDataBlock* pRes = pInfo->pRes;
|
||||
blockDataCleanup(pRes);
|
||||
|
||||
if (pInfo->pCtbCursor == NULL) {
|
||||
pInfo->pCtbCursor = pAPI->metaFn.openCtbCursor(pInfo->readHandle.vnode, pInfo->suid, 1);
|
||||
}
|
||||
|
||||
SArray* aUidTags = pInfo->aUidTags;
|
||||
SArray* aFilterIdxs = pInfo->aFilterIdxs;
|
||||
int32_t count = 0;
|
||||
|
||||
while (1) {
|
||||
taosArrayClearEx(aUidTags, tagScanFreeUidTag);
|
||||
taosArrayClear(aFilterIdxs);
|
||||
|
||||
int32_t numTables = 0;
|
||||
while (numTables < pOperator->resultInfo.capacity) {
|
||||
SMCtbCursor* pCur = pInfo->pCtbCursor;
|
||||
tb_uid_t uid = pAPI->metaFn.ctbCursorNext(pInfo->pCtbCursor);
|
||||
if (uid == 0) {
|
||||
break;
|
||||
}
|
||||
STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
|
||||
info.pTagVal = taosMemoryMalloc(pCur->vLen);
|
||||
memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
|
||||
taosArrayPush(aUidTags, &info);
|
||||
++numTables;
|
||||
}
|
||||
|
||||
if (numTables == 0) {
|
||||
break;
|
||||
}
|
||||
bool ignoreFilterIdx = true;
|
||||
if (pInfo->pTagCond != NULL) {
|
||||
ignoreFilterIdx = false;
|
||||
tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo);
|
||||
} else {
|
||||
ignoreFilterIdx = true;
|
||||
}
|
||||
|
||||
tagScanFillResultBlock(pOperator, pRes, aUidTags, aFilterIdxs, ignoreFilterIdx, pAPI);
|
||||
|
||||
count = ignoreFilterIdx ? taosArrayGetSize(aUidTags): taosArrayGetSize(aFilterIdxs);
|
||||
|
||||
if (count != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
pRes->info.rows = count;
|
||||
pOperator->resultInfo.totalRows += count;
|
||||
return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
|
||||
}
|
||||
|
||||
static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) {
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -2747,14 +2975,23 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
|
|||
|
||||
static void destroyTagScanOperatorInfo(void* param) {
|
||||
STagScanInfo* pInfo = (STagScanInfo*)param;
|
||||
if (pInfo->pCtbCursor != NULL) {
|
||||
pInfo->pStorageAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor, 1);
|
||||
}
|
||||
taosHashCleanup(pInfo->filterCtx.colHash);
|
||||
taosArrayDestroy(pInfo->filterCtx.cInfoList);
|
||||
taosArrayDestroy(pInfo->aFilterIdxs);
|
||||
taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
|
||||
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
taosArrayDestroy(pInfo->matchInfo.pList);
|
||||
pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
|
||||
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
|
||||
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pTagScanNode,
|
||||
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo) {
|
||||
SScanPhysiNode* pPhyNode = (SScanPhysiNode*)pTagScanNode;
|
||||
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
|
|
@ -2776,6 +3013,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
|
|||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->pTagCond = pTagCond;
|
||||
pInfo->pTagIndexCond = pTagIndexCond;
|
||||
pInfo->suid = pPhyNode->suid;
|
||||
pInfo->pStorageAPI = &pTaskInfo->storageAPI;
|
||||
|
||||
pInfo->pTableListInfo = pTableListInfo;
|
||||
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
|
||||
pInfo->readHandle = *pReadHandle;
|
||||
|
|
@ -2787,8 +3029,18 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
|
|||
initResultSizeInfo(&pOperator->resultInfo, 4096);
|
||||
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
|
||||
|
||||
if (pTagScanNode->onlyMetaCtbIdx) {
|
||||
pInfo->aUidTags = taosArrayInit(pOperator->resultInfo.capacity, sizeof(STUidTagInfo));
|
||||
pInfo->aFilterIdxs = taosArrayInit(pOperator->resultInfo.capacity, sizeof(int32_t));
|
||||
pInfo->filterCtx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
|
||||
pInfo->filterCtx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
|
||||
if (pInfo->pTagCond != NULL) {
|
||||
nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx);
|
||||
}
|
||||
}
|
||||
__optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdx : doTagScanFromMetaEntry;
|
||||
pOperator->fpSet =
|
||||
createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL);
|
||||
createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL);
|
||||
|
||||
return pOperator;
|
||||
|
||||
|
|
@ -2931,6 +3183,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
|
|||
pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
|
||||
|
||||
tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit);
|
||||
tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
|
||||
}
|
||||
|
||||
tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL);
|
||||
|
|
@ -2938,17 +3191,22 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
|
|||
// one table has one data block
|
||||
int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
|
||||
|
||||
STableMergeScanSortSourceParam param = {0};
|
||||
param.pOperator = pOperator;
|
||||
STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
|
||||
param->pOperator = pOperator;
|
||||
STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx);
|
||||
pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, NULL);
|
||||
|
||||
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
|
||||
ps->param = ¶m;
|
||||
ps->onlyRef = true;
|
||||
ps->param = param;
|
||||
ps->onlyRef = false;
|
||||
tsortAddSource(pInfo->pSortHandle, ps);
|
||||
|
||||
int32_t code = tsortOpen(pInfo->pSortHandle);
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
if (numOfTable == 1) {
|
||||
tsortSetSingleTableMerge(pInfo->pSortHandle);
|
||||
} else {
|
||||
code = tsortOpen(pInfo->pSortHandle);
|
||||
}
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
T_LONG_JMP(pTaskInfo->env, terrno);
|
||||
|
|
|
|||
|
|
@ -1601,6 +1601,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
|
|||
SSysTableScanInfo* pInfo = pOperator->info;
|
||||
char dbName[TSDB_DB_NAME_LEN] = {0};
|
||||
|
||||
if (isTaskKilled(pOperator->pTaskInfo)) {
|
||||
setOperatorCompleted(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
blockDataCleanup(pInfo->pRes);
|
||||
|
||||
const char* name = tNameGetTableName(&pInfo->name);
|
||||
|
|
|
|||
|
|
@ -876,7 +876,67 @@ bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) {
|
|||
return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark;
|
||||
}
|
||||
|
||||
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
|
||||
static bool tsKeyCompFn(void* l, void* r, void* param) {
|
||||
TSKEY* lTS = (TSKEY*)l;
|
||||
TSKEY* rTS = (TSKEY*)r;
|
||||
SIntervalAggOperatorInfo* pInfo = param;
|
||||
return pInfo->binfo.outputTsOrder == ORDER_ASC ? *lTS < *rTS : *lTS > *rTS;
|
||||
}
|
||||
|
||||
static bool isCalculatedWin(SIntervalAggOperatorInfo* pInfo, const STimeWindow* win, uint64_t tableGroupId) {
|
||||
char keyBuf[sizeof(TSKEY) + sizeof(uint64_t)] = {0};
|
||||
SET_RES_WINDOW_KEY(keyBuf, (char*)&win->skey, sizeof(TSKEY), tableGroupId);
|
||||
return tSimpleHashGet(pInfo->aggSup.pResultRowHashTable, keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))) != NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief check if cur window should be filtered out by limit info
|
||||
* @retval true if should be filtered out
|
||||
* @retval false if not filtering out
|
||||
* @note If no limit info, we skip filtering.
|
||||
* If input/output ts order mismatch, we skip filtering too.
|
||||
* eg. input ts order: desc, and output ts order: asc, limit: 10
|
||||
* IntervalOperator should output the first 10 windows, however, we can't find the first 10 windows until we scan
|
||||
* every tuple in every block.
|
||||
* And the boundedQueue keeps refreshing all records with smaller ts key.
|
||||
*/
|
||||
static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STimeWindow* win, uint64_t groupId) {
|
||||
if (!pOperatorInfo->limited // if no limit info, no filter will be applied
|
||||
|| pOperatorInfo->binfo.inputTsOrder !=
|
||||
pOperatorInfo->binfo.outputTsOrder // if input/output ts order mismatch, no filter
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (pOperatorInfo->limit == 0) return true;
|
||||
|
||||
if (pOperatorInfo->pBQ == NULL) {
|
||||
pOperatorInfo->pBQ = createBoundedQueue(pOperatorInfo->limit - 1, tsKeyCompFn, taosMemoryFree, pOperatorInfo);
|
||||
}
|
||||
|
||||
bool shouldFilter = false;
|
||||
// if BQ has been full, compare it with top of BQ
|
||||
if (taosBQSize(pOperatorInfo->pBQ) == taosBQMaxSize(pOperatorInfo->pBQ) + 1) {
|
||||
PriorityQueueNode* top = taosBQTop(pOperatorInfo->pBQ);
|
||||
shouldFilter = tsKeyCompFn(top->data, &win->skey, pOperatorInfo);
|
||||
}
|
||||
if (shouldFilter) {
|
||||
return true;
|
||||
} else if (isCalculatedWin(pOperatorInfo, win, groupId)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// cur win not been filtered out and not been pushed into BQ yet, push it into BQ
|
||||
PriorityQueueNode node = {.data = taosMemoryMalloc(sizeof(TSKEY))};
|
||||
*((TSKEY*)node.data) = win->skey;
|
||||
|
||||
if (NULL == taosBQPush(pOperatorInfo->pBQ, &node)) {
|
||||
taosMemoryFree(node.data);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
|
||||
int32_t scanFlag) {
|
||||
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
|
||||
|
||||
|
|
@ -891,8 +951,21 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols);
|
||||
SResultRow* pResult = NULL;
|
||||
|
||||
if (tableGroupId != pInfo->curGroupId) {
|
||||
pInfo->handledGroupNum += 1;
|
||||
if (pInfo->slimited && pInfo->handledGroupNum > pInfo->slimit) {
|
||||
return true;
|
||||
} else {
|
||||
pInfo->curGroupId = tableGroupId;
|
||||
destroyBoundedQueue(pInfo->pBQ);
|
||||
pInfo->pBQ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
STimeWindow win =
|
||||
getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->binfo.inputTsOrder);
|
||||
if (filterWindowWithLimit(pInfo, &win, tableGroupId)) return false;
|
||||
|
||||
int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
|
||||
pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
|
||||
|
|
@ -929,7 +1002,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
while (1) {
|
||||
int32_t prevEndPos = forwardRows - 1 + startPos;
|
||||
startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->binfo.inputTsOrder);
|
||||
if (startPos < 0) {
|
||||
if (startPos < 0 || filterWindowWithLimit(pInfo, &nextWin, tableGroupId)) {
|
||||
break;
|
||||
}
|
||||
// null data, failed to allocate more memory buffer
|
||||
|
|
@ -963,6 +1036,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
if (pInfo->timeWindowInterpo) {
|
||||
saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) {
|
||||
|
|
@ -1043,7 +1117,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
|
|||
|
||||
// the pDataBlock are always the same one, no need to call this again
|
||||
setInputDataBlock(pSup, pBlock, pInfo->binfo.inputTsOrder, scanFlag, true);
|
||||
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
|
||||
if (hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag)) break;
|
||||
}
|
||||
|
||||
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->binfo.outputTsOrder);
|
||||
|
|
@ -1495,6 +1569,7 @@ void destroyIntervalOperatorInfo(void* param) {
|
|||
|
||||
cleanupGroupResInfo(&pInfo->groupResInfo);
|
||||
colDataDestroy(&pInfo->twAggSup.timeWindowData);
|
||||
destroyBoundedQueue(pInfo->pBQ);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
|
|
@ -1662,6 +1737,17 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
|
|||
pInfo->interval = interval;
|
||||
pInfo->twAggSup = as;
|
||||
pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
|
||||
if (pPhyNode->window.node.pLimit) {
|
||||
SLimitNode* pLimit = (SLimitNode*)pPhyNode->window.node.pLimit;
|
||||
pInfo->limited = true;
|
||||
pInfo->limit = pLimit->limit + pLimit->offset;
|
||||
}
|
||||
if (pPhyNode->window.node.pSlimit) {
|
||||
SLimitNode* pLimit = (SLimitNode*)pPhyNode->window.node.pSlimit;
|
||||
pInfo->slimited = true;
|
||||
pInfo->slimit = pLimit->limit + pLimit->offset;
|
||||
pInfo->curGroupId = UINT64_MAX;
|
||||
}
|
||||
|
||||
if (pPhyNode->window.pExprs != NULL) {
|
||||
int32_t numOfScalar = 0;
|
||||
|
|
|
|||
|
|
@ -69,8 +69,22 @@ struct SSortHandle {
|
|||
_sort_fetch_block_fn_t fetchfp;
|
||||
_sort_merge_compar_fn_t comparFn;
|
||||
SMultiwayMergeTreeInfo* pMergeTree;
|
||||
|
||||
bool singleTableMerge;
|
||||
|
||||
bool (*abortCheckFn)(void* param);
|
||||
void* abortCheckParam;
|
||||
};
|
||||
|
||||
void tsortSetSingleTableMerge(SSortHandle* pHandle) {
|
||||
pHandle->singleTableMerge = true;
|
||||
}
|
||||
|
||||
void tsortSetAbortCheckFn(SSortHandle *pHandle, bool (*checkFn)(void *), void* param) {
|
||||
pHandle->abortCheckFn = checkFn;
|
||||
pHandle->abortCheckParam = param;
|
||||
}
|
||||
|
||||
static int32_t msortComparFn(const void* pLeft, const void* pRight, void* param);
|
||||
|
||||
// | offset[0] | offset[1] |....| nullbitmap | data |...|
|
||||
|
|
@ -720,11 +734,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
|
|||
|
||||
SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t));
|
||||
while (1) {
|
||||
if (tsortIsClosed(pHandle)) {
|
||||
if (tsortIsClosed(pHandle) || (pHandle->abortCheckFn && pHandle->abortCheckFn(pHandle->abortCheckParam))) {
|
||||
code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED;
|
||||
return code;
|
||||
}
|
||||
|
||||
SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows);
|
||||
if (pDataBlock == NULL) {
|
||||
break;
|
||||
|
|
@ -1458,6 +1471,26 @@ static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) {
|
|||
return &pHandle->tupleHandle;
|
||||
}
|
||||
|
||||
static STupleHandle* tsortSingleTableMergeNextTuple(SSortHandle* pHandle) {
|
||||
if (1 == pHandle->numOfCompletedSources) return NULL;
|
||||
if (pHandle->tupleHandle.pBlock && pHandle->tupleHandle.rowIndex + 1 < pHandle->tupleHandle.pBlock->info.rows) {
|
||||
pHandle->tupleHandle.rowIndex++;
|
||||
} else {
|
||||
if (pHandle->tupleHandle.rowIndex == -1) return NULL;
|
||||
SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
|
||||
SSortSource* source = *pSource;
|
||||
SSDataBlock* pBlock = pHandle->fetchfp(source->param);
|
||||
if (!pBlock || pBlock->info.rows == 0) {
|
||||
setCurrentSourceDone(source, pHandle);
|
||||
pHandle->tupleHandle.pBlock = NULL;
|
||||
return NULL;
|
||||
}
|
||||
pHandle->tupleHandle.pBlock = pBlock;
|
||||
pHandle->tupleHandle.rowIndex = 0;
|
||||
}
|
||||
return &pHandle->tupleHandle;
|
||||
}
|
||||
|
||||
int32_t tsortOpen(SSortHandle* pHandle) {
|
||||
if (pHandle->opened) {
|
||||
return 0;
|
||||
|
|
@ -1475,7 +1508,9 @@ int32_t tsortOpen(SSortHandle* pHandle) {
|
|||
}
|
||||
|
||||
STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
|
||||
if (pHandle->pBoundedQueue)
|
||||
if (pHandle->singleTableMerge)
|
||||
return tsortSingleTableMergeNextTuple(pHandle);
|
||||
else if (pHandle->pBoundedQueue)
|
||||
return tsortPQSortNextTuple(pHandle);
|
||||
else
|
||||
return tsortBufMergeSortNextTuple(pHandle);
|
||||
|
|
|
|||
|
|
@ -2350,7 +2350,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "leastsquares",
|
||||
.type = FUNCTION_TYPE_LEASTSQUARES,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
|
||||
.translateFunc = translateLeastSQR,
|
||||
.getEnvFunc = getLeastSQRFuncEnv,
|
||||
.initFunc = leastSQRFunctionSetup,
|
||||
|
|
@ -2916,8 +2916,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "tail",
|
||||
.type = FUNCTION_TYPE_TAIL,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
|
||||
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateTail,
|
||||
.getEnvFunc = getTailFuncEnv,
|
||||
.initFunc = tailFunctionSetup,
|
||||
|
|
@ -2928,8 +2927,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "unique",
|
||||
.type = FUNCTION_TYPE_UNIQUE,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
|
||||
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateUnique,
|
||||
.getEnvFunc = getUniqueFuncEnv,
|
||||
.initFunc = uniqueFunctionSetup,
|
||||
|
|
|
|||
|
|
@ -848,7 +848,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
|
|||
|
||||
char path[PATH_MAX] = {0};
|
||||
udfdGetFuncBodyPath(udf, path);
|
||||
bool fileExist = !(taosStatFile(path, NULL, NULL) < 0);
|
||||
bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
|
||||
if (fileExist) {
|
||||
strncpy(udf->path, path, PATH_MAX);
|
||||
fnInfo("udfd func body file. reuse existing file %s", path);
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
|
|||
return ctx->offset;
|
||||
} else {
|
||||
int64_t file_size = 0;
|
||||
taosStatFile(ctx->file.buf, &file_size, NULL);
|
||||
taosStatFile(ctx->file.buf, &file_size, NULL, NULL);
|
||||
return (int)file_size;
|
||||
}
|
||||
}
|
||||
|
|
@ -199,7 +199,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
|
|||
code = taosFtruncateFile(ctx->file.pFile, 0);
|
||||
UNUSED(code);
|
||||
|
||||
code = taosStatFile(path, &ctx->file.size, NULL);
|
||||
code = taosStatFile(path, &ctx->file.size, NULL, NULL);
|
||||
UNUSED(code);
|
||||
|
||||
ctx->file.wBufOffset = 0;
|
||||
|
|
|
|||
|
|
@ -564,7 +564,9 @@ static int32_t physiScanCopy(const SScanPhysiNode* pSrc, SScanPhysiNode* pDst) {
|
|||
}
|
||||
|
||||
static int32_t physiTagScanCopy(const STagScanPhysiNode* pSrc, STagScanPhysiNode* pDst) {
|
||||
return physiScanCopy(pSrc, pDst);
|
||||
COPY_BASE_OBJECT_FIELD(scan, physiScanCopy);
|
||||
COPY_SCALAR_FIELD(onlyMetaCtbIdx);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhysiNode* pDst) {
|
||||
|
|
|
|||
|
|
@ -197,8 +197,10 @@ const char* nodesNodeName(ENodeType type) {
|
|||
return "ShowDnodesStmt";
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
return "ShowMnodesStmt";
|
||||
/*
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
return "ShowModulesStmt";
|
||||
*/
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
return "ShowQnodesStmt";
|
||||
case QUERY_NODE_SHOW_SNODES_STMT:
|
||||
|
|
@ -660,6 +662,7 @@ static const char* jkScanLogicPlanDynamicScanFuncs = "DynamicScanFuncs";
|
|||
static const char* jkScanLogicPlanDataRequired = "DataRequired";
|
||||
static const char* jkScanLogicPlanTagCond = "TagCond";
|
||||
static const char* jkScanLogicPlanGroupTags = "GroupTags";
|
||||
static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
|
||||
|
||||
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
|
||||
|
|
@ -701,7 +704,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -746,6 +751,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -1562,7 +1570,7 @@ static const char* jkScanPhysiPlanTableName = "TableName";
|
|||
static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan";
|
||||
|
||||
static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
|
||||
const SScanPhysiNode* pNode = (const SScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = physicPlanNodeToJson(pObj, pJson);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
|
@ -1591,7 +1599,7 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
|
|||
}
|
||||
|
||||
static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
|
||||
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
|
||||
SScanPhysiNode* pNode = (SScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
|
|
@ -1619,6 +1627,30 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static const char* jkTagScanPhysiOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
|
||||
|
||||
static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = physiScanNodeToJson(pObj, pJson);
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkTagScanPhysiOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
|
||||
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = jsonToPhysiScanNode(pJson, pObj);
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetBoolValue(pJson, jkTagScanPhysiOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags";
|
||||
static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort";
|
||||
|
||||
|
|
@ -6590,6 +6622,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
|
|||
case QUERY_NODE_LOGIC_PLAN:
|
||||
return logicPlanToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
|
||||
return physiTagScanNodeToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
|
||||
return physiScanNodeToJson(pObj, pJson);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
|
||||
|
|
@ -6908,6 +6941,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
|
|||
case QUERY_NODE_LOGIC_PLAN:
|
||||
return jsonToLogicPlan(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
|
||||
return jsonToPhysiTagScanNode(pJson, pObj);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
|
||||
return jsonToPhysiScanNode(pJson, pObj);
|
||||
|
|
|
|||
|
|
@ -2003,6 +2003,43 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
enum {
|
||||
PHY_TAG_SCAN_CODE_SCAN = 1,
|
||||
PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX
|
||||
};
|
||||
|
||||
static int32_t physiTagScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
||||
const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = tlvEncodeObj(pEncoder, PHY_TAG_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan);
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX, pNode->onlyMetaCtbIdx);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t msgToPhysiTagScanNode(STlvDecoder* pDecoder, void* pObj) {
|
||||
STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj;
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
STlv* pTlv = NULL;
|
||||
tlvForEach(pDecoder, pTlv, code) {
|
||||
switch (pTlv->type) {
|
||||
case PHY_TAG_SCAN_CODE_SCAN:
|
||||
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan);
|
||||
break;
|
||||
case PHY_TAG_SCAN_CODE_ONLY_META_CTB_IDX:
|
||||
code = tlvDecodeBool(pTlv, &pNode->onlyMetaCtbIdx);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
enum {
|
||||
PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
|
||||
PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
|
||||
|
|
@ -3726,6 +3763,8 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
|||
code = caseWhenNodeToMsg(pObj, pEncoder);
|
||||
break;
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
|
||||
code = physiTagScanNodeToMsg(pObj, pEncoder);
|
||||
break;
|
||||
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
|
||||
code = physiScanNodeToMsg(pObj, pEncoder);
|
||||
break;
|
||||
|
|
@ -3869,6 +3908,8 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
code = msgToCaseWhenNode(pDecoder, pObj);
|
||||
break;
|
||||
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
|
||||
code = msgToPhysiTagScanNode(pDecoder, pObj);
|
||||
break;
|
||||
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
|
||||
code = msgToPhysiScanNode(pDecoder, pObj);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -406,7 +406,7 @@ SNode* nodesMakeNode(ENodeType type) {
|
|||
return makeNode(type, sizeof(SRevokeStmt));
|
||||
case QUERY_NODE_SHOW_DNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
// case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
case QUERY_NODE_SHOW_SNODES_STMT:
|
||||
case QUERY_NODE_SHOW_BNODES_STMT:
|
||||
|
|
@ -982,7 +982,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
break;
|
||||
case QUERY_NODE_SHOW_DNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
// case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
case QUERY_NODE_SHOW_SNODES_STMT:
|
||||
case QUERY_NODE_SHOW_BNODES_STMT:
|
||||
|
|
|
|||
|
|
@ -199,9 +199,10 @@ TEST_F(NodesCloneTest, physiScan) {
|
|||
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
|
||||
STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc;
|
||||
STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst;
|
||||
ASSERT_EQ(pSrcNode->uid, pDstNode->uid);
|
||||
ASSERT_EQ(pSrcNode->suid, pDstNode->suid);
|
||||
ASSERT_EQ(pSrcNode->tableType, pDstNode->tableType);
|
||||
ASSERT_EQ(pSrcNode->scan.uid, pDstNode->scan.uid);
|
||||
ASSERT_EQ(pSrcNode->scan.suid, pDstNode->scan.suid);
|
||||
ASSERT_EQ(pSrcNode->scan.tableType, pDstNode->scan.tableType);
|
||||
ASSERT_EQ(pSrcNode->onlyMetaCtbIdx, pDstNode->onlyMetaCtbIdx);
|
||||
});
|
||||
|
||||
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
|
||||
|
|
|
|||
|
|
@ -690,8 +690,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
|
|||
return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt);
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
return collectMetaKeyFromShowMnodes(pCxt, (SShowStmt*)pStmt);
|
||||
/*
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
return collectMetaKeyFromShowModules(pCxt, (SShowStmt*)pStmt);
|
||||
*/
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
return collectMetaKeyFromShowQnodes(pCxt, (SShowStmt*)pStmt);
|
||||
case QUERY_NODE_SHOW_SNODES_STMT:
|
||||
|
|
|
|||
|
|
@ -263,7 +263,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
|
|||
return authAlterTable(pCxt, (SAlterTableStmt*)pStmt);
|
||||
case QUERY_NODE_SHOW_DNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
// case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
case QUERY_NODE_SHOW_SNODES_STMT:
|
||||
case QUERY_NODE_SHOW_BNODES_STMT:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ static void clearColValArray(SArray* pCols) {
|
|||
int32_t num = taosArrayGetSize(pCols);
|
||||
for (int32_t i = 0; i < num; ++i) {
|
||||
SColVal* pCol = taosArrayGet(pCols, i);
|
||||
if (TSDB_DATA_TYPE_NCHAR == pCol->type) {
|
||||
if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) {
|
||||
taosMemoryFreeClear(pCol->value.pData);
|
||||
}
|
||||
pCol->flag = CV_FLAG_NONE;
|
||||
|
|
@ -237,9 +237,13 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
|
|||
}
|
||||
pVal->value.pData = pUcs4;
|
||||
pVal->value.nData = len;
|
||||
} else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
|
||||
pVal->value.nData = kv->length;
|
||||
pVal->value.pData = (uint8_t*)kv->value;
|
||||
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
pVal->value.nData = kv->length;
|
||||
pVal->value.pData = taosMemoryMalloc(kv->length);
|
||||
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
|
||||
} else {
|
||||
memcpy(&pVal->value.val, &(kv->value), kv->length);
|
||||
}
|
||||
|
|
@ -364,9 +368,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
|
|||
}
|
||||
pVal->value.pData = pUcs4;
|
||||
pVal->value.nData = len;
|
||||
} else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
|
||||
pVal->value.nData = kv->length;
|
||||
pVal->value.pData = (uint8_t*)kv->value;
|
||||
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
|
||||
pVal->value.nData = kv->length;
|
||||
pVal->value.pData = taosMemoryMalloc(kv->length);
|
||||
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
|
||||
} else {
|
||||
memcpy(&pVal->value.val, &(kv->value), kv->length);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -333,7 +333,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta*
|
|||
|
||||
static void destroyColVal(void* p) {
|
||||
SColVal* pVal = p;
|
||||
if (TSDB_DATA_TYPE_NCHAR == pVal->type) {
|
||||
if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type) {
|
||||
taosMemoryFree(pVal->value.pData);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
|
|||
.numOfShowCols = 1,
|
||||
.pShowCols = {"*"}
|
||||
},
|
||||
/*
|
||||
{
|
||||
.showType = QUERY_NODE_SHOW_MODULES_STMT,
|
||||
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
|
||||
|
|
@ -99,6 +100,7 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
|
|||
.numOfShowCols = 1,
|
||||
.pShowCols = {"*"}
|
||||
},
|
||||
*/
|
||||
{
|
||||
.showType = QUERY_NODE_SHOW_QNODES_STMT,
|
||||
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
|
||||
|
|
@ -278,7 +280,8 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
|
|||
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
|
||||
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode);
|
||||
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal);
|
||||
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt);
|
||||
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
|
||||
SSelectStmt** pStmt);
|
||||
static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
|
||||
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery);
|
||||
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery);
|
||||
|
|
@ -772,7 +775,8 @@ static SNodeList* getProjectList(const SNode* pNode) {
|
|||
|
||||
static bool isTimeLineQuery(SNode* pStmt) {
|
||||
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
|
||||
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
|
||||
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) ||
|
||||
(TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
|
||||
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
|
||||
return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
|
||||
} else {
|
||||
|
|
@ -821,6 +825,18 @@ static bool isPrimaryKeyImpl(SNode* pExpr) {
|
|||
FUNCTION_TYPE_IROWTS == pFunc->funcType) {
|
||||
return true;
|
||||
}
|
||||
} else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) {
|
||||
SOperatorNode* pOper = (SOperatorNode*)pExpr;
|
||||
if (OP_TYPE_ADD != pOper->opType && OP_TYPE_SUB != pOper->opType) {
|
||||
return false;
|
||||
}
|
||||
if (!isPrimaryKeyImpl(pOper->pLeft)) {
|
||||
return false;
|
||||
}
|
||||
if (QUERY_NODE_VALUE != nodeType(pOper->pRight)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1596,9 +1612,11 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc
|
|||
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
|
||||
}
|
||||
|
||||
if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
|
||||
if (pSelect->hasInterpFunc &&
|
||||
(FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
|
||||
"%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName);
|
||||
"%s ignoring null value options cannot be used when applying to multiple columns",
|
||||
pFunc->functionName);
|
||||
}
|
||||
|
||||
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
|
||||
|
|
@ -1636,7 +1654,8 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu
|
|||
}
|
||||
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
|
||||
if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
|
||||
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
|
||||
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) &&
|
||||
!isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
|
||||
"%s function requires valid time series input", pFunc->functionName);
|
||||
}
|
||||
|
|
@ -2296,7 +2315,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
|
|||
}
|
||||
}
|
||||
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
|
||||
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
|
||||
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc ||
|
||||
(isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
|
||||
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
|
||||
} else {
|
||||
return rewriteColToSelectValFunc(pCxt, pNode);
|
||||
|
|
@ -2699,7 +2719,8 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) {
|
|||
}
|
||||
|
||||
SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable;
|
||||
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType && TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
|
||||
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType &&
|
||||
TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -3110,7 +3131,8 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
|
|||
code = scalarCalculateConstants(pCastFunc, &pCell->pNode);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) {
|
||||
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
|
||||
code =
|
||||
generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
|
||||
} else if (TSDB_CODE_SUCCESS != code) {
|
||||
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
|
||||
}
|
||||
|
|
@ -3576,7 +3598,6 @@ static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput)
|
|||
pEvery->isDuration = true;
|
||||
pEvery->literal = taosStrdup("1s");
|
||||
|
||||
|
||||
*pOutput = (SNode*)pEvery;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -3675,8 +3696,8 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
|
|||
if (pSelect->pPartitionByList) {
|
||||
int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable);
|
||||
SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0);
|
||||
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) &&
|
||||
1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
|
||||
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && 1 == pSelect->pPartitionByList->length &&
|
||||
(QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
|
||||
pSelect->timeLineResMode = TIME_LINE_MULTI;
|
||||
}
|
||||
|
||||
|
|
@ -3943,9 +3964,9 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
|
|||
}
|
||||
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
|
||||
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
|
||||
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)
|
||||
&& ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID
|
||||
&& ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) &&
|
||||
((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
|
||||
((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
|
||||
((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
|
||||
|
|
@ -5725,7 +5746,6 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN
|
|||
return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq);
|
||||
}
|
||||
|
||||
|
||||
static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName,
|
||||
int32_t* pVgId) {
|
||||
SVgroupInfo vg = {0};
|
||||
|
|
@ -5867,8 +5887,10 @@ int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval*
|
|||
pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
|
||||
pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
|
||||
pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0;
|
||||
pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
|
||||
pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
|
||||
pInterval->sliding =
|
||||
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
|
||||
pInterval->slidingUnit =
|
||||
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
|
||||
pInterval->precision = pStmt->pOptions->tsPrecision;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -6100,7 +6122,8 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) {
|
|||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) {
|
||||
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta,
|
||||
SNodeList** ppProjection) {
|
||||
SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL};
|
||||
nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt);
|
||||
if (colCxt.colNotFound) {
|
||||
|
|
@ -6542,7 +6565,8 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
|
|||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
|
||||
"SUBTABLE expression must be of VARCHAR type");
|
||||
}
|
||||
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) && subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
|
||||
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) &&
|
||||
subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
|
||||
"SUBTABLE expression must not has column when no partition by clause");
|
||||
}
|
||||
|
|
@ -6584,7 +6608,10 @@ typedef struct SProjColPos {
|
|||
} SProjColPos;
|
||||
|
||||
static int32_t projColPosCompar(const void* l, const void* r) {
|
||||
return ((SProjColPos*)l)->colId > ((SProjColPos*)r)->colId;
|
||||
if (((SProjColPos*)l)->colId < ((SProjColPos*)r)->colId) {
|
||||
return -1;
|
||||
}
|
||||
return ((SProjColPos*)l)->colId == ((SProjColPos*)r)->colId ? 0 : 1;
|
||||
}
|
||||
|
||||
static void projColPosDelete(void* p) { nodesDestroyNode(((SProjColPos*)p)->pProj); }
|
||||
|
|
@ -7121,7 +7148,6 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) {
|
||||
SMDropStreamReq dropReq = {0};
|
||||
SName name;
|
||||
|
|
@ -7152,7 +7178,7 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
|
|||
|
||||
static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
|
||||
int64_t filesize = 0;
|
||||
if (taosStatFile(pName, &filesize, NULL) < 0) {
|
||||
if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
|
||||
|
|
@ -7806,7 +7832,8 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) {
|
|||
return pProjections;
|
||||
}
|
||||
|
||||
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) {
|
||||
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList,
|
||||
SSelectStmt** pStmt) {
|
||||
SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
|
||||
if (NULL == pSelect) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
|
@ -7829,7 +7856,6 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs,
|
||||
const char* const pProjCol[], SSelectStmt** pStmt) {
|
||||
SNodeList* pProjectionList = NULL;
|
||||
|
|
@ -7843,13 +7869,15 @@ static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTabl
|
|||
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
|
||||
}
|
||||
|
||||
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) {
|
||||
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
|
||||
SSelectStmt** pStmt) {
|
||||
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
|
||||
}
|
||||
|
||||
static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) {
|
||||
const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET];
|
||||
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt);
|
||||
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols,
|
||||
pStmt);
|
||||
}
|
||||
|
||||
static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) {
|
||||
|
|
@ -7987,8 +8015,8 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis
|
|||
static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
|
||||
SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot;
|
||||
SSelectStmt* pSelect = NULL;
|
||||
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal,
|
||||
-1, NULL, &pSelect);
|
||||
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal,
|
||||
((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
|
||||
}
|
||||
|
|
@ -9149,7 +9177,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
|
|||
case QUERY_NODE_SHOW_USERS_STMT:
|
||||
case QUERY_NODE_SHOW_DNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MNODES_STMT:
|
||||
case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
// case QUERY_NODE_SHOW_MODULES_STMT:
|
||||
case QUERY_NODE_SHOW_QNODES_STMT:
|
||||
case QUERY_NODE_SHOW_FUNCTIONS_STMT:
|
||||
case QUERY_NODE_SHOW_INDEXES_STMT:
|
||||
|
|
|
|||
|
|
@ -847,7 +847,6 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva
|
|||
: (pSelect->hasTimeLineFunc ? getRequireDataOrder(true, pSelect) : DATA_ORDER_LEVEL_IN_BLOCK);
|
||||
pWindow->node.resultDataOrder =
|
||||
pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : getRequireDataOrder(true, pSelect);
|
||||
|
||||
pWindow->pTspk = nodesCloneNode(pInterval->pCol);
|
||||
if (NULL == pWindow->pTspk) {
|
||||
nodesDestroyNode((SNode*)pWindow);
|
||||
|
|
|
|||
|
|
@ -1562,11 +1562,33 @@ static bool planOptNodeListHasTbname(SNodeList* pKeys) {
|
|||
}
|
||||
|
||||
static bool partTagsIsOptimizableNode(SLogicNode* pNode) {
|
||||
return ((QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) ||
|
||||
(QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && NULL != ((SAggLogicNode*)pNode)->pGroupKeys &&
|
||||
NULL != ((SAggLogicNode*)pNode)->pAggFuncs)) &&
|
||||
1 == LIST_LENGTH(pNode->pChildren) &&
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0)));
|
||||
bool ret = 1 == LIST_LENGTH(pNode->pChildren) &&
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0));
|
||||
if (!ret) return ret;
|
||||
switch (nodeType(pNode)) {
|
||||
case QUERY_NODE_LOGIC_PLAN_PARTITION: {
|
||||
if (pNode->pParent && nodeType(pNode->pParent) == QUERY_NODE_LOGIC_PLAN_WINDOW) {
|
||||
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode->pParent;
|
||||
if (pWindow->winType == WINDOW_TYPE_INTERVAL) {
|
||||
// if interval has slimit, we push down partition node to scan, and scan will set groupOrderScan to true
|
||||
// we want to skip groups of blocks after slimit satisfied
|
||||
// if interval only has limit, we do not push down partition node to scan
|
||||
// we want to get grouped output from partition node and make use of limit
|
||||
// if no slimit and no limit, we push down partition node and groupOrderScan is false, cause we do not need
|
||||
// group ordered output
|
||||
if (!pWindow->node.pSlimit && pWindow->node.pLimit) ret = false;
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case QUERY_NODE_LOGIC_PLAN_AGG: {
|
||||
SAggLogicNode* pAgg = (SAggLogicNode*)pNode;
|
||||
ret = pAgg->pGroupKeys && pAgg->pAggFuncs;
|
||||
} break;
|
||||
default:
|
||||
ret = false;
|
||||
break;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static SNodeList* partTagsGetPartKeys(SLogicNode* pNode) {
|
||||
|
|
@ -1707,6 +1729,8 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
|
|||
scanPathOptSetGroupOrderScan(pScan);
|
||||
pParent->hasGroupKeyOptimized = true;
|
||||
}
|
||||
if (pNode->pParent->pSlimit)
|
||||
pScan->groupOrderScan = true;
|
||||
|
||||
NODES_CLEAR_LIST(pNode->pChildren);
|
||||
nodesDestroyNode((SNode*)pNode);
|
||||
|
|
@ -2655,26 +2679,85 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
|
|||
}
|
||||
nodesDestroyNode((SNode*)pAgg);
|
||||
tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode);
|
||||
|
||||
pScanNode->onlyMetaCtbIdx = false;
|
||||
|
||||
pCxt->optimized = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static bool pushDownLimitOptShouldBeOptimized(SLogicNode* pNode) {
|
||||
if (NULL == pNode->pLimit || 1 != LIST_LENGTH(pNode->pChildren)) {
|
||||
if ((NULL == pNode->pLimit && pNode->pSlimit == NULL) || 1 != LIST_LENGTH(pNode->pChildren)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0);
|
||||
// push down to sort node
|
||||
if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) {
|
||||
// if we have pushed down, we skip it
|
||||
if (pChild->pLimit) return false;
|
||||
} else if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pChild) || QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pNode)) {
|
||||
// push down to table scan node
|
||||
// if pNode is sortNode, we skip push down limit info to table scan node
|
||||
if (pChild->pLimit || pChild->pSlimit) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void swapLimit(SLogicNode* pParent, SLogicNode* pChild) {
|
||||
pChild->pLimit = pParent->pLimit;
|
||||
pParent->pLimit = NULL;
|
||||
}
|
||||
|
||||
static void cloneLimit(SLogicNode* pParent, SLogicNode* pChild) {
|
||||
SLimitNode* pLimit = NULL;
|
||||
if (pParent->pLimit) {
|
||||
pChild->pLimit = nodesCloneNode(pParent->pLimit);
|
||||
pLimit = (SLimitNode*)pChild->pLimit;
|
||||
pLimit->limit += pLimit->offset;
|
||||
pLimit->offset = 0;
|
||||
}
|
||||
|
||||
if (pParent->pSlimit) {
|
||||
pChild->pSlimit = nodesCloneNode(pParent->pSlimit);
|
||||
pLimit = (SLimitNode*)pChild->pSlimit;
|
||||
pLimit->limit += pLimit->offset;
|
||||
pLimit->offset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static bool pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo);
|
||||
static bool pushDownLimitTo(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo) {
|
||||
switch (nodeType(pNodeLimitPushTo)) {
|
||||
case QUERY_NODE_LOGIC_PLAN_WINDOW: {
|
||||
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNodeLimitPushTo;
|
||||
if (pWindow->winType != WINDOW_TYPE_INTERVAL) break;
|
||||
cloneLimit(pNodeWithLimit, pNodeLimitPushTo);
|
||||
return true;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_FILL:
|
||||
case QUERY_NODE_LOGIC_PLAN_SORT: {
|
||||
cloneLimit(pNodeWithLimit, pNodeLimitPushTo);
|
||||
SNode* pChild = NULL;
|
||||
FOREACH(pChild, pNodeLimitPushTo->pChildren) { pushDownLimitHow(pNodeLimitPushTo, (SLogicNode*)pChild); }
|
||||
return true;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_SCAN:
|
||||
if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT && pNodeWithLimit->pLimit) {
|
||||
swapLimit(pNodeWithLimit, pNodeLimitPushTo);
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
static bool pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo) {
|
||||
switch (nodeType(pNodeWithLimit)) {
|
||||
case QUERY_NODE_LOGIC_PLAN_PROJECT:
|
||||
case QUERY_NODE_LOGIC_PLAN_FILL:
|
||||
return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo);
|
||||
case QUERY_NODE_LOGIC_PLAN_SORT: {
|
||||
SSortLogicNode* pSort = (SSortLogicNode*)pNodeWithLimit;
|
||||
if (sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys)) return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo);
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
|
||||
|
|
@ -2685,17 +2768,9 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
|
|||
|
||||
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0);
|
||||
nodesDestroyNode(pChild->pLimit);
|
||||
if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) {
|
||||
pChild->pLimit = nodesCloneNode(pNode->pLimit);
|
||||
SLimitNode* pLimit = (SLimitNode*)pChild->pLimit;
|
||||
pLimit->limit += pLimit->offset;
|
||||
pLimit->offset = 0;
|
||||
} else {
|
||||
pChild->pLimit = pNode->pLimit;
|
||||
pNode->pLimit = NULL;
|
||||
}
|
||||
if (pushDownLimitHow(pNode, pChild)) {
|
||||
pCxt->optimized = true;
|
||||
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -2996,6 +3071,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
|
|||
{.pName = "sortNonPriKeyOptimize", .optimizeFunc = sortNonPriKeyOptimize},
|
||||
{.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize},
|
||||
{.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize},
|
||||
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize},
|
||||
{.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
|
||||
{.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize},
|
||||
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
|
||||
|
|
@ -3004,7 +3080,6 @@ static const SOptimizeRule optimizeRuleSet[] = {
|
|||
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize},
|
||||
{.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize},
|
||||
{.pName = "TagScan", .optimizeFunc = tagScanOptimize},
|
||||
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize},
|
||||
{.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize},
|
||||
};
|
||||
// clang-format on
|
||||
|
|
|
|||
|
|
@ -511,6 +511,20 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
|
|||
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
|
||||
}
|
||||
|
||||
static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
|
||||
SPhysiNode** pPhyNode) {
|
||||
STagScanPhysiNode* pScan =
|
||||
(STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
|
||||
if (NULL == pScan) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
|
||||
|
||||
pScan->onlyMetaCtbIdx = pScanLogicNode->onlyMetaCtbIdx;
|
||||
|
||||
return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
|
||||
}
|
||||
|
||||
static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
|
||||
SPhysiNode** pPhyNode) {
|
||||
SLastRowScanPhysiNode* pScan =
|
||||
|
|
@ -646,6 +660,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
|
|||
pCxt->hasScan = true;
|
||||
switch (pScanLogicNode->scanType) {
|
||||
case SCAN_TYPE_TAG:
|
||||
return createTagScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
|
||||
case SCAN_TYPE_BLOCK_INFO:
|
||||
return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode);
|
||||
case SCAN_TYPE_TABLE_COUNT:
|
||||
|
|
|
|||
|
|
@ -498,6 +498,18 @@ static int32_t stbSplRewriteFromMergeNode(SMergeLogicNode* pMerge, SLogicNode* p
|
|||
}
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_WINDOW: {
|
||||
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
|
||||
if (pMerge->node.pLimit) {
|
||||
nodesDestroyNode(pMerge->node.pLimit);
|
||||
pMerge->node.pLimit = NULL;
|
||||
}
|
||||
if (pMerge->node.pSlimit) {
|
||||
nodesDestroyNode(pMerge->node.pSlimit);
|
||||
pMerge->node.pSlimit = NULL;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1613,6 +1613,9 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey*
|
|||
const char* curKey = rocksdb_iter_key(pCur->iter, (size_t*)&kLen);
|
||||
stateSessionKeyDecode((void*)&ktmp, (char*)curKey);
|
||||
|
||||
if (pVal != NULL) *pVal = NULL;
|
||||
if (pVLen != NULL) *pVLen = 0;
|
||||
|
||||
SStateSessionKey* pKTmp = &ktmp;
|
||||
const char* vval = rocksdb_iter_value(pCur->iter, (size_t*)&vLen);
|
||||
char* val = NULL;
|
||||
|
|
@ -1620,19 +1623,23 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey*
|
|||
if (len < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pKTmp->opNum != pCur->number) {
|
||||
taosMemoryFree(val);
|
||||
return -1;
|
||||
}
|
||||
if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) {
|
||||
taosMemoryFree(val);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pVal != NULL) {
|
||||
*pVal = (char*)val;
|
||||
} else {
|
||||
taosMemoryFree(val);
|
||||
}
|
||||
if (pVLen != NULL) *pVLen = len;
|
||||
|
||||
if (pKTmp->opNum != pCur->number) {
|
||||
return -1;
|
||||
}
|
||||
if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) {
|
||||
return -1;
|
||||
}
|
||||
if (pVLen != NULL) *pVLen = len;
|
||||
*pKey = pKTmp->key;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) {
|
|||
const char *file = pNode->raftStorePath;
|
||||
SRaftStore *pStore = &pNode->raftStore;
|
||||
|
||||
if (taosStatFile(file, NULL, NULL) < 0) {
|
||||
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
|
||||
sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file);
|
||||
pStore->currentTerm = 0;
|
||||
pStore->voteFor.addr = 0;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "osEnv.h"
|
||||
#include "tfsInt.h"
|
||||
|
||||
static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg);
|
||||
|
|
@ -113,6 +114,39 @@ SDiskSize tfsGetSize(STfs *pTfs) {
|
|||
return size;
|
||||
}
|
||||
|
||||
bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level) {
|
||||
if (level < 0 || level >= pTfs->nlevel) {
|
||||
return false;
|
||||
}
|
||||
STfsTier *pTier = TFS_TIER_AT(pTfs, level);
|
||||
for (int32_t id = 0; id < pTier->ndisk; id++) {
|
||||
SDiskID diskId = {.level = level, .id = id};
|
||||
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
|
||||
if (pDisk == NULL) {
|
||||
return false;
|
||||
}
|
||||
if (pDisk->size.avail <= 0) {
|
||||
fError("tfs disk space unavailable. level:%d, disk:%d, path:%s", level, id, pDisk->path);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk) {
|
||||
if (level < 0 || level >= pTfs->nlevel) {
|
||||
return false;
|
||||
}
|
||||
|
||||
STfsTier *pTier = TFS_TIER_AT(pTfs, level);
|
||||
if (disk < 0 || disk >= pTier->ndisk) {
|
||||
return false;
|
||||
}
|
||||
SDiskID diskId = {.level = level, .id = disk};
|
||||
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
|
||||
return pDisk->size.avail >= tsDataSpace.reserved;
|
||||
}
|
||||
|
||||
int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) {
|
||||
if (level < 0 || level >= pTfs->nlevel) {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
|
|||
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
|
||||
int64_t fileSize = 0;
|
||||
taosStatFile(fnameStr, &fileSize, NULL);
|
||||
taosStatFile(fnameStr, &fileSize, NULL, NULL);
|
||||
|
||||
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
|
||||
if (pFile == NULL) {
|
||||
|
|
@ -305,7 +305,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
|
|||
|
||||
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
int32_t mtime = 0;
|
||||
if (taosStatFile(fnameStr, NULL, &mtime) < 0) {
|
||||
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
|
||||
return -1;
|
||||
|
|
@ -354,7 +354,7 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
|
|||
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
|
||||
int64_t fileSize = 0;
|
||||
taosStatFile(fnameStr, &fileSize, NULL);
|
||||
taosStatFile(fnameStr, &fileSize, NULL, NULL);
|
||||
int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
|
||||
int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
|
||||
|
||||
|
|
@ -437,7 +437,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
|
|||
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
|
||||
|
||||
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
|
||||
int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
|
||||
int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
|
||||
if (code < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
|
||||
|
|
@ -523,7 +523,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
|
|||
walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
|
||||
int64_t fileSize = 0;
|
||||
|
||||
if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
|
||||
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
|
||||
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
|
|
@ -936,7 +936,7 @@ int walLoadMeta(SWal* pWal) {
|
|||
walBuildMetaName(pWal, metaVer, fnameStr);
|
||||
// read metafile
|
||||
int64_t fileSize = 0;
|
||||
taosStatFile(fnameStr, &fileSize, NULL);
|
||||
taosStatFile(fnameStr, &fileSize, NULL, NULL);
|
||||
if (fileSize == 0) {
|
||||
(void)taosRemoveFile(fnameStr);
|
||||
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);
|
||||
|
|
|
|||
|
|
@ -95,10 +95,10 @@ void osCleanup() {}
|
|||
|
||||
bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; }
|
||||
|
||||
bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
|
||||
|
||||
bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; }
|
||||
|
||||
bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
|
||||
|
||||
bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; }
|
||||
|
||||
bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
|
|||
#endif
|
||||
}
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
|
||||
#ifdef WINDOWS
|
||||
struct _stati64 fileStat;
|
||||
int32_t code = _stati64(path, &fileStat);
|
||||
|
|
@ -211,6 +211,10 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
|
|||
*mtime = fileStat.st_mtime;
|
||||
}
|
||||
|
||||
if (atime != NULL) {
|
||||
*atime = fileStat.st_mtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ int32_t qsortHelper(const void* p1, const void* p2, const void* param) {
|
|||
|
||||
// todo refactor: 1) move away; 2) use merge sort instead; 3) qsort is not a stable sort actually.
|
||||
void taosSort(void* base, int64_t sz, int64_t width, __compar_fn_t compar) {
|
||||
#if defined(WINDOWS) || defined(_ALPINE)
|
||||
#if defined(WINDOWS_STASH) || defined(_ALPINE)
|
||||
void* param = compar;
|
||||
taosqsort(base, sz, width, param, qsortHelper);
|
||||
#else
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue