xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into test/3.0/TD-29957

This commit is contained in:
chenhaoran 2024-05-17 16:49:11 +08:00
parent 3ecdd57673 5760ac25e2
commit b58d21f195
130 changed files with 6824 additions and 3047 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmake/cmake.options
View File

@ -64,13 +64,6 @@ IF(${TD_WINDOWS})
ON
)
MESSAGE("build geos Win32")
option(
BUILD_GEOS
"If build geos on Windows"
ON
)
ELSEIF (TD_DARWIN_64)
IF(${BUILD_TEST})
add_definitions(-DCOMPILER_SUPPORTS_CXX13)
@ -79,7 +72,7 @@ ENDIF ()
option(
BUILD_GEOS
"If build geos on Windows"
"If build with geos"
ON
)
@ -95,6 +88,12 @@ option(
ON
)
option(
BUILD_PCRE2
"If build with pcre2"
ON
)
option(
JEMALLOC_ENABLED
"If build with jemalloc"

2
cmake/curl_CMakeLists.txt.in
View File

@ -12,7 +12,7 @@ ExternalProject_Add(curl2
BUILD_IN_SOURCE TRUE
BUILD_ALWAYS 1
UPDATE_COMMAND ""
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 #--enable-debug
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 --without-libpsl #--enable-debug
BUILD_COMMAND make -j
INSTALL_COMMAND make install
TEST_COMMAND ""

13
cmake/pcre2_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# pcre2
ExternalProject_Add(pcre2
GIT_REPOSITORY https://github.com/PCRE2Project/pcre2.git
GIT_TAG pcre2-10.43
SOURCE_DIR "${TD_CONTRIB_DIR}/pcre2"
#BINARY_DIR "${TD_CONTRIB_DIR}/pcre2"
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

3
cmake/xz_CMakeLists.txt.in
View File

@ -1,7 +1,5 @@
# xz
if (${TD_LINUX})
ExternalProject_Add(xz
GIT_REPOSITORY https://github.com/xz-mirror/xz.git
GIT_TAG v5.4.4
@ -14,4 +12,3 @@ ExternalProject_Add(xz
INSTALL_COMMAND ""
TEST_COMMAND ""
)
endif()

2
cmake/zlib_CMakeLists.txt.in
View File

@ -2,7 +2,6 @@
# zlib
if (${TD_LINUX})
ExternalProject_Add(zlib
GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
GIT_TAG v1.2.11
@ -14,4 +13,3 @@ ExternalProject_Add(zlib
INSTALL_COMMAND ""
TEST_COMMAND ""
)
endif()

13
contrib/CMakeLists.txt
View File

@ -183,6 +183,11 @@ if(${BUILD_GEOS})
cat("${TD_SUPPORT_DIR}/geos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
#
if(${BUILD_PCRE2})
cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# download dependencies
configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -250,6 +255,10 @@ target_include_directories(
# zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
if(${TD_DARWIN})
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=deprecated-non-prototype")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-non-prototype")
endif(${TD_DARWIN})
add_subdirectory(zlib EXCLUDE_FROM_ALL)
target_include_directories(
zlibstatic
@ -601,6 +610,10 @@ if(${BUILD_GEOS})
)
endif(${BUILD_GEOS})
if (${BUILD_PCRE2})
add_subdirectory(pcre2 EXCLUDE_FROM_ALL)
endif(${BUILD_PCRE2})
# ================================================================================================
# Build test
# ================================================================================================

2
docs/en/01-index.md
View File

@ -13,7 +13,7 @@ TDengine greatly improves the efficiency of data ingestion, querying, and storag
If you are a developer, please read the [Developer Guide](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, and make a few changes to accommodate your application, and it will work.
We live in the era of big data, and scale-up is unable to meet the growing needs of the business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to [Cluster Deployment](../deployment).
We live in the era of big data, and scale-up is unable to meet the growing needs of the business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to [Cluster Deployment](./deployment).
TDengine uses ubiquitous SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll-up, interpolation, and time-weighted average, among many others. The [SQL Reference](./taos-sql) chapter describes the SQL syntax in detail and lists the various supported commands and functions.

2
docs/en/07-develop/01-connect/index.md
View File

@ -36,7 +36,7 @@ For REST and native connections, client libraries provide similar APIs for perfo
Key differences:
3. The REST connection is more accessible with cross-platform support, however it results in a 30% performance downgrade.
1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](../../client-libraries/cpp#parameter-binding-api), [Subscription](../../client-libraries/cpp#subscription-and-consumption-api), etc.
1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](../../client-libraries/cpp#parameter-binding-api), [Subscription](../../client-libraries/cpp#subscription-api), etc.
## Install Client Driver taosc

2
docs/en/07-develop/02-model/index.mdx
View File

@ -41,7 +41,7 @@ Without the current database specified, table name must be preceded with the cor
## Create STable
In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](../../concept/#model_table1), the SQL statement below can be used to create the super table.
In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](../../concept/), the SQL statement below can be used to create the super table.
```sql
CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);

2
docs/en/07-develop/03-insert-data/30-influxdb-line.mdx
View File

@ -46,7 +46,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
:::
For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](../../../reference/schemaless/#Schemaless-Line-Protocol)
For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](../../../reference/schemaless/)
## Examples

2
docs/en/08-client-libraries/03-cpp.mdx
View File

@ -32,7 +32,7 @@ The version number of the TDengine client driver and the version number of the T
## Installation Steps
Please refer to the [Installation Steps](../#installation-steps) for TDengine client driver installation
Please refer to [Install Client Driver](../#install-client-driver) for TDengine client driver installation
## Establishing a connection

4
docs/en/08-client-libraries/04-java.mdx
View File

@ -160,7 +160,7 @@ For Java connector, the jts library can be used to easily create GEOMETRY type o
Before using Java client library to connect to the database, the following conditions are required.
- Java 1.8 or above runtime environment and Maven 3.6 or above installed
- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Installing Client Driver](../#Install-Client-Driver)
- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Install Client Driver](../#install-client-driver)
### Install the client library
@ -368,7 +368,7 @@ The configuration parameters in properties are as follows.
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection.
- TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20.
For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/#Client-Only).
For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/#configuration-file-on-client-side).
### Priority of configuration parameters

6
docs/en/08-client-libraries/05-go.mdx
View File

@ -129,7 +129,7 @@ username:password@protocol(address)/dbname?param=value
_taosSql_ implements Go's `database/sql/driver` interface via cgo. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver.
Use `taosSql` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName`, DSN supports the following parameters.
Use `taosSql` as `driverName` and use a correct DSN as `dataSourceName`, DSN supports the following parameters.
* cfg specifies the `taos.cfg` directory
@ -160,7 +160,7 @@ func main() {
_taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver.
Use `taosRestful` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN.
Use `taosRestful` as `driverName` and use a correct DSN as `dataSourceName` with the following parameters supported by the DSN.
* `disableCompression` whether to accept compressed data, default is true do not accept compressed data, set to false if transferring data using gzip compression.
* `readBufferSize` The default size of the buffer for reading data is 4K (4096), which can be adjusted upwards when the query result has a lot of data.
@ -191,7 +191,7 @@ func main() {
_taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver (driver-go minimum version 3.0.2).
Use `taosWS` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN.
Use `taosWS` as `driverName` and use a correct DSN as `dataSourceName` with the following parameters supported by the DSN.
* `writeTimeout` The timeout to send data via WebSocket.
* `readTimeout` The timeout to receive response data via WebSocket.

8
docs/en/08-client-libraries/06-rust.mdx
View File

@ -321,7 +321,7 @@ let rs = taos.query_with_req_id("select * from stable where tag1 is null", 1)?;
TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases.
Parameter binding details see [API Reference](#stmt-api)
Parameter binding details see [API Reference](#bind-interface)
<RustBind />
@ -578,11 +578,7 @@ Note that Rust asynchronous functions and an asynchronous runtime are required.
In addition, this structure is also the entry point for Parameter Binding and Line Protocol Interface. Please refer to the specific API descriptions for usage.
<p>
<a id="stmt-api" style={{color:'#141414'}}>
Bind Interface
</a>
</p>
### Bind Interface
Similar to the C interface, Rust provides the bind interface's wrapping. First, the [Taos][struct.taos] object creates a parameter binding object [Stmt] for an SQL statement.

2
docs/en/08-client-libraries/07-python.mdx
View File

@ -95,7 +95,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the
1. Install Python. The recent taospy package requires Python 3.6.2+. The earlier versions of taospy require Python 3.7+. The taos-ws-py package requires Python 3.7+. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it.
2. Install [pip](https://pypi.org/project/pip/). In most cases, the Python installer comes with the pip utility. If not, please refer to [pip documentation](https://pip.pypa.io/en/stable/installation/) to install it.
If you use a native connection, you will also need to [Install Client Driver](../#Install-Client-Driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI.
If you use a native connection, you will also need to [Install Client Driver](../#install-client-driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI.
### Install via pip

2
docs/en/08-client-libraries/08-node.mdx
View File

@ -58,7 +58,7 @@ Please refer to [version support list](../#version-support)
### Pre-installation preparation
- Install the Node.js development environment
- If you are using the REST client library, skip this step. However, if you use the native client library, please install the TDengine client driver. Please refer to [Install Client Driver](../#Install-Client-Driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS.
- If you are using the REST client library, skip this step. However, if you use the native client library, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS.
<Tabs defaultValue="Linux">
<TabItem value="Linux" label="Linux system installation dependencies">

2
docs/en/08-client-libraries/80-php.mdx
View File

@ -40,7 +40,7 @@ Because the version of TDengine client driver is tightly associated with that of
### Install TDengine Client Driver
Regarding how to install TDengine client driver please refer to [Install Client Driver](../#installation-steps)
Regarding how to install TDengine client driver please refer to [Install Client Driver](../#install-client-driver)
### Install php-tdengine

2
docs/en/14-reference/05-taosbenchmark.md
View File

@ -92,7 +92,7 @@ taosBenchmark -f <json file>
</details>
## Command-line argument in detail
## Command-line arguments in detail
- **-f/--file &lt;json file&gt;** :
specify the configuration file to use. This file includes All parameters. Users should not use this parameter with other parameters on the command-line. There is no default value.

4
docs/en/14-reference/14-taosKeeper.md
View File

@ -23,7 +23,7 @@ Methods of installing taosKeeper:
### Configuration
taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: [Command-line arguments](#command-line-arguments-in-detail), [environment variable](#environment-variable-in-detail) and [configuration file](#configuration-file-parameters-in-detail). The precedence of those is Command-line, environment variable and configuration file.
taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: Command-line arguments, environment variable and configuration file. The precedence of those is Command-line, environment variable and configuration file.
**Make sure that the TDengine cluster is running correctly before running taosKeeper.** Ensure that the monitoring service in TDengine has been started. At least the values of `monitor` and `monitorFqdn` need to be set in `taos.cfg`.
@ -32,7 +32,7 @@ monitor 1
monitorFqdn localhost # taoskeeper's FQDN
```
For more information, see [TDengine Monitoring Configuration](../config/#monitoring).
For more information, see [TDengine Monitoring Configuration](../config/#monitoring-parameters).
### Quick Launch

2
docs/en/20-third-party/03-telegraf.md vendored
View File

@ -73,6 +73,6 @@ Query OK, 3 row(s) in set (0.013269s)
- TDengine take influxdb format data and create unique ID for table names by the rule.
The user can configure `smlChildTableName` parameter to generate specified table names if he/she needs. And he/she also need to insert data with specified data format.
For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](../../reference/schemaless/#Schemaless-Line-Protocol)
For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](../../reference/schemaless/)
:::

13
include/common/tdataformat.h
View File

@ -95,6 +95,15 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
#define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL)
#define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)
#define tRowGetKey(_pRow, _pKey) \
do { \
(_pKey)->ts = (_pRow)->ts; \
(_pKey)->numOfPKs = 0; \
if ((_pRow)->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow), (_pKey)); \
} \
} while (0)
// SValueColumn ================================
typedef struct {
int8_t cmprAlg; // filled by caller
@ -125,8 +134,8 @@ void tRowDestroy(SRow *pRow);
int32_t tRowSort(SArray *aRowP);
int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const void *p1, const void *p2);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================

2
include/common/tglobal.h
View File

@ -258,7 +258,7 @@ extern bool tsExperimental;
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc);
bool tsc, bool isDumpCfg);
void taosCleanupCfg();
int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer);

4
include/common/tgrant.h
View File

@ -56,6 +56,10 @@ typedef enum {
TSDB_GRANT_VIEW,
TSDB_GRANT_MULTI_TIER,
TSDB_GRANT_BACKUP_RESTORE,
TSDB_GRANT_OBJECT_STORAGE,
TSDB_GRANT_ACTIVE_ACTIVE,
TSDB_GRANT_DUAL_REPLICA_HA,
TSDB_GRANT_DB_ENCRYPTION,
} EGrantType;
int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);

4
include/common/tmsg.h
View File

@ -329,6 +329,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_DB_ALIVE_STMT,
QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT,
QUERY_NODE_RESTORE_DNODE_STMT,
QUERY_NODE_RESTORE_QNODE_STMT,
QUERY_NODE_RESTORE_MNODE_STMT,
@ -2425,10 +2426,11 @@ int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistrib
void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq);
typedef struct {
int32_t useless;
int32_t reserved;
int32_t vgId;
int32_t sqlLen;
char* sql;
char db[TSDB_DB_FNAME_LEN];
} SBalanceVgroupLeaderReq;
int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);

2
include/libs/executor/storageapi.h
View File

@ -270,7 +270,7 @@ typedef struct SStoreMeta {
int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio);
void* (*storeGetIndexInfo)();
void* (*storeGetIndexInfo)(void *pVnode);
void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);

1
include/libs/nodes/cmdnodes.h
View File

@ -582,6 +582,7 @@ typedef struct SBalanceVgroupStmt {
typedef struct SBalanceVgroupLeaderStmt {
ENodeType type;
int32_t vgId;
char dbName[TSDB_DB_NAME_LEN];
} SBalanceVgroupLeaderStmt;
typedef struct SMergeVgroupStmt {

28
include/os/osSemaphore.h
View File

@ -33,6 +33,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#define tsem2_t tsem_t
#define tsem2_init tsem_init
#define tsem2_wait tsem_wait
#define tsem2_timewait tsem_timewait
#define tsem2_post tsem_post
#define tsem2_destroy tsem_destroy
#elif defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include <windows.h>
@ -44,6 +51,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#define tsem2_t tsem_t
#define tsem2_init tsem_init
#define tsem2_wait tsem_wait
#define tsem2_timewait tsem_timewait
#define tsem2_post tsem_post
#define tsem2_destroy tsem_destroy
#else
#define tsem_t sem_t
@ -53,6 +67,20 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
#define tsem_post sem_post
#define tsem_destroy sem_destroy
typedef struct tsem2_t {
TdThreadMutex mutex;
TdThreadCond cond;
TdThreadCondAttr attr;
int count;
} tsem2_t;
// #define tsem2_t sem_t
int tsem2_init(tsem2_t* sem, int pshared, unsigned int value);
int tsem2_wait(tsem2_t* sem);
int tsem2_timewait(tsem2_t* sem, int64_t milis);
int tsem2_post(tsem2_t* sem);
int tsem2_destroy(tsem2_t* sem);
#endif
#if defined(_TD_DARWIN_64)

4
include/os/osString.h
View File

@ -22,7 +22,11 @@ extern "C" {
typedef wchar_t TdWchar;
typedef int32_t TdUcs4;
#if !defined(DISALLOW_NCHAR_WITHOUT_ICONV) && defined(DARWIN)
#include "iconv.h"
#else
typedef void *iconv_t;
#endif
typedef enum { M2C = 0, C2M } ConvType;
// If the error is in a third-party library, place this header file under the third-party library header file.

1
include/os/osThread.h
View File

@ -222,6 +222,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const
int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr);
int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared);
int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr);
int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId);
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared);
int32_t taosThreadDetach(TdThread thread);
int32_t taosThreadEqual(TdThread t1, TdThread t2);

1
include/util/taoserror.h
View File

@ -327,6 +327,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)

4
include/util/tgeosctx.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include <geos_c.h>
#include <tpcre2.h>
typedef struct SGeosContext {
GEOSContextHandle_t handle;
@ -31,6 +32,9 @@ typedef struct SGeosContext {
GEOSWKBReader *WKBReader;
GEOSWKBWriter *WKBWriter;
pcre2_code *WKTRegex;
pcre2_match_data *WKTMatchData;
char errMsg[512];
} SGeosContext;

34
include/util/tpcre2.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_ULIT_PCRE2_H_
#define _TD_ULIT_PCRE2_H_
#define PCRE2_CODE_UNIT_WIDTH 8
#include "pcre2.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const char* pattern);
int32_t doRegExec(const char* pString, pcre2_code* pRegex, pcre2_match_data* pMatchData);
void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData);
#ifdef __cplusplus
}
#endif
#endif // _TD_UTIL_PAGEDBUF_H_

3
packaging/tools/makepkg.sh
View File

@ -101,6 +101,7 @@ else
${script_dir}/remove.sh \
${script_dir}/set_core.sh \
${script_dir}/startPre.sh \
${script_dir}/quick_deploy.sh \
${script_dir}/taosd-dump-cfg.gdb"
fi
@ -258,7 +259,7 @@ cp ${install_files} ${install_dir}
cp ${install_dir}/install.sh install_temp.sh
if [ "$verMode" == "cluster" ]; then
sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" install_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" install_temp.sh
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" install_temp.sh
cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
sed -i "s/emailName=\"taosdata.com\"/emailName=\"${cusDomain}\"/g" install_temp.sh

7
source/client/CMakeLists.txt
View File

@ -23,13 +23,6 @@ target_link_libraries(
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
)
if(TD_DARWIN_ARM64)
target_link_libraries(
taos
PRIVATE "-arch x86_64"
)
endif()
if(TD_WINDOWS)
INCLUDE_DIRECTORIES(jni/windows)
INCLUDE_DIRECTORIES(jni/windows/win32)

2
source/client/src/clientEnv.c
View File

@ -725,7 +725,7 @@ void taos_init_imp(void) {
return;
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true) != 0) {
tscInitRes = -1;
return;
}

2
source/client/src/clientHb.c
View File

@ -738,7 +738,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
_return:
releaseTscObj(connKey->tscRid);
if (code) {
tscError("hb got user auth info failed since %s", terrstr(code));
tscError("hb got user auth info failed since %s", tstrerror(code));
}
return code;

33
source/client/src/clientRawBlockWrite.c
View File

@ -30,6 +30,11 @@ static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3
static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
int8_t t, SColCmprWrapper* pColCmprRow) {
int8_t buildDefaultCompress = 0;
if (pColCmprRow->nCols <= 0) {
buildDefaultCompress = 1;
}
char* string = NULL;
cJSON* json = cJSON_CreateObject();
if (json == NULL) {
@ -70,13 +75,20 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
cJSON_AddItemToObject(column, "isPrimarykey", isPk);
cJSON_AddItemToArray(columns, column);
if (pColCmprRow == NULL || pColCmprRow->nCols <= i) {
if (pColCmprRow == NULL) {
continue;
}
uint32_t alg = 0;
if (buildDefaultCompress) {
alg = createDefaultColCmprByType(s->type);
} else {
SColCmpr* pColCmpr = pColCmprRow->pColCmpr + i;
const char* encode = columnEncodeStr(COMPRESS_L1_TYPE_U32(pColCmpr->alg));
const char* compress = columnCompressStr(COMPRESS_L2_TYPE_U32(pColCmpr->alg));
const char* level = columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pColCmpr->alg));
alg = pColCmpr->alg;
}
const char* encode = columnEncodeStr(COMPRESS_L1_TYPE_U32(alg));
const char* compress = columnCompressStr(COMPRESS_L2_TYPE_U32(alg));
const char* level = columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(alg));
cJSON* encodeJson = cJSON_CreateString(encode);
cJSON_AddItemToObject(column, "encode", encodeJson);
@ -767,14 +779,25 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
int8_t createDefaultCompress = 0;
SColCmprWrapper* p = &req.colCmpr;
if (p->nCols == 0) {
createDefaultCompress = 1;
}
// build create stable
pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
SSchema* pSchema = req.schemaRow.pSchema + i;
SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
strcpy(field.name, pSchema->name);
SColCmpr *p = &req.colCmpr.pColCmpr[i];
if (createDefaultCompress) {
field.compress = createDefaultColCmprByType(pSchema->type);
} else {
SColCmpr* p = &req.colCmpr.pColCmpr[i];
field.compress = p->alg;
}
taosArrayPush(pReq.pColumns, &field);
}
pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));

18
source/client/src/clientTmq.c
View File

@ -105,7 +105,7 @@ struct tmq_t {
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
tsem_t rspSem;
tsem2_t rspSem;
};
typedef struct SAskEpInfo {
@ -727,7 +727,7 @@ static void generateTimedTask(int64_t refId, int32_t type) {
*pTaskType = type;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
@ -742,7 +742,7 @@ void tmqReplayTask(void* param, void* tmrId) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) goto END;
tsem_post(&tmq->rspSem);
tsem2_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
END:
taosMemoryFree(param);
@ -1033,7 +1033,7 @@ void tmqFreeImpl(void* handle) {
}
taosFreeQall(tmq->qall);
tsem_destroy(&tmq->rspSem);
tsem2_destroy(&tmq->rspSem);
taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
taos_close_internal(tmq->pTscObj);
@ -1121,7 +1121,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->consumerId = tGenIdPI64();
// init semaphore
if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed")
@ -1132,7 +1132,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
tsem_destroy(&pTmq->rspSem);
tsem2_destroy(&pTmq->rspSem);
SET_ERROR_MSG_TMQ("init tscObj failed")
goto _failed;
}
@ -1434,7 +1434,7 @@ END:
taosReleaseRef(tmqMgmt.rsetId, refId);
FAIL:
if (tmq) tsem_post(&tmq->rspSem);
if (tmq) tsem2_post(&tmq->rspSem);
taosMemoryFree(pParam);
if (pMsg) taosMemoryFreeClear(pMsg->pData);
if (pMsg) taosMemoryFreeClear(pMsg->pEpSet);
@ -2132,10 +2132,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, tmq->epoch, startTime, currentTime);
return NULL;
}
tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
tsem2_timewait(&tmq->rspSem, (timeout - elapsedTime));
} else {
// use tsem_timewait instead of tsem_wait to avoid unexpected stuck
tsem_timewait(&tmq->rspSem, 1000);
tsem2_timewait(&tmq->rspSem, 1000);
}
}
}

10
source/common/src/cos.c
View File

@ -27,6 +27,8 @@ static S3UriStyle uriStyleG = S3UriStylePath;
static int retriesG = 5;
static int timeoutMsG = 0;
extern int8_t tsS3Oss;
int32_t s3Begin() {
S3Status status;
const char *hostname = tsS3Hostname;
@ -42,6 +44,9 @@ int32_t s3Begin() {
}
protocolG = !tsS3Https;
if (tsS3Oss) {
uriStyleG = S3UriStyleVirtualHost;
}
return 0;
}
@ -58,6 +63,11 @@ static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n");
goto _exit;
}
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");

7
source/common/src/tcol.c
View File

@ -250,7 +250,7 @@ bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OP
strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnEncode(encode);
return checkColumnEncode(encode) && validColEncode(type, columnEncodeVal(encode));
}
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(compress)) return true;
@ -267,7 +267,8 @@ bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRES
strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnCompress(compress);
return checkColumnCompress(compress) && validColCompress(type, columnCompressVal(compress));
}
bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(level)) return true;
@ -288,7 +289,7 @@ bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTI
strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnLevel(level);
return checkColumnLevel(level) && validColCompressLevel(type, columnLevelVal(level));
}
void setColEncode(uint32_t* compress, uint8_t l1) {

19
source/common/src/tdatablock.c
View File

@ -2411,13 +2411,24 @@ _end:
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) {
char tmp[TSDB_TABLE_NAME_LEN] = {0};
if (stbName == NULL) {
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%" PRIu64, groupId);
} else {
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%" PRIu64, stbName, groupId);
if (stbName == NULL){
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
}else{
int32_t i = strlen(stbName) - 1;
for(; i >= 0; i--){
if (stbName[i] == '.'){
break;
}
}
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId);
}
ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end
strcat(ctbName, tmp);
for(int i = 0; i < strlen(ctbName); i++){
if(ctbName[i] == '.'){
ctbName[i] = '_';
}
}
}
// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.

8
source/common/src/tdataformat.c
View File

@ -1183,8 +1183,7 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in
}
}
void tRowGetKey(SRow *row, SRowKey *key) {
key->ts = row->ts;
void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->numOfPKs = row->numOfPKs;
if (key->numOfPKs == 0) {
@ -1283,10 +1282,7 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) {
// NOTE:
// set key->numOfPKs to 0 as the smallest key with ts
// set key->numOfPKs to (TD_MAX_PK_COLS + 1) as the largest key with ts
int32_t tRowKeyCompare(const void *p1, const void *p2) {
SRowKey *key1 = (SRowKey *)p1;
SRowKey *key2 = (SRowKey *)p2;
FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
if (key1->ts < key2->ts) {
return -1;
} else if (key1->ts > key2->ts) {

38
source/common/src/tglobal.c
View File

@ -45,8 +45,8 @@ char tsEncryptAlgorithm[16] = {0};
char tsEncryptScope[100] = {0};
EEncryptAlgor tsiEncryptAlgorithm = 0;
EEncryptScope tsiEncryptScope = 0;
//char tsAuthCode[500] = {0};
//char tsEncryptKey[17] = {0};
// char tsAuthCode[500] = {0};
// char tsEncryptKey[17] = {0};
char tsEncryptKey[17] = {0};
// common
@ -240,8 +240,8 @@ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
// lossy compress 7
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
// can close lossy compress.
char tsLossyColumns[32] = "float|double"; // "float|double" means all float and double columns can be lossy compressed.
// set empty can close lossy compress.
// below option can take effect when tsLossyColumns not empty
float tsFPrecision = 1E-8; // float column precision
double tsDPrecision = 1E-16; // double column precision
@ -296,6 +296,8 @@ char tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
char tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
char tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
int8_t tsS3Enabled = false;
int8_t tsS3EnabledCfg = false;
int8_t tsS3Oss = false;
int8_t tsS3StreamEnabled = false;
int8_t tsS3Https = true;
@ -366,10 +368,15 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
}
}
char *oss = strstr(tsS3Endpoint, "aliyuncs.");
if (oss) {
tsS3Oss = true;
}
if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3)
#ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
tsS3EnabledCfg = true;
#endif
tsS3StreamEnabled = true;
#endif
@ -563,11 +570,12 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT,
CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
0)
if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, CFG_SCOPE_CLIENT,
CFG_DYN_CLIENT) != 0)
@ -618,11 +626,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfSupportVnodes = tsNumOfCores * 2;
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
//if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
// if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
@ -749,8 +758,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
@ -807,7 +815,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -1181,7 +1188,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
//tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
// tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
@ -1268,7 +1275,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
@ -1388,7 +1394,7 @@ static int32_t taosCheckGlobalCfg() {
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) {
bool tsc, bool isDumpCfg) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
@ -1435,7 +1441,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false);
if(isDumpCfg) cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) {
return -1;

4
source/common/src/tmisce.c
View File

@ -228,6 +228,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
SConfigIter* pIter = cfgCreateIter(pConf);
cfgLock(pConf);
while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol;
@ -253,6 +255,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
numOfRows++;
}
cfgUnLock(pConf);
pBlock->info.rows = numOfRows;
cfgDestroyIter(pIter);

11
source/common/src/tmsg.c
View File

@ -5962,9 +5962,11 @@ int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgr
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
if (tEncodeI32(&encoder, pReq->reserved) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
ENCODESQL();
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -5977,12 +5979,15 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
}
DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);

40
source/common/test/dataformatTest.cpp
View File

@ -20,6 +20,7 @@
#include <tglobal.h>
#include <tmsg.h>
#include <iostream>
#include <tdatablock.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -475,6 +476,45 @@ TEST(testCase, AllNormTest) {
taosMemoryFree(pTSchema);
}
TEST(testCase, StreamAllNormTest) {
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(NULL, ctbName, groupId);
ASSERT_STREQ("_12345", ctbName);
}
TEST(testCase, StreamWithStbName) {
char stbName[] = "1.table.stb";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("_stb_12345", ctbName);
}
TEST(testCase, StreamWithoutDotInStbName) {
char stbName[] = "table";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("_table_12345", ctbName);
}
TEST(testCase, StreamWithoutDotInStbName2) {
char stbName[] = "";
char ctbName[TSDB_TABLE_NAME_LEN] = {0};
uint64_t groupId = 12345;
buildCtbNameAddGroupId(stbName, ctbName, groupId);
ASSERT_STREQ("__12345", ctbName);
}
#if 1
TEST(testCase, NoneTest) {
const static int nCols = 14;

12
source/dnode/mgmt/exe/dmMain.c
View File

@ -205,11 +205,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if(i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("encrypt key is too short, it should be great or equal to %d\n", ENCRYPT_KEY_LEN_MIN);
printf("Error: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1;
}
if (len > ENCRYPT_KEY_LEN) {
printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
printf("Error: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
@ -325,7 +325,7 @@ int main(int argc, char const *argv[]) {
}
if (dmParseArgs(argc, argv) != 0) {
printf("failed to start since parse args error\n");
//printf("failed to start since parse args error\n");
taosCleanupArgs();
return -1;
}
@ -380,7 +380,11 @@ int mainWindows(int argc, char **argv) {
dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
bool isDumpCfg = true;
if(global.generateCode) {
isDumpCfg = false;
}
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0, isDumpCfg) != 0) {
dError("failed to start since read config error");
taosCloseLog();
taosCleanupArgs();

4
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -37,7 +37,7 @@ static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
dTrace("msg:%p, get from snode-write queue", pMsg);
int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL);
if (code < 0) {
dGError("snd, msg:%p failed to process write since %s", pMsg, terrstr(code));
dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code));
if (pMsg->info.handle != NULL) {
tmsgSendRsp(pMsg);
}
@ -58,7 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
dTrace("msg:%p, get from snode-stream queue", pMsg);
int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
if (code < 0) {
dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
smSendRsp(pMsg, terrno);
}

10
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -316,7 +316,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
code = terrno;
code = terrno != 0 ? terrno : -1;
goto _OVER;
}
@ -344,21 +344,21 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr());
vmReleaseVnode(pMgmt, pVnode);
tFreeSCreateVnodeReq(&req);
code = terrno;
code = terrno != 0 ? terrno : -1;
return code;
}
SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, true);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
code = terrno;
code = terrno != 0 ? terrno : -1;
goto _OVER;
}
code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
if (code != 0) {
dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
code = terrno;
code = terrno != 0 ? terrno : code;
goto _OVER;
}
@ -379,7 +379,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) {
code = terrno;
code = terrno != 0 ? terrno : code;
goto _OVER;
}

8
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -103,7 +103,7 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
terrstr(code));
tstrerror(code));
vmSendRsp(pMsg, code);
}
@ -201,7 +201,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
case QUERY_QUEUE:
code = vnodePreprocessQueryMsg(pVnode->pImpl, pMsg);
if (code) {
dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, terrstr(code));
dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, tstrerror(code));
} else {
dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pQueryQ, pMsg);
@ -219,13 +219,13 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
if (!vmDataSpaceSufficient(pVnode)) {
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
code = terrno;
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
terrno = TSDB_CODE_VND_NO_WRITE_AUTH;
code = terrno;
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {

2
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -17,6 +17,7 @@
#include "dmMgmt.h"
#include "audit.h"
#include "libs/function/tudf.h"
#include "tgrant.h"
#define DM_INIT_AUDIT() \
do { \
@ -150,6 +151,7 @@ static bool dmCheckDataDirVersion() {
extern int32_t s3Begin();
extern void s3End();
extern int8_t tsS3Enabled;
#endif

8
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -18,12 +18,10 @@
#include "dmNodes.h"
#include "index.h"
#include "qworker.h"
#include "tstream.h"
#ifdef TD_TSZ
#include "tcompression.h"
#include "tglobal.h"
#include "tgrant.h"
#endif
#include "tstream.h"
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@ -48,10 +46,8 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER;
}
#ifdef TD_TSZ
// compress module init
tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
#endif
pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
@ -120,10 +116,8 @@ void dmCleanupDnode(SDnode *pDnode) {
indexCleanup();
taosConvDestroy();
#ifdef TD_TSZ
// compress destroy
tsCompressExit();
#endif
dDebug("dnode is closed, ptr:%p", pDnode);
}

17
source/dnode/mnode/impl/src/mndDb.c
View File

@ -821,6 +821,10 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
goto _exit;
}
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8 " in mnode leader since it's not loaded", pReq->db,
@ -903,6 +907,13 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
goto _OVER;
}
if (createReq.replications == 2) {
if ((terrno = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
code = terrno;
goto _OVER;
}
}
if ((code = mndCheckDbEncryptKey(pMnode, &createReq)) != 0) {
terrno = code;
goto _OVER;
@ -1163,6 +1174,12 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
goto _OVER;
}
if (alterReq.replications == 2) {
if ((code = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
goto _OVER;
}
}
int32_t numOfTopics = 0;
if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) {
goto _OVER;

3
source/dnode/vnode/CMakeLists.txt
View File

@ -1,6 +1,9 @@
# vnode
add_subdirectory(src/tqCommon)
add_library(vnode STATIC "")
if(${TD_DARWIN})
target_compile_options(vnode PRIVATE -Wno-error=single-bit-bitfield-constant-conversion)
endif(${TD_DARWIN})
set(
VNODE_SOURCE_FILES
"src/vnd/vnodeOpen.c"

54
source/dnode/vnode/src/inc/tsdb.h
View File

@ -120,12 +120,32 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
#define tsdbRowFromBlockData(BLOCKDATA, IROW) \
((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
#define TSDBROW_INIT_KEY(_ROW, _KEY) \
{ \
if ((_ROW)->type == TSDBROW_ROW_FMT) { \
_KEY.version = (_ROW)->version; \
_KEY.ts = (_ROW)->pTSRow->ts; \
} else { \
_KEY.version = (_ROW)->pBlockData->aVersion[(_ROW)->iRow]; \
_KEY.ts = (_ROW)->pBlockData->aTSKEY[(_ROW)->iRow]; \
} \
}
#define tColRowGetKey(_pBlock, _irow, _key) \
{ \
(_key)->ts = (_pBlock)->aTSKEY[(_irow)]; \
(_key)->numOfPKs = 0; \
if ((_pBlock)->nColData > 0) { \
tColRowGetPrimaryKey((_pBlock), (_irow), (_key)); \
} \
}
void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
int32_t tsdbRowCompare(const void *p1, const void *p2);
int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
// STSDBRowIter
@ -902,9 +922,16 @@ typedef struct {
SColVal colVal;
} SLastCol;
typedef struct {
int8_t lflag;
STsdbRowKey tsdbRowKey;
SColVal colVal;
} SLastUpdateCtx;
int32_t tsdbOpenCache(STsdb *pTsdb);
void tsdbCloseCache(STsdb *pTsdb);
int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *row);
int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow);
int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData);
int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
@ -946,7 +973,28 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
// #define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
if (pIter == NULL) return NULL;
if (pIter->pRow) {
return pIter->pRow;
}
if (pIter->backward) {
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return NULL;
}
} else {
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return NULL;
}
}
pIter->pRow = &pIter->row;
pIter->row = pIter->pNode->row;
return pIter->pRow;
}
typedef struct {
int64_t suid;

342
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -16,6 +16,7 @@
#include "functionMgt.h"
#include "tsdb.h"
#include "tsdbDataFileRW.h"
#include "tsdbIter.h"
#include "tsdbReadUtil.h"
#include "vnd.h"
@ -359,7 +360,10 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) {
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData;
pLastCol->colVal.value.pData = NULL;
if (pLastCol->colVal.value.nData > 0) {
pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
}
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else {
pLastCol->colVal.value.val = pLastColV0->colVal.value.val;
@ -372,7 +376,7 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
return NULL;
}
SLastCol* pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
SLastCol *pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
if (NULL == pLastCol) {
return NULL;
}
@ -401,10 +405,13 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
offset += sizeof(SValue);
if (IS_VAR_DATA_TYPE(pLastCol->rowKey.pks[i].type)) {
pLastCol->rowKey.pks[i].pData = NULL;
if (pLastCol->rowKey.pks[i].nData > 0) {
pLastCol->rowKey.pks[i].pData = (uint8_t *)value + offset;
offset += pLastCol->rowKey.pks[i].nData;
}
}
}
if (offset > size) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
@ -940,10 +947,37 @@ typedef struct {
SLastKey key;
} SIdxKey;
static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
uint8_t *pVal = NULL;
static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) {
uint8_t *pFree = NULL;
int nData = 0;
if (IS_VAR_DATA_TYPE(pOld->type)) {
pFree = pOld->pData;
nData = pOld->nData;
}
*pOld = *pNew;
if (IS_VAR_DATA_TYPE(pNew->type)) {
if (nData < pNew->nData) {
pOld->pData = taosMemoryCalloc(1, pNew->nData);
} else {
pOld->pData = pFree;
pFree = NULL;
}
if (pNew->nData) {
memcpy(pOld->pData, pNew->pData, pNew->nData);
} else {
pFree = pOld->pData;
pOld->pData = NULL;
}
}
taosMemoryFreeClear(pFree);
return 0;
}
static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
// update rowkey
pLastCol->rowKey.ts = pRowKey->ts;
pLastCol->rowKey.numOfPKs = pRowKey->numOfPKs;
@ -951,87 +985,43 @@ static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal
SValue *pPKValue = &pLastCol->rowKey.pks[i];
SValue *pNewPKValue = &pRowKey->pks[i];
if (IS_VAR_DATA_TYPE(pPKValue->type)) {
pVal = pPKValue->pData;
nData = pPKValue->nData;
}
*pPKValue = *pNewPKValue;
if (IS_VAR_DATA_TYPE(pPKValue->type)) {
if (nData < pPKValue->nData) {
taosMemoryFree(pVal);
pPKValue->pData = taosMemoryCalloc(1, pNewPKValue->nData);
} else {
pPKValue->pData = pVal;
}
if (pNewPKValue->nData) {
memcpy(pPKValue->pData, pNewPKValue->pData, pNewPKValue->nData);
}
}
(void)tsdbCacheUpdateValue(pPKValue, pNewPKValue);
}
// update colval
if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
nData = pLastCol->colVal.value.nData;
pVal = pLastCol->colVal.value.pData;
}
pLastCol->colVal = *pColVal;
if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
if (nData < pColVal->value.nData) {
taosMemoryFree(pVal);
pLastCol->colVal.value.pData = taosMemoryCalloc(1, pColVal->value.nData);
} else {
pLastCol->colVal.value.pData = pVal;
}
if (pColVal->value.nData) {
memcpy(pLastCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
}
pLastCol->colVal.cid = pColVal->cid;
pLastCol->colVal.flag = pColVal->flag;
(void)tsdbCacheUpdateValue(&pLastCol->colVal.value, &pColVal->value);
if (!pLastCol->dirty) {
pLastCol->dirty = 1;
}
}
int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow) {
static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) {
if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) {
return 0;
}
int32_t code = 0;
// 1, fetch schema
STSchema *pTSchema = NULL;
int32_t sver = TSDBROW_SVERSION(pRow);
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return -1;
}
// 2, iterate col values into array
SArray *aColVal = taosArrayInit(32, sizeof(SColVal));
STSDBRowIter iter = {0};
tsdbRowIterOpen(&iter, pRow, pTSchema);
for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
taosArrayPush(aColVal, pColVal);
}
tsdbRowClose(&iter);
// 3, build keys & multi get from rocks
int num_keys = TARRAY_SIZE(aColVal);
int num_keys = TARRAY_SIZE(updCtxArray);
SArray *remainCols = NULL;
SLRUCache *pCache = pTsdb->lruCache;
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(pRow, &tsdbRowKey);
SRowKey *pRowKey = &tsdbRowKey.key;
taosThreadMutexLock(&pTsdb->lruMutex);
for (int i = 0; i < num_keys; ++i) {
SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i);
int16_t cid = pColVal->cid;
SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
SLastKey *key = &(SLastKey){.lflag = LFLAG_LAST_ROW, .uid = uid, .cid = cid};
int8_t lflag = updCtx->lflag;
SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
SColVal *pColVal = &updCtx->colVal;
if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) {
continue;
}
SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid};
size_t klen = ROCKS_KEY_LEN;
LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
if (h) {
@ -1047,23 +1037,6 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
}
taosArrayPush(remainCols, &(SIdxKey){i, *key});
}
if (COL_VAL_IS_VALUE(pColVal)) {
key->lflag = LFLAG_LAST;
LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
if (h) {
SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
if (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1) {
tsdbCacheUpdateLastCol(pLastCol, pRowKey, pColVal);
}
taosLRUCacheRelease(pCache, h, false);
} else {
if (!remainCols) {
remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey));
}
taosArrayPush(remainCols, &(SIdxKey){i, *key});
}
}
}
if (remainCols) {
@ -1091,12 +1064,17 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
for (int i = 0; i < num_keys; ++i) {
SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i];
SColVal *pColVal = (SColVal *)TARRAY_DATA(aColVal) + idxKey->idx;
// SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, idxKey->idx);
SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
SColVal *pColVal = &updCtx->colVal;
SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]);
SLastCol *PToFree = pLastCol;
if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
continue;
}
if (IS_LAST_ROW_KEY(idxKey->key)) {
int32_t cmp_res = 1;
if (pLastCol) {
@ -1142,48 +1120,6 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
taosMemoryFree(value);
}
} else {
if (COL_VAL_IS_VALUE(pColVal)) {
if (NULL == pLastCol || (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1)) {
char *value = NULL;
size_t vlen = 0;
SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
tsdbCacheSerialize(&lastColTmp, &value, &vlen);
taosThreadMutexLock(&pTsdb->rCache.rMutex);
rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen);
taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
pLastCol = &lastColTmp;
SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
*pTmpLastCol = *pLastCol;
pLastCol = pTmpLastCol;
size_t charge = sizeof(*pLastCol);
for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
SValue *pValue = &pLastCol->rowKey.pks[i];
if (IS_VAR_DATA_TYPE(pValue->type)) {
reallocVarDataVal(pValue);
charge += pValue->nData;
}
}
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
reallocVarData(&pLastCol->colVal);
charge += pLastCol->colVal.value.nData;
}
LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge,
tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
if (status != TAOS_LRU_STATUS_OK) {
code = -1;
}
taosMemoryFree(value);
}
}
}
taosMemoryFreeClear(PToFree);
@ -1203,11 +1139,152 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
taosThreadMutexUnlock(&pTsdb->lruMutex);
_exit:
taosArrayDestroy(aColVal);
taosMemoryFree(pTSchema);
return code;
}
int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow,
SRow **aRow) {
int32_t code = 0;
// 1. prepare last
TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version};
STSchema *pTSchema = NULL;
int32_t sver = TSDBROW_SVERSION(&lRow);
SArray *ctxArray = NULL;
SSHashObj *iColHash = NULL;
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto _exit;
}
TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
int32_t nCol = pTSchema->numOfCols;
ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx));
iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
// 1. prepare by lrow
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&lRow, &tsdbRowKey);
STSDBRowIter iter = {0};
tsdbRowIterOpen(&iter, &lRow, pTSchema);
int32_t iCol = 0;
for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) {
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
taosArrayPush(ctxArray, &updateCtx);
if (!COL_VAL_IS_VALUE(pColVal)) {
tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0);
continue;
}
updateCtx.lflag = LFLAG_LAST;
taosArrayPush(ctxArray, &updateCtx);
}
tsdbRowClose(&iter);
// 2. prepare by the other rows
for (int32_t iRow = nRow - 2; iRow >= 0; --iRow) {
if (tSimpleHashGetSize(iColHash) == 0) {
break;
}
tRow.pTSRow = aRow[iRow];
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&tRow, &tsdbRowKey);
void *pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(iColHash, pIte, &iter)) != NULL) {
int32_t iCol = ((int32_t *)pIte)[0];
SColVal colVal = COL_VAL_NONE(0, 0);
tsdbRowGetColVal(&tRow, pTSchema, iCol, &colVal);
if (COL_VAL_IS_VALUE(&colVal)) {
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
taosArrayPush(ctxArray, &updateCtx);
tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter);
}
}
}
// 3. do update
tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
_exit:
taosMemoryFreeClear(pTSchema);
taosArrayDestroy(ctxArray);
tSimpleHashCleanup(iColHash);
return code;
}
int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) {
int32_t code = 0;
TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1);
STSchema *pTSchema = NULL;
int32_t sver = TSDBROW_SVERSION(&lRow);
SArray *ctxArray = NULL;
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto _exit;
}
ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx));
// 1. prepare last
TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
SColData *pColData = &pBlockData->aColData[iColData];
if ((pColData->flag & HAS_VALUE) != HAS_VALUE) {
continue;
}
for (tRow.iRow = pBlockData->nRow - 1; tRow.iRow >= 0; --tRow.iRow) {
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&tRow, &tsdbRowKey);
uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow);
if (colType == 2) {
SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type);
tColDataGetValue(pColData, tRow.iRow, &colVal);
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
taosArrayPush(ctxArray, &updateCtx);
break;
}
}
}
// 2. prepare last row
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&lRow, &tsdbRowKey);
STSDBRowIter iter = {0};
tsdbRowIterOpen(&iter, &lRow, pTSchema);
for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
taosArrayPush(ctxArray, &updateCtx);
}
tsdbRowClose(&iter);
// 3. do update
tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
_exit:
taosMemoryFreeClear(pTSchema);
taosArrayDestroy(ctxArray);
return 0;
}
static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
int nCols, int16_t *slotIds);
@ -1481,7 +1558,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
taosArraySet(pLastArray, idxKey->idx, pLastCol);
// taosArrayRemove(remainCols, i);
if (!pTmpColArray) {
if (/*!pTmpColArray*/ lastTmpIndexArray && !lastTmpColArray) {
continue;
}
if (/*!pTmpColArray*/ lastrowTmpIndexArray && lastrowTmpColArray) {
continue;
}

34
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -629,14 +629,12 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
SMemSkipListNode *pos[SL_MAX_LEVEL];
TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
STsdbRowKey key;
TSDBROW lRow; // last row
// first row
tsdbRowGetKey(&tRow, &key);
tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit;
pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
lRow = tRow;
// remain row
++tRow.iRow;
@ -653,7 +651,6 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
}
if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit;
lRow = tRow;
++tRow.iRow;
}
@ -664,7 +661,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData);
}
// SMemTable
@ -688,7 +685,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
SMemSkipListNode *pos[SL_MAX_LEVEL];
TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
int32_t iRow = 0;
TSDBROW lRow;
// backward put first data
tRow.pTSRow = aRow[iRow++];
@ -696,7 +692,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0);
if (code) goto _exit;
lRow = tRow;
pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
@ -717,8 +712,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1);
if (code) goto _exit;
lRow = tRow;
iRow++;
}
}
@ -727,7 +720,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
pTbData->maxKey = key.key.ts;
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow);
}
// SMemTable
@ -806,26 +799,3 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
_exit:
return aTbDataP;
}
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
if (pIter == NULL) return NULL;
if (pIter->pRow) {
return pIter->pRow;
}
if (pIter->backward) {
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return NULL;
}
} else {
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return NULL;
}
}
pIter->pRow = &pIter->row;
pIter->row = pIter->pNode->row;
return pIter->pRow;
}

2
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -828,7 +828,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTre
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));
SRowKey rkey1, rkey2;
SRowKey rkey1 = {0}, rkey2 = {0};
tRowGetKeyEx(&pIter1->rInfo.row, &rkey1);
tRowGetKeyEx(&pIter2->rInfo.row, &rkey2);

89
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -24,6 +24,16 @@
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))
#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey) \
do { \
(_pKey)->ts = (_pBlock)->aTSKEY[(_irow)]; \
(_pKey)->numOfPKs = 0; \
if ((_slotId) != -1) { \
tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \
} \
} while (0)
#define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey))
typedef struct {
bool overlapWithNeighborBlock;
@ -75,11 +85,9 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey
static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);
static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader);
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
if (p2 == NULL) {
return 1;
}
@ -101,13 +109,7 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
}
}
static void tColRowGetKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
pKey->ts = pBlock->aTSKEY[irow];
if (slotId == -1) {
pKey->numOfPKs = 0;
return;
}
static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
SColData* pColData = &pBlock->aColData[slotId];
SColVal cv;
tColDataGetValue(pColData, irow, &cv);
@ -131,13 +133,7 @@ static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) {
return n;
}
static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
pKey->ts = pRow->ts;
pKey->numOfPKs = pRow->numOfPKs;
if (pKey->numOfPKs == 0) {
return;
}
static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
@ -1735,7 +1731,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
}
SRowKey k;
SRowKey k = {0};
tRowGetKeyEx(pRow, &k);
STSchema* pSchema = NULL;
@ -1939,7 +1935,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
pfKey = NULL;
}
SRowKey k, ik;
SRowKey k = {0}, ik = {0};
tRowGetKeyEx(pRow, &k);
tRowGetKeyEx(piRow, &ik);
@ -2086,7 +2082,7 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
}
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
SRowKey rowKey;
SRowKey rowKey = {0};
while (1) {
TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader);
@ -3559,7 +3555,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
return false;
}
TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
if (!pIter->hasVal) {
return NULL;
}
@ -3567,7 +3563,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
int32_t order = pReader->info.order;
TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter);
TSDBKEY key = TSDBROW_KEY(pRow);
TSDBKEY key;
TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false;
return NULL;
@ -3593,7 +3590,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
pRow = tsdbTbDataIterGet(pIter->iter);
key = TSDBROW_KEY(pRow);
TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false;
return NULL;
@ -3632,11 +3629,13 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArra
break;
}
if (pCurKey->numOfPKs > 0) {
SRowKey nextKey = {0};
tRowGetKeyEx(pRow, &nextKey);
if (pkCompEx(pCurKey, &nextKey) != 0) {
break;
}
}
STSchema* pTSchema = NULL;
if (pRow->type == TSDBROW_ROW_FMT) {
@ -3785,15 +3784,17 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
return TSDB_CODE_SUCCESS;
}
if (pKey->numOfPKs > 0) {
SRowKey nextRowKey = {0};
tRowGetKeyEx(pNextRow, &nextRowKey);
if (pKey->numOfPKs > 0 && pkCompEx(pKey, &nextRowKey) != 0) {
if (pkCompEx(pKey, &nextRowKey) != 0) {
*pResRow = current;
*freeTSRow = false;
return TSDB_CODE_SUCCESS;
}
}
}
}
terrno = 0;
int32_t code = 0;
@ -3801,7 +3802,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
// start to merge duplicated rows
STSchema* pTSchema = NULL;
if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory
pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(&current), pReader, uid);
pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid);
if (pTSchema == NULL) {
return terrno;
}
@ -3814,7 +3815,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
STSchema* pTSchema1 = NULL;
if (pNextRow->type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory
pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid);
pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid);
if (pTSchema1 == NULL) {
return terrno;
}
@ -3926,24 +3927,35 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
// todo refactor
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
if (piter->hasVal) {
TSDBKEY k = TSDBROW_KEY(pRow);
if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
tRowGetKeyEx(pRow, &rowKey);
if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) {
pRow = NULL;
}
}
if (piiter->hasVal) {
TSDBKEY k = TSDBROW_KEY(piRow);
if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
tRowGetKeyEx(piRow, &irowKey);
if ((irowKey.ts >= endKey && asc) || (irowKey.ts <= endKey && !asc)) {
piRow = NULL;
}
}
if (piter->hasVal && piiter->hasVal && pRow != NULL && piRow != NULL) {
tRowGetKeyEx(pRow, &rowKey);
tRowGetKeyEx(piRow, &irowKey);
if (pRow != NULL && piRow != NULL) {
int32_t code = TSDB_CODE_SUCCESS;
if (rowKey.numOfPKs == 0) {
if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) { // ik.ts < k.ts
code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
} else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) {
code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
} else { // ik.ts == k.ts
*freeTSRow = true;
pResRow->type = TSDBROW_ROW_FMT;
code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
} else {
int32_t ret = pkCompEx(&rowKey, &irowKey);
if (ret != 0) {
if ((ret > 0 && asc) || (ret < 0 && (!asc))) { // ik.ts < k.ts
@ -3959,17 +3971,16 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
return code;
}
}
}
return code;
}
if (piter->hasVal && pRow != NULL) {
tRowGetKeyEx(pRow, &rowKey);
return doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
}
if (piiter->hasVal && piRow != NULL) {
tRowGetKeyEx(piRow, &irowKey);
return doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
}
@ -4095,7 +4106,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
if (row.type == TSDBROW_ROW_FMT) {
code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
if (code == TSDB_CODE_SUCCESS) {
tRowGetKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts;
pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs;
if (row.pTSRow->numOfPKs > 0) {
tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
}
}
if (freeTSRow) {

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -212,7 +212,7 @@ static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader
int32_t numOfPks = pReader->suppInfo.numOfPks;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int8_t type = pReader->suppInfo.pk.type;
int8_t bytes = pReader->suppInfo.pk.bytes;
int32_t bytes = pReader->suppInfo.pk.bytes;
SRowKey* pRowKey = &pScanInfo->lastProcKey;
if (asc) {
@ -1056,7 +1056,7 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) {
const SSttKeyRange* px1 = p1;
const SSttKeyRange* px2 = p2;
int32_t ret = tRowKeyCompare(&px1, px2);
int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);
return ret;
}

14
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -39,13 +39,17 @@ extern "C" {
} while (0);
#define tRowGetKeyEx(_pRow, _pKey) \
do { \
{ \
if ((_pRow)->type == TSDBROW_ROW_FMT) { \
tRowGetKey((_pRow)->pTSRow, (_pKey)); \
} else { \
tColRowGetKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
(_pKey)->ts = (_pRow)->pTSRow->ts; \
if ((_pRow)->pTSRow->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow)->pTSRow, (_pKey)); \
} \
} while (0)
} else { \
(_pKey)->ts = (_pRow)->pBlockData->aTSKEY[(_pRow)->iRow]; \
tColRowGetPrimaryKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
} \
}
typedef enum {
READER_STATUS_SUSPEND = 0x1,

10
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -757,6 +757,16 @@ _exit:
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
int32_t code = 0;
extern int8_t tsS3EnabledCfg;
int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE);
if (expired && tsS3Enabled) {
tsdbWarn("s3 grant expired: %d", expired);
tsS3Enabled = false;
} else if (!expired && tsS3EnabledCfg) {
tsS3Enabled = true;
}
if (!tsS3Enabled) {
return code;
}

13
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -601,11 +601,14 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
STColumn *pTColumn = &pTSchema->columns[iCol];
SValue value;
ASSERT(iCol > 0);
if (pRow->type == TSDBROW_ROW_FMT) {
tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal);
} else if (pRow->type == TSDBROW_COL_FMT) {
if (iCol == 0) {
*pColVal =
COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID,
((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pRow->pBlockData->aTSKEY[pRow->iRow]}));
} else {
SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
if (pColData) {
@ -613,6 +616,7 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
} else {
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
}
}
} else {
ASSERT(0);
}
@ -628,10 +632,7 @@ void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key) {
}
}
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
key->ts = pBlock->aTSKEY[irow];
key->numOfPKs = 0;
void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
for (int32_t i = 0; i < pBlock->nColData; i++) {
SColData *pColData = &pBlock->aColData[i];
if (pColData->cflag & COL_IS_KEY) {

22
source/libs/catalog/src/ctgCache.c
View File

@ -3226,14 +3226,24 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx
// get tb cache
pName = taosArrayGet(pList, i);
pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
if (!pTbCache || !pTbCache->pMeta) {
if (!pTbCache) {
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
CTG_LOCK(CTG_READ, &pTbCache->metaLock);
if (!pTbCache->pMeta) {
CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
taosHashRelease(dbCache->tbCache, pTbCache);
continue;
}
uint64_t suid = pTbCache->pMeta->suid;
int8_t tbType = pTbCache->pMeta->tableType;
CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
taosHashRelease(dbCache->tbCache, pTbCache);
SName tsmaSourceTbName = *pName;
@ -3445,16 +3455,17 @@ static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const S
int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname);
int32_t code = 0;
SCtgDBCache* pDbCache = NULL;
SCtgCacheOperation* pOp = NULL;
SCtgDBCache *pDbCache = NULL;
SCtgCacheOperation *pOp = NULL;
char dbFName[TSDB_DB_FNAME_LEN];
SCtgTSMACache *pCtgCache = NULL;
tNameGetFullDbName(pName, dbFName);
CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache));
if (NULL == pDbCache || !pDbCache->tsmaCache) {
goto _return;
}
SCtgTSMACache *pCtgCache = taosHashGet(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
if (!pCtgCache) goto _return;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
@ -3471,10 +3482,11 @@ int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
CTG_ERR_JRET(ctgEnqueue(pCtg, pOp));
taosHashRelease(pDbCache->tsmaCache, pCtgCache);
return TSDB_CODE_SUCCESS;
_return:
if (pCtgCache) taosHashRelease(pDbCache->tsmaCache, pCtgCache);
if (pOp) {
taosMemoryFree(pOp->data);
taosMemoryFree(pOp);

3
source/libs/executor/CMakeLists.txt
View File

@ -1,6 +1,9 @@
aux_source_directory(src EXECUTOR_SRC)
add_library(executor STATIC ${EXECUTOR_SRC})
if(${TD_DARWIN})
target_compile_options(executor PRIVATE -Wno-error=deprecated-non-prototype)
endif(${TD_DARWIN})
target_link_libraries(executor
PRIVATE os util common function parser planner qcom scalar nodes index wal tdb geometry

10
source/libs/executor/inc/executorInt.h
View File

@ -446,7 +446,13 @@ typedef struct STimeWindowAggSupp {
SColumnInfoData timeWindowData; // query time window info for scalar function execution.
} STimeWindowAggSupp;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
} SSteamOpBasicInfo;
typedef struct SStreamScanInfo {
SSteamOpBasicInfo basic;
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
SExprSupp tbnameCalSup;
@ -568,10 +574,6 @@ typedef struct SOpCheckPointInfo {
SHashObj* children; // key:child id
} SOpCheckPointInfo;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
} SSteamOpBasicInfo;
typedef struct SStreamIntervalOperatorInfo {
SOptrBasicInfo binfo; // basic info
SSteamOpBasicInfo basic;

32
source/libs/executor/inc/streamexecutorInt.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef STREAM_EXECUTORINT_H
#define STREAM_EXECUTORINT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "executorInt.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
#ifdef __cplusplus
}
#endif
#endif // STREAM_EXECUTORINT_H

2
source/libs/executor/src/aggregateoperator.c
View File

@ -484,7 +484,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
if (!osTempSpaceAvailable()) {
code = TSDB_CODE_NO_DISKSPACE;
qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", terrstr(code), pKey, tsTempDir);
qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", tstrerror(code), pKey, tsTempDir);
return code;
}

11
source/libs/executor/src/executor.c
View File

@ -824,14 +824,17 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
while(1) {
taosWLockLatch(&pTaskInfo->lock);
while (qTaskIsExecuting(pTaskInfo)) {
taosMsleep(10);
}
if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
taosWUnLockLatch(&pTaskInfo->lock);
taosMsleep(100);
} else { // not running now
pTaskInfo->code = rspCode;
taosWUnLockLatch(&pTaskInfo->lock);
return TSDB_CODE_SUCCESS;
}
}
}
bool qTaskIsExecuting(qTaskInfo_t qinfo) {

10
source/libs/executor/src/scanoperator.c
View File

@ -20,6 +20,7 @@
#include "os.h"
#include "querynodes.h"
#include "systable.h"
#include "streamexecutorInt.h"
#include "tname.h"
#include "tdatablock.h"
@ -2426,10 +2427,13 @@ void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
if (!pInfo->pState) {
return;
}
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
// other properties are recovered from the execution plan
@ -2582,6 +2586,7 @@ FETCH_NEXT_BLOCK:
case STREAM_NORMAL:
case STREAM_GET_ALL:
printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
return pBlock;
case STREAM_RETRIEVE: {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
@ -2622,6 +2627,7 @@ FETCH_NEXT_BLOCK:
if (pInfo->pDeleteDataRes->info.rows > 0) {
printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
return pInfo->pDeleteDataRes;
} else {
goto FETCH_NEXT_BLOCK;
@ -2639,6 +2645,7 @@ FETCH_NEXT_BLOCK:
if (pInfo->pDeleteDataRes->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
return pInfo->pDeleteDataRes;
} else {
goto FETCH_NEXT_BLOCK;
@ -2652,6 +2659,7 @@ FETCH_NEXT_BLOCK:
break;
}
printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
return pBlock;
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
qDebug("stream scan mode:%d, %s", pInfo->scanMode, id);
@ -2659,6 +2667,7 @@ FETCH_NEXT_BLOCK:
case STREAM_SCAN_FROM_RES: {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey, pInfo->pRes);
setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
pInfo->pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
@ -2762,6 +2771,7 @@ FETCH_NEXT_BLOCK:
}
doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes);
setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);

5
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -17,6 +17,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tdatablock.h"
@ -415,6 +416,7 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
@ -422,6 +424,8 @@ void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
@ -550,6 +554,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator);

5
source/libs/executor/src/streameventwindowoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -458,6 +459,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
@ -465,6 +467,8 @@ void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static SSDataBlock* buildEventResult(SOperatorInfo* pOperator) {
@ -531,6 +535,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {

30
source/libs/executor/src/streamexecutorInt.c Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
pBasicInfo->updateOperatorInfo = true;
}
}
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
return pBasicInfo->updateOperatorInfo;
}
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
pBasicInfo->updateOperatorInfo = false;
}

16
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -1211,6 +1212,7 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
@ -1218,6 +1220,8 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {
@ -1347,6 +1351,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
pInfo->binfo.pRes->info.type = pBlock->info.type;
@ -2690,6 +2695,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
@ -2697,6 +2703,8 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
@ -2766,6 +2774,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -3176,6 +3185,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -3673,6 +3683,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
@ -3680,6 +3691,8 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
}
static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) {
@ -3746,6 +3759,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -4069,6 +4083,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
@ -4465,6 +4480,7 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
}
pInfo->numOfDatapack++;
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
pInfo->binfo.pRes->info.type = pBlock->info.type;

10
source/libs/executor/src/sysscanoperator.c
View File

@ -549,7 +549,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
if (!pInfo->pCur || !pInfo->pSchema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("sysTableScanUserCols failed since %s", terrstr(terrno));
qError("sysTableScanUserCols failed since %s", terrstr());
blockDataDestroy(pDataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
@ -1651,6 +1651,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
SSysTableScanInfo* pInfo = pOperator->info;
char dbName[TSDB_DB_NAME_LEN] = {0};
while (1) {
if (isTaskKilled(pOperator->pTaskInfo)) {
setOperatorCompleted(pOperator);
return NULL;
@ -1692,10 +1694,14 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
}
return pBlock->info.rows > 0 ? pBlock : NULL;
if (pBlock->info.rows == 0) {
continue;
}
return pBlock;
} else {
return NULL;
}
}
}
static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name,

5
source/libs/executor/src/timesliceoperator.c
View File

@ -204,6 +204,11 @@ static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumn
SRowKey cur = {.ts = currentTs, .numOfPKs = (pPkCol != NULL)? 1:0};
if (pPkCol != NULL) {
cur.pks[0].type = pPkCol->info.type;
if (IS_VAR_DATA_TYPE(pPkCol->info.type)) {
cur.pks[0].pData = (uint8_t*)colDataGetVarData(pPkCol, curIndex);
} else {
memcpy(&cur.pks[0].val, colDataGetData(pPkCol, curIndex), pPkCol->info.bytes);
}
}
// let's discard the duplicated ts

4
source/libs/function/src/udfd.c
View File

@ -842,7 +842,7 @@ void udfdGetFuncBodyPath(const SUdf *udf, char *path) {
int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
if (!osDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
fnError("udfd create shared library failed since %s", terrstr(terrno));
fnError("udfd create shared library failed since %s", terrstr());
return terrno;
}
@ -1424,7 +1424,7 @@ int main(int argc, char *argv[]) {
printf("failed to start since init log error\n");
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
fnError("failed to start since read config error");
return -2;
}

2
source/libs/function/test/runUdf.c
View File

@ -127,7 +127,7 @@ int aggregateFuncTest() {
int main(int argc, char *argv[]) {
parseArgs(argc, argv);
initLog();
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
fnError("failed to start since read config error");
return -1;
}

70
source/libs/geometry/src/geosWrapper.c
View File

@ -86,6 +86,67 @@ _exit:
return code;
}
static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
int ret = 0;
char *wktPatternWithSpace = taosMemoryCalloc(4, 1024);
sprintf(
wktPatternWithSpace,
"^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))|polygon( *)z?m?( "
"*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
"*)\\)))|multipoint( *)z?m?( *)((empty)|(\\(( "
"*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))(( *)(,)( "
"*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))( *))*( "
"*)\\)))|multilinestring( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
"*)\\)))|multipolygon( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( *)((empty)|(\\(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
"*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( "
"*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$");
ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace);
taosMemoryFree(wktPatternWithSpace);
return ret;
}
int32_t initCtxGeomFromText() {
int32_t code = TSDB_CODE_FAILED;
SGeosContext* geosCtx = getThreadLocalGeosCtx();
@ -113,6 +174,10 @@ int32_t initCtxGeomFromText() {
}
}
if (geosCtx->WKTRegex == NULL) {
if (initWktRegex(&geosCtx->WKTRegex, &geosCtx->WKTMatchData) != 0) return code;
}
return TSDB_CODE_SUCCESS;
}
@ -125,6 +190,11 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t
GEOSGeometry *geom = NULL;
unsigned char *wkb = NULL;
if (doRegExec(inputWKT, geosCtx->WKTRegex, geosCtx->WKTMatchData) != 0) {
code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
goto _exit;
}
geom = GEOSWKTReader_read_r(geosCtx->handle, geosCtx->WKTReader, inputWKT);
if (geom == NULL) {
code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;

4
source/libs/monitorfw/CMakeLists.txt
View File

@ -5,5 +5,7 @@ target_include_directories(
PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
if(${TD_DARWIN})
target_compile_options(monitorfw PRIVATE -Wno-error=deprecated-pragma)
endif(${TD_DARWIN})
target_link_libraries(monitorfw os util common transport)

6
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -187,6 +187,8 @@ const char* nodesNodeName(ENodeType type) {
return "BalanceVgroupStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return "BalanceVgroupLeaderStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return "BalanceVgroupLeaderStmt";
case QUERY_NODE_MERGE_VGROUP_STMT:
return "MergeVgroupStmt";
case QUERY_NODE_SHOW_DB_ALIVE_STMT:
@ -7607,6 +7609,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_MERGE_VGROUP_STMT:
return mergeVgroupStmtToJson(pObj, pJson);
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@ -7953,6 +7957,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_BALANCE_VGROUP_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to deserialize.
case QUERY_NODE_MERGE_VGROUP_STMT:
return jsonToMergeVgroupStmt(pJson, pObj);

3
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -473,6 +473,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SBalanceVgroupStmt));
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
case QUERY_NODE_MERGE_VGROUP_STMT:
return makeNode(type, sizeof(SMergeVgroupStmt));
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@ -1161,6 +1163,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field
case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field
break;
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:

1
source/libs/parser/inc/parAst.h
View File

@ -274,6 +274,7 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId);
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);

1
source/libs/parser/inc/sql.y
View File

@ -773,6 +773,7 @@ cmd ::= KILL COMPACT NK_INTEGER(A).
/************************************************ merge/redistribute/ vgroup ******************************************/
cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); }
cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); }
cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B). { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
cmd ::= SPLIT VGROUP NK_INTEGER(A). { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }

10
source/libs/parser/src/parAstCreater.c
View File

@ -2809,6 +2809,16 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI
return (SNode*)pStmt;
}
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){
CHECK_PARSER_STATUS(pCxt);
SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
}
return (SNode*)pStmt;
}
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
CHECK_PARSER_STATUS(pCxt);
SMergeVgroupStmt* pStmt = (SMergeVgroupStmt*)nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);

24
source/libs/parser/src/parAstParser.c
View File

@ -42,7 +42,6 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol
return TSDB_CODE_SUCCESS;
}
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
SAstCreateContext cxt;
initAstCreateContext(pParseCxt, &cxt);
@ -534,6 +533,10 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt*
static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal,
((SValueNode*)pStmt->pTbName)->literal, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
}
@ -631,8 +634,8 @@ static int32_t collectMetaKeyFromShowCompacts(SCollectMetaKeyCxt* pCxt, SShowStm
}
static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_COMPACT_DETAILS,
pCxt->pMetaCache);
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB,
TSDB_INS_TABLE_COMPACT_DETAILS, pCxt->pMetaCache);
return code;
}
@ -696,7 +699,6 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
return code;
}
static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS,
pCxt->pMetaCache);
@ -755,24 +757,22 @@ static int32_t collectMetaKeyFromRevoke(SCollectMetaKeyCxt* pCxt, SRevokeStmt* p
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreateViewStmt* pStmt) {
int32_t code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
AUTH_TYPE_ALTER, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) {
int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName,
pStmt->viewName, AUTH_TYPE_ALTER, pCxt->pMetaCache);
return code;
}
@ -783,7 +783,7 @@ static int32_t collectMetaKeyFromCreateTSMAStmt(SCollectMetaKeyCxt* pCxt, SCreat
code = reserveTSMAInfoInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
char dstTbName[TSDB_TABLE_NAME_LEN] = {0};
snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s"TSMA_RES_STB_POSTFIX, pStmt->tableName);
snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s" TSMA_RES_STB_POSTFIX, pStmt->tableName);
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);

17
source/libs/parser/src/parInsertUtil.c
View File

@ -677,6 +677,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
p += sizeof(uint64_t);
int8_t* fields = p;
if(*fields >= TSDB_DATA_TYPE_MAX || *fields < 0){
uError("fields type error:%d", *fields);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
p += numOfCols * (sizeof(int8_t) + sizeof(int32_t));
int32_t* colLength = (int32_t*)p;
@ -688,12 +693,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo;
if (tFields != NULL && numFields != numOfCols) {
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d != raw numOfCols:%d", numFields, numOfCols);
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to data cols:%d", numFields, numOfCols);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
if (tFields != NULL && numFields > boundInfo->numOfBound) {
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound);
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -702,8 +707,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -732,8 +737,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
if (strcmp(pColSchema->name, tFields[i].name) == 0) {
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}

41
source/libs/parser/src/parTranslater.c
View File

@ -4338,7 +4338,7 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
}
#ifdef TD_ENTERPRISE
if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) {
if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType && !pCurrSmt->tagScan) {
return translateView(pCxt, pTable, &name);
}
translateAudit(pCxt, pRealTable, &name);
@ -10578,6 +10578,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId;
if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
int32_t code =
buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
tFreeSBalanceVgroupLeaderReq(&req);
@ -11263,6 +11264,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
break;
case QUERY_NODE_MERGE_VGROUP_STMT:
code = translateMergeVgroup(pCxt, (SMergeVgroupStmt*)pNode);
break;
@ -11867,6 +11871,37 @@ static int32_t rewriteShowVgroups(STranslateContext* pCxt, SQuery* pQuery) {
return code;
}
static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) {
int32_t code = 0;
SName name;
STableMeta* pTableMeta = NULL;
code = getTargetMeta(pCxt,
toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal,
((SValueNode*)pShow->pTbName)->literal, &name),
&pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
goto _exit;
}
if (TSDB_SUPER_TABLE != pTableMeta->tableType && TSDB_CHILD_TABLE != pTableMeta->tableType) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC,
"The _TAGS pseudo column can only be used for child table and super table queries");
goto _exit;
}
_exit:
taosMemoryFreeClear(pTableMeta);
return code;
}
static int32_t rewriteShowTags(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = checkShowTags(pCxt, (SShowStmt*)pQuery->pRoot);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteShow(pCxt, pQuery);
}
return code;
}
static SNode* createTagsFunction() {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
@ -13170,7 +13205,6 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_APPS_STMT:
case QUERY_NODE_SHOW_CONSUMERS_STMT:
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
@ -13181,6 +13215,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_TSMAS_STMT:
code = rewriteShow(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_TAGS_STMT:
code = rewriteShowTags(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_VGROUPS_STMT:
code = rewriteShowVgroups(pCxt, pQuery);
break;

4364
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

7
source/libs/planner/src/planOptimizer.c
View File

@ -246,11 +246,10 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
if (pNode->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
if (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) return true;
} else {
if (!pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) ||
WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)
return !scanPathOptHaveNormalCol(((SPartitionLogicNode*)pNode)->pPartitionKeys);
}
return false;
}
if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&

2
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -2740,9 +2740,11 @@ int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey,
if (pKey->groupId == groupId) {
return 0;
}
if (pVal != NULL) {
taosMemoryFree((void*)*pVal);
*pVal = NULL;
}
}
return -1;
}
int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {

2
source/libs/stream/src/streamMeta.c
View File

@ -1631,6 +1631,8 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
}
} else {
streamMetaRUnLock(pMeta);
stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
streamId, taskId, pMeta->vgId);
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;

7
source/libs/stream/src/tstreamFileState.c
View File

@ -557,7 +557,6 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
const int32_t BATCH_LIMIT = 256;
int64_t st = taosGetTimestampMs();
int32_t numOfElems = listNEles(pSnapshot);
SListNode* pNode = NULL;
int idx = streamStateGetCfIdx(pFileState->pFileStore, pFileState->cfName);
@ -589,8 +588,11 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
}
taosMemoryFree(buf);
if (streamStateGetBatchSize(batch) > 0) {
int32_t numOfElems = streamStateGetBatchSize(batch);
if (numOfElems > 0) {
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
} else {
goto _end;
}
streamStateClearBatch(batch);
@ -609,6 +611,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
}
_end:
streamStateDestroyBatch(batch);
return code;
}

2
source/libs/sync/src/syncMain.c
View File

@ -463,7 +463,7 @@ _DEL_WAL:
pSyncNode->snapshottingIndex, lastApplyIndex);
} else {
sNError(pSyncNode, "wal snapshot begin error since:%s, index:%" PRId64 ", last apply index:%" PRId64,
terrstr(terrno), pSyncNode->snapshottingIndex, lastApplyIndex);
terrstr(), pSyncNode->snapshottingIndex, lastApplyIndex);
atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID);
}

6
source/libs/transport/inc/transComm.h
View File

@ -123,7 +123,7 @@ typedef struct {
STransMsg* pRsp;
SEpSet epSet;
int8_t hasEpSet;
tsem_t* pSem;
tsem2_t* pSem;
int8_t inited;
SRWLatch latch;
} STransSyncMsg;
@ -302,8 +302,8 @@ int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf);
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
bool transReadComplete(SConnBuffer* connBuf);
int transResetBuffer(SConnBuffer* connBuf);
int transDumpFromBuffer(SConnBuffer* connBuf, char** buf);
int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf);
int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf);
int transSetConnOption(uv_tcp_t* stream, int keepalive);

15
source/libs/transport/src/transCli.c
View File

@ -382,13 +382,18 @@ void cliHandleResp(SCliConn* conn) {
STransMsgHead* pHead = NULL;
int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead);
int8_t resetBuf = conn->status == ConnAcquire ? 0 : 1;
int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead, resetBuf);
if (msgLen <= 0) {
taosMemoryFree(pHead);
tDebug("%s conn %p recv invalid packet ", CONN_GET_INST_LABEL(conn), conn);
return;
}
if (resetBuf == 0) {
tTrace("%s conn %p not reset read buf", transLabel(pTransInst), conn);
}
if (transDecompressMsg((char**)&pHead, msgLen) < 0) {
tDebug("%s conn %p recv invalid packet, failed to decompress", CONN_GET_INST_LABEL(conn), conn);
}
@ -2450,7 +2455,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
pSyncMsg->hasEpSet = 1;
epsetAssign(&pSyncMsg->epSet, &pCtx->epSet);
}
tsem_post(pSyncMsg->pSem);
tsem2_post(pSyncMsg->pSem);
taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef);
} else {
rpcFreeCont(pResp->pCont);
@ -2679,8 +2684,8 @@ _RETURN:
return ret;
}
int64_t transCreateSyncMsg(STransMsg* pTransMsg) {
tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
tsem_init(sem, 0, 0);
tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t));
tsem2_init(sem, 0, 0);
STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg));
@ -2740,7 +2745,7 @@ int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STr
goto _RETURN;
}
ret = tsem_timewait(pSyncMsg->pSem, timeoutMs);
ret = tsem2_timewait(pSyncMsg->pSem, timeoutMs);
if (ret < 0) {
pRsp->code = TSDB_CODE_TIMEOUT_ERROR;
ret = TSDB_CODE_TIMEOUT_ERROR;

10
source/libs/transport/src/transComm.c
View File

@ -126,7 +126,7 @@ int transClearBuffer(SConnBuffer* buf) {
return 0;
}
int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) {
static const int HEADSIZE = sizeof(STransMsgHead);
SConnBuffer* p = connBuf;
@ -137,7 +137,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
if (total >= HEADSIZE && !p->invalid) {
*buf = taosMemoryCalloc(1, total);
memcpy(*buf, p->buf, total);
if (transResetBuffer(connBuf) < 0) {
if (transResetBuffer(connBuf, resetBuf) < 0) {
return -1;
}
} else {
@ -146,7 +146,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
return total;
}
int transResetBuffer(SConnBuffer* connBuf) {
int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) {
SConnBuffer* p = connBuf;
if (p->total < p->len) {
int left = p->len - p->total;
@ -159,9 +159,11 @@ int transResetBuffer(SConnBuffer* connBuf) {
p->total = 0;
p->len = 0;
if (p->cap > BUFFER_CAP) {
if (resetBuf) {
p->cap = BUFFER_CAP;
p->buf = taosMemoryRealloc(p->buf, p->cap);
}
}
} else {
ASSERTS(0, "invalid read from sock buf");
return -1;
@ -681,7 +683,7 @@ void transDestroySyncMsg(void* msg) {
if (msg == NULL) return;
STransSyncMsg* pSyncMsg = msg;
tsem_destroy(pSyncMsg->pSem);
tsem2_destroy(pSyncMsg->pSem);
taosMemoryFree(pSyncMsg->pSem);
transFreeMsg(pSyncMsg->pRsp->pCont);
taosMemoryFree(pSyncMsg->pRsp);

7
source/libs/transport/src/transSvr.c
View File

@ -342,11 +342,15 @@ static bool uvHandleReq(SSvrConn* pConn) {
STransMsgHead* pHead = NULL;
int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead);
int8_t resetBuf = pConn->status == ConnAcquire ? 0 : 1;
int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead, resetBuf);
if (msgLen <= 0) {
tError("%s conn %p read invalid packet", transLabel(pTransInst), pConn);
return false;
}
if (resetBuf == 0) {
tTrace("%s conn %p not reset read buf", transLabel(pTransInst), pConn);
}
if (transDecompressMsg((char**)&pHead, msgLen) < 0) {
tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn);
@ -676,6 +680,7 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
taosMemoryFree(smsg);
}
static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); }
static void destroyAllConn(SWorkThrd* pThrd) {
tTrace("thread %p destroy all conn ", pThrd);
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {

105
source/os/src/osSemaphore.c
View File

@ -215,14 +215,6 @@ int32_t taosGetAppName(char* name, int32_t* len) {
return 0;
}
int32_t tsem_wait(tsem_t* sem) {
int ret = 0;
do {
ret = sem_wait(sem);
} while (ret != 0 && errno == EINTR);
return ret;
}
int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
int ret = 0;
@ -241,4 +233,101 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
return ret;
}
int32_t tsem_wait(tsem_t* sem) {
int ret = 0;
do {
ret = sem_wait(sem);
} while (ret != 0 && errno == EINTR);
return ret;
}
int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) {
int ret = taosThreadMutexInit(&sem->mutex, NULL);
if (ret != 0) return ret;
ret = taosThreadCondAttrInit(&sem->attr);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
return ret;
}
ret = taosThreadCondAttrSetclock(&sem->attr, CLOCK_MONOTONIC);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondAttrDestroy(&sem->attr);
return ret;
}
ret = taosThreadCondInit(&sem->cond, &sem->attr);
if (ret != 0)
{
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondAttrDestroy(&sem->attr);
return ret;
}
sem->count = value;
return 0;
}
int tsem2_post(tsem2_t *sem) {
taosThreadMutexLock(&sem->mutex);
sem->count++;
taosThreadCondSignal(&sem->cond);
taosThreadMutexUnlock(&sem->mutex);
return 0;
}
int tsem2_destroy(tsem2_t* sem) {
taosThreadMutexDestroy(&sem->mutex);
taosThreadCondDestroy(&sem->cond);
taosThreadCondAttrDestroy(&sem->attr);
return 0;
}
int32_t tsem2_wait(tsem2_t* sem) {
taosThreadMutexLock(&sem->mutex);
while (sem->count <= 0) {
int ret = taosThreadCondWait(&sem->cond, &sem->mutex);
if (0 == ret) {
continue;
} else {
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
}
sem->count--;
taosThreadMutexUnlock(&sem->mutex);
return 0;
}
int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
int ret = 0;
taosThreadMutexLock(&sem->mutex);
if (sem->count <= 0) {
struct timespec ts = {0};
if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) {
taosThreadMutexUnlock(&sem->mutex);
return -1;
}
ts.tv_sec += ms / 1000;
ts.tv_nsec += (ms % 1000) * 1000000;
ts.tv_sec += ts.tv_nsec / 1000000000;
ts.tv_nsec %= 1000000000;
while (sem->count <= 0) {
ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts);
if (ret != 0) {
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
}
}
sem->count--;
taosThreadMutexUnlock(&sem->mutex);
return ret;
}
#endif

10
source/os/src/osThread.c
View File

@ -170,6 +170,16 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) {
#endif
}
int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) {
#ifdef __USE_WIN_THREAD
return 0;
#elif defined(__APPLE__)
return 0;
#else
return pthread_condattr_setclock(attr, clockId);
#endif
}
int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) {
#ifdef __USE_WIN_THREAD
return 0;

Some files were not shown because too many files have changed in this diff Show More