xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

other: merge 3.0

This commit is contained in:
Haojun Liao 2022-10-28 22:27:57 +08:00
parent 029a4a85b3 9b99c69785
commit 307b7e0ac8
111 changed files with 5233 additions and 3889 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Jenkinsfile2
View File

@ -173,8 +173,8 @@ def pre_test_build_mac() {
'''
sh '''
cd ${WK}/debug
cmake .. -DBUILD_TEST=true
make -j8
cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
make -j10
'''
sh '''
date
@ -428,7 +428,7 @@ pipeline {
date
rm -rf ${WKC}/debug
cd ${WKC}/tests/parallel_test
time ./container_build.sh -w ${WKDIR} -t 8 -e
time ./container_build.sh -w ${WKDIR} -t 10 -e
rm -f /tmp/cases.task
./collect_cases.sh -e
'''

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG f20eb34
GIT_TAG 7321fbb
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

54
docs/en/14-reference/03-connector/05-go.mdx
View File

@ -353,7 +353,7 @@ The `af` package encapsulates TDengine advanced functions such as connection man
Closes the connection.
#### Subscribe to
#### Subscribe
* `func NewConsumer(conf *Config) (*Consumer, error)`
@ -387,7 +387,7 @@ Close consumer.
* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error`
Write to influxDB line protocol.
Write to InfluxDB line protocol.
* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error`
@ -413,7 +413,7 @@ Close consumer.
* `func (stmt *InsertStmt) SetTableName(name string) error`
Bind the set table name parameter.
Bind the table name parameter.
* `func (stmt *InsertStmt) SetSubTableName(name string) error`
@ -463,6 +463,54 @@ Close consumer.
For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go)
### parameter binding via WebSocket
* `func NewConnector(config *Config) (*Connector, error)`
Create a connection.
* `func (c *Connector) Init() (*Stmt, error)`
Initialize the parameters.
* `func (c *Connector) Close() error`
Close the connection.
* `func (s *Stmt) Prepare(sql string) error`
Parameter binding preprocessing SQL statement.
* `func (s *Stmt) SetTableName(name string) error`
Bind the table name parameter.
* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error`
Set tags.
* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error`
Parameter bind multiple rows of data.
* `func (s *Stmt) AddBatch() error`
Add to a parameter-bound batch.
* `func (s *Stmt) Exec() error`
Execute a parameter binding.
* `func (s *Stmt) GetAffectedRows() int`
Gets the number of affected rows inserted by the parameter binding.
* `func (s *Stmt) Close() error`
Closes the parameter binding.
For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go)
## API Reference
Full API see [driver-go documentation](https://pkg.go.dev/github.com/taosdata/driver-go/v3)

52
docs/zh/08-connector/20-go.mdx
View File

@ -8,7 +8,7 @@ title: TDengine Go Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx"
import Preparation from "./_preparation.mdx"
import GoInsert from "../07-develop/03-insert-data/_go_sql.mdx"
import GoInfluxLine from "../07-develop/03-insert-data/_go_line.mdx"
import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx"
@ -389,7 +389,7 @@ func main() {
* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error`
写入 influxDB 行协议。
写入 InfluxDB 行协议。
* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error`
@ -465,6 +465,54 @@ func main() {
完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go)
### 通过 WebSocket 进行参数绑定
* `func NewConnector(config *Config) (*Connector, error)`
创建连接。
* `func (c *Connector) Init() (*Stmt, error)`
初始化参数。
* `func (c *Connector) Close() error`
关闭连接。
* `func (s *Stmt) Prepare(sql string) error`
参数绑定预处理 SQL 语句。
* `func (s *Stmt) SetTableName(name string) error`
参数绑定设置表名。
* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error`
参数绑定设置标签。
* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error`
参数绑定多行数据。
* `func (s *Stmt) AddBatch() error`
添加到参数绑定批处理。
* `func (s *Stmt) Exec() error`
执行参数绑定。
* `func (s *Stmt) GetAffectedRows() int`
获取参数绑定插入受影响行数。
* `func (s *Stmt) Close() error`
结束参数绑定。
完整参数绑定示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go)
## API 参考
全部 API 见 [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3)

2
docs/zh/08-connector/26-rust.mdx
View File

@ -7,7 +7,7 @@ title: TDengine Rust Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx"
import Preparation from "./_preparation.mdx"
import RustInsert from "../07-develop/03-insert-data/_rust_sql.mdx"
import RustBind from "../07-develop/03-insert-data/_rust_stmt.mdx"
import RustQuery from "../07-develop/04-query-data/_rust.mdx"

4
docs/zh/08-connector/35-node.mdx
View File

@ -7,7 +7,7 @@ title: TDengine Node.js Connector
import Tabs from "@theme/Tabs";
import TabItem from "@theme/TabItem";
import Preparition from "./_preparition.mdx";
import Preparation from "./_preparation.mdx";
import NodeInsert from "../07-develop/03-insert-data/_js_sql.mdx";
import NodeInfluxLine from "../07-develop/03-insert-data/_js_line.mdx";
import NodeOpenTSDBTelnet from "../07-develop/03-insert-data/_js_opts_telnet.mdx";
@ -249,4 +249,4 @@ Node.js 连接器的使用参见[视频教程](https://www.taosdata.com/blog/202
## API 参考
[API 参考](https://docs.taosdata.com/api/td2.0-connector/)
[API 参考](https://docs.taosdata.com/api/td2.0-connector/)

2
docs/zh/08-connector/40-csharp.mdx
View File

@ -7,7 +7,7 @@ title: C# Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx"
import Preparation from "./_preparation.mdx"
import CSInsert from "../07-develop/03-insert-data/_cs_sql.mdx"
import CSInfluxLine from "../07-develop/03-insert-data/_cs_line.mdx"
import CSOpenTSDBTelnet from "../07-develop/03-insert-data/_cs_opts_telnet.mdx"

0
docs/zh/08-connector/_preparition.mdx → docs/zh/08-connector/_preparation.mdx
View File

2
docs/zh/12-taos-sql/24-show.md
View File

@ -201,7 +201,7 @@ SHOW TOPICS;
SHOW TRANSACTIONS;
```
显示当前系统中正在执行的事务的信息
显示当前系统中正在执行的事务的信息(该事务仅针对除普通表以外的元数据级别)
## SHOW USERS

4
include/common/tglobal.h
View File

@ -60,8 +60,8 @@ extern int32_t tsNumOfVnodeSyncThreads;
extern int32_t tsNumOfVnodeRsmaThreads;
extern int32_t tsNumOfQnodeQueryThreads;
extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeSharedThreads;
extern int32_t tsNumOfSnodeUniqueThreads;
extern int32_t tsNumOfSnodeStreamThreads;
extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsRpcQueueMemoryAllowed;
// monitor

3
include/common/tmsg.h
View File

@ -1726,6 +1726,9 @@ typedef struct {
#define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_DEFAULT_IGNORE_EXPIRED 0
#define STREAM_FILL_HISTORY_ON 1
#define STREAM_FILL_HISTORY_OFF 0
#define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF
typedef struct {
char name[TSDB_STREAM_FNAME_LEN];

259
include/common/ttokendef.h
View File

@ -203,135 +203,136 @@
#define TK_WINDOW_CLOSE 185
#define TK_IGNORE 186
#define TK_EXPIRED 187
#define TK_SUBTABLE 188
#define TK_KILL 189
#define TK_CONNECTION 190
#define TK_TRANSACTION 191
#define TK_BALANCE 192
#define TK_VGROUP 193
#define TK_MERGE 194
#define TK_REDISTRIBUTE 195
#define TK_SPLIT 196
#define TK_DELETE 197
#define TK_INSERT 198
#define TK_NULL 199
#define TK_NK_QUESTION 200
#define TK_NK_ARROW 201
#define TK_ROWTS 202
#define TK_TBNAME 203
#define TK_QSTART 204
#define TK_QEND 205
#define TK_QDURATION 206
#define TK_WSTART 207
#define TK_WEND 208
#define TK_WDURATION 209
#define TK_IROWTS 210
#define TK_QTAGS 211
#define TK_CAST 212
#define TK_NOW 213
#define TK_TODAY 214
#define TK_TIMEZONE 215
#define TK_CLIENT_VERSION 216
#define TK_SERVER_VERSION 217
#define TK_SERVER_STATUS 218
#define TK_CURRENT_USER 219
#define TK_COUNT 220
#define TK_LAST_ROW 221
#define TK_CASE 222
#define TK_END 223
#define TK_WHEN 224
#define TK_THEN 225
#define TK_ELSE 226
#define TK_BETWEEN 227
#define TK_IS 228
#define TK_NK_LT 229
#define TK_NK_GT 230
#define TK_NK_LE 231
#define TK_NK_GE 232
#define TK_NK_NE 233
#define TK_MATCH 234
#define TK_NMATCH 235
#define TK_CONTAINS 236
#define TK_IN 237
#define TK_JOIN 238
#define TK_INNER 239
#define TK_SELECT 240
#define TK_DISTINCT 241
#define TK_WHERE 242
#define TK_PARTITION 243
#define TK_BY 244
#define TK_SESSION 245
#define TK_STATE_WINDOW 246
#define TK_SLIDING 247
#define TK_FILL 248
#define TK_VALUE 249
#define TK_NONE 250
#define TK_PREV 251
#define TK_LINEAR 252
#define TK_NEXT 253
#define TK_HAVING 254
#define TK_RANGE 255
#define TK_EVERY 256
#define TK_ORDER 257
#define TK_SLIMIT 258
#define TK_SOFFSET 259
#define TK_LIMIT 260
#define TK_OFFSET 261
#define TK_ASC 262
#define TK_NULLS 263
#define TK_ABORT 264
#define TK_AFTER 265
#define TK_ATTACH 266
#define TK_BEFORE 267
#define TK_BEGIN 268
#define TK_BITAND 269
#define TK_BITNOT 270
#define TK_BITOR 271
#define TK_BLOCKS 272
#define TK_CHANGE 273
#define TK_COMMA 274
#define TK_COMPACT 275
#define TK_CONCAT 276
#define TK_CONFLICT 277
#define TK_COPY 278
#define TK_DEFERRED 279
#define TK_DELIMITERS 280
#define TK_DETACH 281
#define TK_DIVIDE 282
#define TK_DOT 283
#define TK_EACH 284
#define TK_FAIL 285
#define TK_FILE 286
#define TK_FOR 287
#define TK_GLOB 288
#define TK_ID 289
#define TK_IMMEDIATE 290
#define TK_IMPORT 291
#define TK_INITIALLY 292
#define TK_INSTEAD 293
#define TK_ISNULL 294
#define TK_KEY 295
#define TK_MODULES 296
#define TK_NK_BITNOT 297
#define TK_NK_SEMI 298
#define TK_NOTNULL 299
#define TK_OF 300
#define TK_PLUS 301
#define TK_PRIVILEGE 302
#define TK_RAISE 303
#define TK_REPLACE 304
#define TK_RESTRICT 305
#define TK_ROW 306
#define TK_SEMI 307
#define TK_STAR 308
#define TK_STATEMENT 309
#define TK_STRING 310
#define TK_TIMES 311
#define TK_UPDATE 312
#define TK_VALUES 313
#define TK_VARIABLE 314
#define TK_VIEW 315
#define TK_WAL 316
#define TK_FILL_HISTORY 188
#define TK_SUBTABLE 189
#define TK_KILL 190
#define TK_CONNECTION 191
#define TK_TRANSACTION 192
#define TK_BALANCE 193
#define TK_VGROUP 194
#define TK_MERGE 195
#define TK_REDISTRIBUTE 196
#define TK_SPLIT 197
#define TK_DELETE 198
#define TK_INSERT 199
#define TK_NULL 200
#define TK_NK_QUESTION 201
#define TK_NK_ARROW 202
#define TK_ROWTS 203
#define TK_TBNAME 204
#define TK_QSTART 205
#define TK_QEND 206
#define TK_QDURATION 207
#define TK_WSTART 208
#define TK_WEND 209
#define TK_WDURATION 210
#define TK_IROWTS 211
#define TK_QTAGS 212
#define TK_CAST 213
#define TK_NOW 214
#define TK_TODAY 215
#define TK_TIMEZONE 216
#define TK_CLIENT_VERSION 217
#define TK_SERVER_VERSION 218
#define TK_SERVER_STATUS 219
#define TK_CURRENT_USER 220
#define TK_COUNT 221
#define TK_LAST_ROW 222
#define TK_CASE 223
#define TK_END 224
#define TK_WHEN 225
#define TK_THEN 226
#define TK_ELSE 227
#define TK_BETWEEN 228
#define TK_IS 229
#define TK_NK_LT 230
#define TK_NK_GT 231
#define TK_NK_LE 232
#define TK_NK_GE 233
#define TK_NK_NE 234
#define TK_MATCH 235
#define TK_NMATCH 236
#define TK_CONTAINS 237
#define TK_IN 238
#define TK_JOIN 239
#define TK_INNER 240
#define TK_SELECT 241
#define TK_DISTINCT 242
#define TK_WHERE 243
#define TK_PARTITION 244
#define TK_BY 245
#define TK_SESSION 246
#define TK_STATE_WINDOW 247
#define TK_SLIDING 248
#define TK_FILL 249
#define TK_VALUE 250
#define TK_NONE 251
#define TK_PREV 252
#define TK_LINEAR 253
#define TK_NEXT 254
#define TK_HAVING 255
#define TK_RANGE 256
#define TK_EVERY 257
#define TK_ORDER 258
#define TK_SLIMIT 259
#define TK_SOFFSET 260
#define TK_LIMIT 261
#define TK_OFFSET 262
#define TK_ASC 263
#define TK_NULLS 264
#define TK_ABORT 265
#define TK_AFTER 266
#define TK_ATTACH 267
#define TK_BEFORE 268
#define TK_BEGIN 269
#define TK_BITAND 270
#define TK_BITNOT 271
#define TK_BITOR 272
#define TK_BLOCKS 273
#define TK_CHANGE 274
#define TK_COMMA 275
#define TK_COMPACT 276
#define TK_CONCAT 277
#define TK_CONFLICT 278
#define TK_COPY 279
#define TK_DEFERRED 280
#define TK_DELIMITERS 281
#define TK_DETACH 282
#define TK_DIVIDE 283
#define TK_DOT 284
#define TK_EACH 285
#define TK_FAIL 286
#define TK_FILE 287
#define TK_FOR 288
#define TK_GLOB 289
#define TK_ID 290
#define TK_IMMEDIATE 291
#define TK_IMPORT 292
#define TK_INITIALLY 293
#define TK_INSTEAD 294
#define TK_ISNULL 295
#define TK_KEY 296
#define TK_MODULES 297
#define TK_NK_BITNOT 298
#define TK_NK_SEMI 299
#define TK_NOTNULL 300
#define TK_OF 301
#define TK_PLUS 302
#define TK_PRIVILEGE 303
#define TK_RAISE 304
#define TK_REPLACE 305
#define TK_RESTRICT 306
#define TK_ROW 307
#define TK_SEMI 308
#define TK_STAR 309
#define TK_STATEMENT 310
#define TK_STRING 311
#define TK_TIMES 312
#define TK_UPDATE 313
#define TK_VALUES 314
#define TK_VARIABLE 315
#define TK_VIEW 316
#define TK_WAL 317
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

4
include/dnode/snode/snode.h
View File

@ -68,8 +68,8 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
* @param pMsg The request message
* @param pRsp The response message
*/
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp);
int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg);
#ifdef __cplusplus
}

8
include/libs/catalog/catalog.h
View File

@ -203,6 +203,14 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta);
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* pVgroup, bool* exists);
/**
* Force refresh DB's local cached vgroup info.
* @param pCtg (input, got with catalogGetHandle)

1
include/libs/nodes/cmdnodes.h
View File

@ -374,6 +374,7 @@ typedef struct SStreamOptions {
int8_t triggerType;
SNode* pDelay;
SNode* pWatermark;
int8_t fillHistory;
int8_t ignoreExpired;
} SStreamOptions;

3
include/libs/nodes/plannodes.h
View File

@ -567,6 +567,7 @@ typedef struct SSubplan {
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
SNode* pTagCond;
SNode* pTagIndexCond;
bool showRewrite;
} SSubplan;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
@ -585,8 +586,6 @@ typedef struct SQueryPlan {
SExplainInfo explainInfo;
} SQueryPlan;
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);
const char* dataOrderStr(EDataOrderLevel order);
#ifdef __cplusplus

1
include/libs/planner/planner.h
View File

@ -61,7 +61,6 @@ int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);
int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan);
char* qQueryPlanToString(const SQueryPlan* pPlan);
SQueryPlan* qStringToQueryPlan(const char* pStr);
void qDestroyQueryPlan(SQueryPlan* pPlan);

2
include/libs/stream/streamState.h
View File

@ -99,6 +99,8 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
// char* streamStateSessionDump(SStreamState* pState);
#ifdef __cplusplus
}
#endif

53
packaging/cfg/taos.cfg
View File

@ -5,42 +5,50 @@
# #
########################################################
# The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started
######### 0. Client only configurations #############
# The interval for TDengine CLI to send heartbeat to mnode
# shellActivityTimer 3
############### 1. Cluster End point ############################
# The end point of the first dnode in the cluster to be connected to when this dnode or a TDengine CLI `taos` is started
# firstEp hostname:6030
# The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started
# The end point of the second dnode to be connected to if the firstEp is not available
# secondEp
# The FQDN of the host where `taosd` will be started. It can be IP address
############### 2. Configuration Parameters of current dnode #####
# The FQDN of the host on which this dnode will be started. It can be IP address
# fqdn hostname
# The port for external access after `taosd` is started
# The port for external access after this dnode is started
# serverPort 6030
# The maximum number of connections a dnode can accept
# maxShellConns 5000
# maxShellConns 5000
# The directory for writing log files
# The directory for writing log files, if you are using Windows platform please change to Windows path
# logDir /var/log/taos
# All data files are stored in this directory
# All data files are stored in this directory, if you are using Windows platform please change to Windows path
# dataDir /var/lib/taos
# temporary file's directory
# temporary file's directory, if you are using Windows platform please change to Windows path
# tempDir /tmp/
# Switch for allowing TDengine to collect and report service usage information
# telemetryReporting 1
# The maximum number of vnodes supported by dnode
# The maximum number of vnodes supported by this dnode
# supportVnodes 0
# The interval of dnode reporting status to mnode
# The interval of this dnode reporting status to mnode, [1..10] seconds
# statusInterval 1
# The interval for TDengine CLI to send heartbeat to mnode
# shellActivityTimer 3
# The minimum sliding window time, milli-second
# minSlidingTime 10
@ -66,13 +74,14 @@
# system time zone
# timezone Asia/Shanghai (CST, +0800)
# system time zone (for windows 10)
# timezone UTC-8
# system locale
# locale en_US.UTF-8
# default system charset
# system charset
# charset UTF-8
# stop writing logs when the disk size of the log folder is less than this value
@ -81,7 +90,7 @@
# stop writing temporary files when the disk size of the tmp folder is less than this value
# minimalTmpDirGB 1.0
# if disk free space is less than this value, taosd service exit directly within startup process
# if free disk space is less than this value, this dnode will fail to start
# minimalDataDirGB 2.0
# enable/disable system monitor
@ -91,14 +100,18 @@
# max number of lines per log filters
# numOfLogLines 10000000
# enable/disable async log
# write log in async way: 1 - async, 0 - sync
# asyncLog 1
# time of keeping log files, days
# time period of keeping log files, in days
# logKeepDays 0
# The following parameters are used for debug purpose only.
# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
############ 3. Debug Flag and levels #############################################
# The following parameters are used for debug purpose only by this dnode.
# debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
# Available debug levels are:
# 131: output warning and error
# 135: output debug, warning and error
# 143: output trace, debug, warning and error to log
@ -166,4 +179,4 @@
# metaDebugFlag 131
# generate core file when service crash
# enableCoreFile 1
# enableCoreFile 1

38
source/common/src/tglobal.c
View File

@ -55,8 +55,8 @@ int32_t tsNumOfVnodeSyncThreads = 2;
int32_t tsNumOfVnodeRsmaThreads = 2;
int32_t tsNumOfQnodeQueryThreads = 4;
int32_t tsNumOfQnodeFetchThreads = 1;
int32_t tsNumOfSnodeSharedThreads = 2;
int32_t tsNumOfSnodeUniqueThreads = 2;
int32_t tsNumOfSnodeStreamThreads = 4;
int32_t tsNumOfSnodeWriteThreads = 1;
// monitor
bool tsEnableMonitor = true;
@ -133,7 +133,7 @@ int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
// stream scheduler
bool tsSchedStreamToSnode = true;
bool tsDeployOnSnode = true;
/*
* minimum scale for whole system, millisecond by default
@ -390,13 +390,13 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
// tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
// if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeSharedThreads, 2, 1024, 0) != 0) return -1;
tsNumOfSnodeStreamThreads = tsNumOfCores / 4;
tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, 0) != 0) return -1;
tsNumOfSnodeUniqueThreads = tsNumOfCores / 4;
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 2, 1024, 0) != 0) return -1;
tsNumOfSnodeWriteThreads = tsNumOfCores / 4;
tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, 0) != 0) return -1;
tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
@ -542,17 +542,17 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfSnodeSharedThreads = numOfCores / 4;
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
pItem->i32 = tsNumOfSnodeSharedThreads;
tsNumOfSnodeStreamThreads = numOfCores / 4;
tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
pItem->i32 = tsNumOfSnodeStreamThreads;
pItem->stype = stype;
}
pItem = cfgGetItem(tsCfg, "numOfSnodeUniqueThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfSnodeUniqueThreads = numOfCores / 4;
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4);
pItem->i32 = tsNumOfSnodeUniqueThreads;
tsNumOfSnodeWriteThreads = numOfCores / 4;
tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
pItem->i32 = tsNumOfSnodeWriteThreads;
pItem->stype = stype;
}
@ -696,8 +696,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@ -946,9 +946,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
*/
} else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) {
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
} else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) {
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
} else if (strcasecmp("numOfLogLines", name) == 0) {
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
}

1
source/common/src/tmsg.c
View File

@ -4996,6 +4996,7 @@ int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStrea
}
void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
taosArrayDestroy(pReq->pTags);
taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->ast);
}

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -124,6 +124,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

15
source/dnode/mgmt/mgmt_snode/inc/smInt.h
View File

@ -30,9 +30,9 @@ typedef struct SSnodeMgmt {
SMsgCb msgCb;
const char *path;
const char *name;
int8_t uniqueWorkerInUse;
SArray *uniqueWorkers; // SArray<SMultiWorker*>
SSingleWorker sharedWorker;
int8_t writeWorkerInUse;
SArray *writeWroker; // SArray<SMultiWorker*>
SSingleWorker streamWorker;
} SSnodeMgmt;
// smHandle.c
@ -43,13 +43,14 @@ int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg);
// smWorker.c
int32_t smStartWorker(SSnodeMgmt *pMgmt);
void smStopWorker(SSnodeMgmt *pMgmt);
int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pMsg);
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_DND_SNODE_INT_H_*/
#endif /*_TD_DND_SNODE_INT_H_*/

14
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -69,13 +69,13 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

1
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -45,6 +45,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->name = pInput->name;
pMgmt->msgCb = pInput->msgCb;
pMgmt->msgCb.mgmt = pMgmt;
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)smPutMsgToQueue;
SSnodeOpt option = {0};
smInitOption(pMgmt, &option);

126
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -26,18 +26,24 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) {
tmsgSendRsp(&rsp);
}
static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SSnodeMgmt *pMgmt = pInfo->ahandle;
for (int32_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
const STraceId *trace = &pMsg->info.traceId;
dTrace("msg:%p, get from snode-unique queue", pMsg);
if (sndProcessUMsg(pMgmt->pSnode, pMsg) < 0) {
ASSERT(0);
dTrace("msg:%p, get from snode-write queue", pMsg);
int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL);
if (code < 0) {
dGError("snd, msg:%p failed to process write since %s", pMsg, terrstr(code));
if (pMsg->info.handle != NULL) {
tmsgSendRsp(pMsg);
}
} else {
smSendRsp(pMsg, 0);
}
smSendRsp(pMsg, 0);
dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont);
@ -45,13 +51,15 @@ static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t num
}
}
static void smProcessSharedQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = pInfo->ahandle;
static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = pInfo->ahandle;
const STraceId *trace = &pMsg->info.traceId;
dTrace("msg:%p, get from snode-shared queue", pMsg);
if (sndProcessSMsg(pMgmt->pSnode, pMsg) < 0) {
dTrace("msg:%p, get from snode-stream queue", pMsg);
int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
if (code < 0) {
dGError("snd, msg:%p failed to process stream since %s", pMsg, terrstr(code));
smSendRsp(pMsg, terrno);
ASSERT(0);
}
dTrace("msg:%p, is freed", pMsg);
@ -60,44 +68,44 @@ static void smProcessSharedQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
}
int32_t smStartWorker(SSnodeMgmt *pMgmt) {
pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->uniqueWorkers == NULL) {
pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < tsNumOfSnodeUniqueThreads; i++) {
SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pUniqueWorker == NULL) {
for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
SMultiWorkerCfg cfg = {
.max = 1,
.name = "snode-unique",
.fp = smProcessUniqueQueue,
.name = "snode-write",
.fp = smProcessWriteQueue,
.param = pMgmt,
};
if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) {
if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr());
return -1;
}
if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) {
if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
SSingleWorkerCfg cfg = {
.min = tsNumOfSnodeSharedThreads,
.max = tsNumOfSnodeSharedThreads,
.name = "snode-shared",
.fp = (FItem)smProcessSharedQueue,
.min = tsNumOfSnodeStreamThreads,
.max = tsNumOfSnodeStreamThreads,
.name = "snode-stream",
.fp = (FItem)smProcessStreamQueue,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->sharedWorker, &cfg)) {
if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) {
dError("failed to start snode shared-worker since %s", terrstr());
return -1;
}
@ -107,30 +115,50 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
}
void smStopWorker(SSnodeMgmt *pMgmt) {
for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i);
for (int32_t i = 0; i < taosArrayGetSize(pMgmt->writeWroker); i++) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, i);
tMultiWorkerCleanup(pWorker);
}
taosArrayDestroy(pMgmt->uniqueWorkers);
tSingleWorkerCleanup(&pMgmt->sharedWorker);
taosArrayDestroy(pMgmt->writeWroker);
tSingleWorkerCleanup(&pMgmt->streamWorker);
dDebug("snode workers are closed");
}
static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
SMsgHead *pHead = pMsg->pCont;
pHead->vgId = htonl(pHead->vgId);
return pHead->vgId % tsNumOfSnodeUniqueThreads;
}
int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
}
static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
/*SMsgHead *pHead = pMsg->pCont;*/
/*pHead->workerType = htonl(pHead->workerType);*/
/*return pHead->workerType;*/
SSnode *pSnode = pMgmt->pSnode;
if (pSnode == NULL) {
dError("snode: msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pMsg, terrstr(),
TMSG_INFO(pMsg->msgType), qtype);
return -1;
}
SMsgHead *pHead = pRpc->pCont;
pHead->contLen = htonl(pHead->contLen);
pHead->vgId = SNODE_HANDLE;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
switch (qtype) {
case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break;
case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break;
default:
ASSERT(0);
}
return 0;
}
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
@ -141,9 +169,8 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0;
}
int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t index = smGetSWIdFromMsg(pMsg);
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
@ -154,19 +181,14 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0;
}
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->sharedWorker;
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
}
int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t workerType = smGetSWTypeFromMsg(pMsg);
if (workerType == SND_WORKER_TYPE__SHARED) {
return smPutNodeMsgToSharedQueue(pMgmt, pMsg);
if (pMsg->msgType == TDMT_STREAM_TASK_DISPATCH) {
sndEnqueueStreamDispatch(pMgmt->pSnode, pMsg);
} else {
return smPutNodeMsgToUniqueQueue(pMgmt, pMsg);
taosWriteQitem(pWorker->queue, pMsg);
}
return 0;
}

3
source/dnode/mgmt/test/sut/inc/server.h
View File

@ -20,7 +20,8 @@ class TestServer {
public:
bool Start();
void Stop();
bool runnning;
private:
TdThread threadId;
};

18
source/dnode/mgmt/test/sut/src/server.cpp
View File

@ -16,8 +16,18 @@
#include "sut.h"
void* serverLoop(void* param) {
dmInit();
dmRun();
TestServer* server = (TestServer*)param;
server->runnning = false;
if (dmInit() != 0) {
return NULL;
}
server->runnning = true;
if (dmRun() != 0) {
return NULL;
}
dmCleanup();
return NULL;
}
@ -26,10 +36,10 @@ bool TestServer::Start() {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
taosThreadCreate(&threadId, &thAttr, serverLoop, NULL);
taosThreadCreate(&threadId, &thAttr, serverLoop, this);
taosThreadAttrDestroy(&thAttr);
taosMsleep(2100);
return true;
return runnning;
}
void TestServer::Stop() {

5
source/dnode/mgmt/test/sut/src/sut.cpp
View File

@ -53,7 +53,10 @@ void Testbase::Init(const char* path, int16_t port) {
taosMkDir(path);
InitLog(TD_TMP_DIR_PATH "td");
server.Start();
if (!server.Start()) {
printf("failed to start server, exit\n");
exit(0);
};
client.Init("root", "taosdata");
showRsp = NULL;
}

4
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -80,8 +80,8 @@ typedef struct {
} SProfileMgmt;
typedef struct {
SRWLatch lock;
char email[TSDB_FQDN_LEN];
TdThreadMutex lock;
char email[TSDB_FQDN_LEN];
} STelemMgmt;
typedef struct {

6
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -32,7 +32,7 @@
#include "tname.h"
#include "tuuid.h"
extern bool tsSchedStreamToSnode;
extern bool tsDeployOnSnode;
static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
int32_t childId = taosArrayGetSize(pArray);
@ -190,7 +190,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan,
pTask->nodeId = SNODE_HANDLE;
pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode);
plan->execNode.nodeId = 0;
plan->execNode.nodeId = SNODE_HANDLE;
plan->execNode.epSet = pTask->epSet;
if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
@ -373,7 +373,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return -1;
}
if (tsSchedStreamToSnode) {
if (tsDeployOnSnode) {
SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode);
if (pSnode == NULL) {
SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid);

12
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -127,8 +127,10 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
if (!tsEnableTelem) return 0;
taosWLockLatch(&pMgmt->lock);
taosThreadMutexLock(&pMgmt->lock);
char* pCont = mndBuildTelemetryReport(pMnode);
taosThreadMutexUnlock(&pMgmt->lock);
if (pCont != NULL) {
if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) {
mError("failed to send telemetry report");
@ -137,18 +139,20 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) {
}
taosMemoryFree(pCont);
}
taosWUnLockLatch(&pMgmt->lock);
return 0;
}
int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
taosInitRWLatch(&pMgmt->lock);
taosThreadMutexInit(&pMgmt->lock, NULL);
taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
return 0;
}
void mndCleanupTelem(SMnode* pMnode) {}
void mndCleanupTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
taosThreadMutexDestroy(&pMgmt->lock);
}

93
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -60,7 +60,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg);
mndSetMsgHandle(pMnode, TDMT_MND_MERGE_VGROUP, mndProcessSplitVgroupMsg);
mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg);
mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups);
@ -617,35 +617,6 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
return epset;
}
static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
SVgObj *pVgroup = pObj;
int64_t uid = *(int64_t *)p1;
int8_t *pReplica = p2;
int32_t *pNumOfVgroups = p3;
if (pVgroup->dbUid == uid) {
*pReplica = TMAX(*pReplica, pVgroup->replica);
(*pNumOfVgroups)++;
}
return true;
}
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
*pReplica = 1;
*pNumOfVgroups = 0;
sdbTraverse(pSdb, SDB_VGROUP, mndGetVgroupMaxReplicaFp, &pDb->uid, pReplica, pNumOfVgroups);
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -1670,35 +1641,81 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
if (newVg1.replica == 1) {
if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, -1) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
} else if (newVg1.replica == 3) {
SVnodeGid del1 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, -1) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
} else {
goto _OVER;
}
mInfo("vgId:%d, vgroup info after adjust replica, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
SVgObj newVg2 = {0};
memcpy(&newVg1, &newVg2, sizeof(SVgObj));
memcpy(&newVg2, &newVg1, sizeof(SVgObj));
newVg1.replica = 1;
newVg1.hashEnd = (newVg1.hashBegin + newVg1.hashEnd) / 2;
newVg1.hashEnd = newVg1.hashBegin / 2 + newVg1.hashEnd / 2;
memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));
newVg2.replica = 1;
newVg2.hashBegin = newVg1.hashEnd + 1;
memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid));
memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));
memset(&newVg2.vnodeGid[1], 0, sizeof(SVnodeGid));
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId,
newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER;
// adjust vgroup
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
#if 0
// adjust vgroup replica
if (pDb->cfg.replications != newVg1.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
}
if (pDb->cfg.replications != newVg2.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
}
#endif
{
pRaw = mndVgroupActionEncode(&newVg1);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
{
pRaw = mndVgroupActionEncode(&newVg2);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId,
newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
_OVER:
mndTransDrop(pTrans);

84
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -367,6 +367,80 @@ bool sdbTraverseFail(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, in
return false;
}
#ifndef WINDOWS
TEST_F(MndTestSdb, 00_API) {
SMnode mnode = {0};
SSdbOpt opt = {0};
opt.pMnode = &mnode;
opt.path = TD_TMP_DIR_PATH "mnode_test_sdb";
taosRemoveDir(opt.path);
SSdb *pSdb = sdbInit(&opt);
SSdbTable table = {.sdbType = SDB_USER, .keyType = SDB_KEY_BINARY};
sdbSetTable(pSdb, table);
// sdbRow.c
SSdbRow *pRow1 = sdbAllocRow(-128);
ASSERT_EQ(pRow1 == NULL, 1);
void *pRow2 = sdbGetRowObj(NULL);
ASSERT_EQ(pRow2 == NULL, 1);
//sdbRaw.c
SStrObj strObj;
SSdbRaw *pRaw1 = NULL;
strSetDefault(&strObj, 1);
pRaw1 = strEncode(&strObj);
int32_t id = sdbGetIdFromRaw(pSdb, pRaw1);
ASSERT_EQ(id, -2);
SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128);
ASSERT_EQ(pRaw2 == NULL, 1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), -1);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), -1);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), -1);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), -1);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), -1);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), -1);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), -1);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), -1);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), -1);
ASSERT_EQ(sdbGetRawTotalSize(NULL), -1);
// sdbHash.c
EXPECT_STREQ(sdbTableName((ESdbType)100), "undefine");
EXPECT_STREQ(sdbStatusName((ESdbStatus)100), "undefine");
ASSERT_EQ(sdbGetTableVer(pSdb, (ESdbType)100), -1);
SSdbRaw *pRaw3 = sdbAllocRaw((ESdbType)-12, 1, 128);
ASSERT_NE(sdbWriteWithoutFree(pSdb, pRaw3), 0);
pSdb->hashObjs[1] = NULL;
SSdbRaw *pRaw4 = sdbAllocRaw((ESdbType)1, 1, 128);
ASSERT_NE(sdbWriteWithoutFree(pSdb, pRaw4), 0);
}
#endif
TEST_F(MndTestSdb, 01_Write_Str) {
void *pIter = NULL;
int32_t num = 0;
@ -885,8 +959,8 @@ TEST_F(MndTestSdb, 01_Read_Str) {
ASSERT_EQ(sdbWrite(pSdb, pRaw), 0);
pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key);
ASSERT_EQ(pI32Obj, nullptr);
int32_t code = terrno;
ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_DROPPING);
// int32_t code = terrno;
// ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_DROPPING);
}
{
@ -899,8 +973,8 @@ TEST_F(MndTestSdb, 01_Read_Str) {
ASSERT_EQ(sdbWrite(pSdb, pRaw), 0);
pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key);
ASSERT_EQ(pI32Obj, nullptr);
int32_t code = terrno;
ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING);
// int32_t code = terrno;
// ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING);
}
{
@ -934,4 +1008,4 @@ TEST_F(MndTestSdb, 01_Read_Str) {
sdbCleanup(pSdb);
ASSERT_EQ(mnode.insertTimes, 9);
ASSERT_EQ(mnode.deleteTimes, 9);
}
}

3
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -654,15 +654,12 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
}
if (config > 0) {
ASSERT(pSdb->commitConfig == config);
pSdb->commitConfig = config;
}
if (term > 0) {
ASSERT(pSdb->commitTerm == term);
pSdb->commitTerm = term;
}
if (index > 0) {
ASSERT(pSdb->commitIndex == index);
pSdb->commitIndex = index;
}

4
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -241,9 +241,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) {
SdbDecodeFp decodeFp = pSdb->decodeFps[pRaw->type];
SSdbRow *pRow = (*decodeFp)(pRaw);
if (pRow == NULL) {
return terrno;
}
if (pRow == NULL) return terrno;
pRow->type = pRaw->type;

4
source/dnode/snode/inc/sndInt.h
View File

@ -31,7 +31,9 @@ extern "C" {
#endif
typedef struct SSnode {
SMsgCb msgCb;
char* path;
SStreamMeta* pMeta;
SMsgCb msgCb;
} SSnode;
#if 0

344
source/dnode/snode/src/snode.c
View File

@ -15,197 +15,197 @@
#include "executor.h"
#include "sndInt.h"
#include "tstream.h"
#include "tuuid.h"
/*SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { return NULL; }*/
/*void sndClose(SSnode *pSnode) {}*/
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { return 0; }
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { return 0; }
void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) {
char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamDispatchReq req;
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
if (tDecodeStreamDispatchReq(&decoder, &req) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
tDecoderClear(&decoder);
goto FAIL;
}
tDecoderClear(&decoder);
int32_t taskId = req.taskId;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessDispatchReq(pTask, &req, &rsp, false);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
return;
}
FAIL:
if (pMsg->info.handle == NULL) return;
SRpcMsg rsp = {
.code = code,
.info = pMsg->info,
};
tmsgSendRsp(&rsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->inputQueue = streamQueueOpen();
pTask->outputQueue = streamQueueOpen();
if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) {
return -1;
}
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
pTask->pMsgCb = &pSnode->msgCb;
pTask->startVer = ver;
pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
if (pTask->pState == NULL) {
return -1;
}
SReadHandle mgHandle = {
.vnode = NULL,
.numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
.pStateBackend = pTask->pState,
};
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
ASSERT(pTask->exec.executor);
return 0;
}
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pSnode->path = strdup(path);
if (pSnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb;
#if 0
pSnode->pMeta = sndMetaNew();
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE);
if (pSnode->pMeta == NULL) {
taosMemoryFree(pSnode);
return NULL;
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
#endif
return pSnode;
FAIL:
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode);
return NULL;
}
void sndClose(SSnode *pSnode) {
/*sndMetaDelete(pSnode->pMeta);*/
streamMetaClose(pSnode->pMeta);
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode);
}
int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; }
#if 0
SStreamMeta *sndMetaNew() {
SStreamMeta *pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
if (pMeta == NULL) {
return NULL;
}
pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pMeta->pHash == NULL) {
taosMemoryFree(pMeta);
return NULL;
}
return pMeta;
}
void sndMetaDelete(SStreamMeta *pMeta) {
taosHashCleanup(pMeta->pHash);
taosMemoryFree(pMeta);
}
int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) {
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *));
}
SStreamTask *sndMetaGetTask(SStreamMeta *pMeta, int32_t taskId) {
return taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
}
int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) {
SStreamTask *pTask = taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
if (pTask == NULL) {
return -1;
}
taosMemoryFree(pTask->exec.qmsg);
// TODO:free executor
taosMemoryFree(pTask);
return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t));
}
static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
int32_t code;
// 1.deserialize msg and build task
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) {
return -1;
}
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t *)msg, msgLen);
if (tDecodeSStreamTask(&decoder, pTask) < 0) {
ASSERT(0);
code = tDecodeSStreamTask(&decoder, pTask);
if (code < 0) {
tDecoderClear(&decoder);
taosMemoryFree(pTask);
return -1;
}
tDecoderClear(&decoder);
pTask->execStatus = TASK_EXEC_STATUS__IDLE;
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
pTask->inputQueue = streamQueueOpen();
pTask->outputQueue = streamQueueOpen();
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_INPUT_STATUS__NORMAL;
// 2.save task
code = streamMetaAddTask(pSnode->pMeta, -1, pTask);
if (code < 0) {
return -1;
}
if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) goto FAIL;
pTask->pMsgCb = &pNode->msgCb;
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
ASSERT(pTask->exec.executor);
streamSetupTrigger(pTask);
qInfo("deploy stream: stream id %" PRId64 " task id %d child id %d on snode", pTask->streamId, pTask->taskId,
pTask->selfChildId);
taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));
// 3.go through recover steps to fill history
if (pTask->fillHistory) {
streamSetParamForRecover(pTask);
streamAggRecoverPrepare(pTask);
}
return 0;
FAIL:
if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
if (pTask) taosMemoryFree(pTask);
return -1;
}
static int32_t sndProcessTaskRunReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
return streamMetaRemoveTask(pSnode->pMeta, pReq->taskId);
}
int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) {
SStreamTaskRunReq *pReq = pMsg->pCont;
int32_t taskId = pReq->taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessRunReq(pTask);
return 0;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
streamProcessRunReq(pTask);
return 0;
} else {
return -1;
}
}
static int32_t sndProcessTaskDispatchReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) {
char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
SStreamDispatchReq req;
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen);
tDecodeStreamDispatchReq(&decoder, &req);
int32_t taskId = req.taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessDispatchReq(pTask, &req, &rsp, true);
return 0;
}
int32_t taskId = req.taskId;
static int32_t sndProcessTaskRecoverReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamTaskRecoverReq *pReq = pMsg->pCont;
int32_t taskId = pReq->taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessRecoverReq(pTask, pReq, pMsg);
return 0;
}
static int32_t sndProcessTaskDispatchRsp(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t taskId = pRsp->taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessDispatchRsp(pTask, pRsp);
return 0;
}
static int32_t sndProcessTaskRecoverRsp(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamTaskRecoverRsp *pRsp = pMsg->pCont;
int32_t taskId = pRsp->rspTaskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessRecoverRsp(pTask, pRsp);
return 0;
}
static int32_t sndProcessTaskDropReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msg = pMsg->pCont;
int32_t msgLen = pMsg->contLen;
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
int32_t code = taosHashRemove(pMeta->pHash, &pReq->taskId, sizeof(int32_t));
ASSERT(code == 0);
if (code == 0) {
// sendrsp
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessDispatchReq(pTask, &req, &rsp, exec);
return 0;
} else {
return -1;
}
return code;
return 0;
}
static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
@ -213,53 +213,64 @@ static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder);
int32_t taskId = req.dstTaskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessRetrieveReq(pTask, &req, &rsp);
tDeleteStreamRetrieveReq(&req);
return 0;
} else {
return -1;
}
}
int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) {
SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t taskId = pRsp->taskId;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
streamProcessDispatchRsp(pTask, pRsp);
return 0;
} else {
return -1;
}
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessRetrieveReq(pTask, &req, &rsp);
return 0;
}
static int32_t sndProcessTaskRetrieveRsp(SSnode *pNode, SRpcMsg *pMsg) {
int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) {
//
return 0;
}
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
// stream deploy
// stream stop/resume
// operator exec
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead);
switch (pMsg->msgType) {
case TDMT_STREAM_TASK_DEPLOY:
return sndProcessTaskDeployReq(pSnode, pMsg);
return sndProcessTaskDeployReq(pSnode, pReq, len);
case TDMT_STREAM_TASK_DROP:
return sndProcessTaskDropReq(pSnode, pMsg);
return sndProcessTaskDropReq(pSnode, pReq, len);
default:
ASSERT(0);
}
return 0;
}
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
switch (pMsg->msgType) {
case TDMT_STREAM_TASK_RUN:
return sndProcessTaskRunReq(pSnode, pMsg);
case TDMT_STREAM_TASK_DISPATCH:
return sndProcessTaskDispatchReq(pSnode, pMsg);
case TDMT_STREAM_TASK_RECOVER:
return sndProcessTaskRecoverReq(pSnode, pMsg);
return sndProcessTaskDispatchReq(pSnode, pMsg, true);
case TDMT_STREAM_RETRIEVE:
return sndProcessTaskRecoverReq(pSnode, pMsg);
return sndProcessTaskRetrieveReq(pSnode, pMsg);
case TDMT_STREAM_TASK_DISPATCH_RSP:
return sndProcessTaskDispatchRsp(pSnode, pMsg);
case TDMT_STREAM_TASK_RECOVER_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE_RSP:
return sndProcessTaskRetrieveRsp(pSnode, pMsg);
default:
@ -267,4 +278,3 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
}
return 0;
}
#endif

3
source/dnode/vnode/inc/vnode.h
View File

@ -155,14 +155,17 @@ typedef struct STsdbReader STsdbReader;
int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num);
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
STsdbReader **ppReader, const char *idstr);
void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader);
bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
bool tsdbIsAscendingOrder(STsdbReader *pReader);
void *tsdbGetIdx(SMeta *pMeta);
void *tsdbGetIvtIdx(SMeta *pMeta);
uint64_t getReaderMaxVersion(STsdbReader *pReader);

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -719,7 +719,6 @@ typedef struct SCacheRowsReader {
STableKeyInfo *pTableList; // table id list
int32_t numOfTables;
SSttBlockLoadInfo *pLoadInfo;
STsdbReadSnap *pReadSnap;
SDataFReader *pDataFReader;

6
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -701,6 +701,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
#endif
for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
SSDataBlock *output = taosArrayGetP(pResList, i);
smaDebug("uid:%ld, groupid:%ld", output->info.uid, output->info.groupId);
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
SSubmitReq *pReq = NULL;
@ -713,12 +715,12 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
taosMemoryFreeClear(pReq);
smaError("vgId:%d, process submit req for rsma table suid:%" PRIu64 ", uid:%"PRIu64", level %" PRIi8 " failed since %s",
smaError("vgId:%d, process submit req for rsma suid:%"PRIu64", uid:%" PRIu64 " level %" PRIi8 " failed since %s",
SMA_VID(pSma), suid, output->info.groupId, pItem->level, terrstr());
goto _err;
}
smaDebug("vgId:%d, process submit req for rsma table suid:%" PRIu64 ", uid:%"PRIu64", level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%"PRIu64", level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
SMA_VID(pSma), suid, output->info.groupId, pItem->level, output->info.version, htonl(pReq->header.contLen));
taosMemoryFreeClear(pReq);

3
source/dnode/vnode/src/tq/tq.c
View File

@ -1272,7 +1272,6 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
}
int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
//
SStreamTaskRunReq* pReq = pMsg->pCont;
int32_t taskId = pReq->taskId;
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
@ -1285,7 +1284,6 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
}
int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
ASSERT(0);
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
@ -1349,7 +1347,6 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
return streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
}

2
source/dnode/vnode/src/tq/tqSink.c
View File

@ -42,7 +42,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl
if (metaGetTableEntryByName(&mr, name) < 0) {
metaReaderClear(&mr);
taosMemoryFree(name);
return -1;
continue;
}
int64_t uid = mr.me.uid;

2
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -333,7 +333,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
code = TSDB_CODE_INVALID_PARA;
}
_end:
_end:
tsdbDataFReaderClose(&pr->pDataFReaderLast);
tsdbDataFReaderClose(&pr->pDataFReader);

2
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -290,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
// only apply to the child tables, ordinary tables will not incur this filter procedure.
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
if (size > 1) {
if (size >= 1) {
SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);

42
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -426,7 +426,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
return true;
}
_err:
_err:
return false;
}
@ -528,7 +528,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
*ppReader = pReader;
return code;
_end:
_end:
tsdbReaderClose(pReader);
*ppReader = NULL;
return code;
@ -580,7 +580,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader,
pReader->cost.headFileLoadTime += (et1 - st) / 1000.0;
_end:
_end:
taosArrayDestroy(aBlockIdx);
return code;
}
@ -954,7 +954,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1;
tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain,
unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr);
@ -981,7 +981,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, code:%s %s",
", rows:%d, code:%s %s",
pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow,
tstrerror(code), pReader->idStr);
return code;
@ -990,7 +990,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64
", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow,
pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr);
@ -1405,8 +1405,8 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock
// log the reason why load the datablock for profile
if (loadDataBlock) {
tsdbDebug("%p uid:%" PRIu64
" need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
"overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
" need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
"overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock,
pReader->idStr);
@ -1441,7 +1441,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64
" - %" PRId64 " %s",
" - %" PRId64 " %s",
pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey,
pReader->idStr);
@ -2039,7 +2039,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid,
@ -2060,7 +2060,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid,
@ -2295,7 +2295,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
}
}
_end:
_end:
pResBlock->info.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
blockDataUpdateTsWindow(pResBlock, 0);
@ -2307,7 +2307,7 @@ _end:
if (pResBlock->info.rows > 0) {
tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
" rows:%d, elapsed time:%.2f ms %s",
" rows:%d, elapsed time:%.2f ms %s",
pReader, pResBlock->info.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, el, pReader->idStr);
}
@ -2388,13 +2388,13 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
taosArrayDestroy(pDelData);
pBlockScanInfo->iter.index =
ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index;
pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index;
pBlockScanInfo->lastBlockDelIndex = pBlockScanInfo->iter.index;
return code;
_err:
_err:
taosArrayDestroy(pDelData);
return code;
}
@ -2725,7 +2725,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
if (pBlockIter->numOfBlocks == 0) {
_begin:
_begin:
code = doLoadLastBlockSequentially(pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -2814,8 +2814,8 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
int8_t precision = pVnode->config.tsdbCfg.precision;
int64_t now = taosGetTimestamp(precision);
int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
: 1000000L);
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
: 1000000L);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
@ -3634,7 +3634,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code;
_err:
_err:
tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
return code;
}
@ -3976,7 +3976,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
}
tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64
" in query %s",
" in query %s",
pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey,
pReader->idStr);
@ -4167,7 +4167,7 @@ int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr
}
tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
_exit:
_exit:
return code;
}

13
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -369,8 +369,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_FETCH_RSP:
return qWorkerProcessRspMsg(pVnode, pVnode->pQuery, pMsg, 0);
//case TDMT_SCH_CANCEL_TASK:
// return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
// case TDMT_SCH_CANCEL_TASK:
// return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_DROP_TASK:
return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_QUERY_HEARTBEAT:
@ -385,9 +385,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return tqProcessPollReq(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_RUN:
return tqProcessTaskRunReq(pVnode->pTq, pMsg);
#if 0
case TDMT_STREAM_TASK_DISPATCH:
// return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, pInfo->workerId != 0);
return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true);
#endif
/*case TDMT_STREAM_TASK_RECOVER:*/
/*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/
case TDMT_STREAM_RETRIEVE:
@ -1163,8 +1164,10 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
for (int32_t i = 0; i < sz; i++) {
SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i);
int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts);
if (code) {
// TODO
if (code < 0) {
terrno = code;
vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64,
TD_VID(pVnode), terrstr(), deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts);
}
}
taosArrayDestroy(deleteReq.deleteReqs);

538
source/libs/catalog/test/catalogTests.cpp
View File

@ -56,6 +56,13 @@ enum {
CTGT_RSP_STBMETA,
CTGT_RSP_MSTBMETA,
CTGT_RSP_INDEXINFO_E,
CTGT_RSP_USERAUTH,
CTGT_RSP_TBLCFG,
CTGT_RSP_TBLINDEX,
CTGT_RSP_DBCFG,
CTGT_RSP_QNODELIST,
CTGT_RSP_UDF,
CTGT_RSP_SVRVER,
CTGT_RSP_TBMETA_NOT_EXIST,
};
@ -71,6 +78,10 @@ int32_t ctgTestVgVersion = 1;
int32_t ctgTestVgNum = 10;
int32_t ctgTestColNum = 2;
int32_t ctgTestTagNum = 1;
int32_t ctgTestQnodeNum = 3;
int32_t ctgTestIndexNum = 3;
int32_t ctgTestFuncNum = 2;
int32_t ctgTestFuncType = 3;
int32_t ctgTestSVersion = 1;
int32_t ctgTestTVersion = 1;
int32_t ctgTestSuid = 2;
@ -82,6 +93,7 @@ char *ctgTestDbname = "1.db1";
char *ctgTestTablename = "table1";
char *ctgTestCTablename = "ctable1";
char *ctgTestSTablename = "stable1";
char *ctgTestUsername = "user1";
char *ctgTestCurrentCTableName = NULL;
char *ctgTestCurrentTableName = NULL;
char *ctgTestCurrentSTableName = NULL;
@ -311,6 +323,7 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
strcpy(addr->fqdn, "a0");
addr->port = n + 22;
}
vg.numOfTable = i % 2;
taosArrayPush(usedbRsp.pVgroupInfos, &vg);
}
@ -531,6 +544,197 @@ void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMs
}
void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SGetUserAuthRsp userRsp = {0};
strcpy(userRsp.user, ctgTestUsername);
userRsp.version = 1;
userRsp.superAuth = 1;
int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSGetUserAuthRsp(pReq, contLen, &userRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
}
void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableCfgRsp tblRsp = {0};
strcpy(tblRsp.tbName, ctgTestTablename);
tblRsp.numOfColumns = ctgTestColNum;
tblRsp.pSchemas = (SSchema *)taosMemoryMalloc((tblRsp.numOfTags + tblRsp.numOfColumns) * sizeof(SSchema));
SSchema *s = NULL;
s = &tblRsp.pSchemas[0];
s->type = TSDB_DATA_TYPE_TIMESTAMP;
s->colId = 1;
s->bytes = 8;
strcpy(s->name, "ts");
s = &tblRsp.pSchemas[1];
s->type = TSDB_DATA_TYPE_INT;
s->colId = 2;
s->bytes = 4;
strcpy(s->name, "col1");
int32_t contLen = tSerializeSTableCfgRsp(NULL, 0, &tblRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSTableCfgRsp(pReq, contLen, &tblRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
tFreeSTableCfgRsp(&tblRsp);
}
void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableIndexRsp tblRsp = {0};
strcpy(tblRsp.tbName, ctgTestSTablename);
tblRsp.pIndex = taosArrayInit(ctgTestIndexNum, sizeof(STableIndexInfo));
STableIndexInfo info = {0};
for (int32_t i = 0; i < ctgTestIndexNum; ++i) {
info.interval = 1 + i;
info.expr = (char*)taosMemoryCalloc(1, 10);
taosArrayPush(tblRsp.pIndex, &info);
}
int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSTableIndexRsp(pReq, contLen, &tblRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
tFreeSTableIndexRsp(&tblRsp);
}
void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
SDbCfgRsp dbRsp = {0};
dbRsp.numOfVgroups = ctgTestVgNum;
int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSDbCfgRsp(pReq, contLen, &dbRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
}
void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SQnodeListRsp qlistRsp = {0};
qlistRsp.qnodeList = taosArrayInit(10, sizeof(SQueryNodeLoad));
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad nodeLoad = {0};
nodeLoad.addr.nodeId = i;
(void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad);
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSQnodeListRsp(pReq, rspLen, &qlistRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
tFreeSQnodeListRsp(&qlistRsp);
}
void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SRetrieveFuncRsp funcRsp = {0};
funcRsp.numOfFuncs = 1;
funcRsp.pFuncInfos = taosArrayInit(1, sizeof(SFuncInfo));
SFuncInfo funcInfo = {0};
strcpy(funcInfo.name, "func1");
funcInfo.funcType = ctgTestFuncType;
(void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo);
int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSRetrieveFuncRsp(pReq, rspLen, &funcRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
tFreeSRetrieveFuncRsp(&funcRsp);
}
void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SServerVerRsp verRsp = {0};
strcpy(verRsp.ver, "1.0");
int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSServerVerRsp(pReq, rspLen, &verRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
}
void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (pMsg->msgType) {
case TDMT_MND_USE_DB:
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_VND_TABLE_CFG:
case TDMT_MND_TABLE_CFG:
ctgTestRspTableCfg(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_GET_TABLE_INDEX:
ctgTestRspTableIndex(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_GET_DB_CFG:
ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_QNODE_LIST:
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_RETRIEVE_FUNC:
ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_SERVER_VERSION:
ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
break;
default:
break;
}
return;
}
void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (ctgTestRspFunc[ctgTestRspIdx]) {
case CTGT_RSP_VGINFO:
@ -551,10 +755,32 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp
case CTGT_RSP_INDEXINFO_E:
ctgTestRspErrIndexInfo(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_USERAUTH:
ctgTestRspUserAuth(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBLCFG:
ctgTestRspTableCfg(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBMETA_NOT_EXIST:
ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBLINDEX:
ctgTestRspTableIndex(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_DBCFG:
ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_QNODELIST:
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_UDF:
ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_SVRVER:
ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
break;
default:
ctgTestRspAuto(shandle, pEpSet, pMsg, pRsp);
break;
}
@ -976,10 +1202,24 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
taosMsleep(50);
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n != 1) {
taosMsleep(50);
} else {
break;
}
}
memset(&vgInfo, 0, sizeof(vgInfo));
bool exists = false;
code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
ASSERT_EQ(exists, true);
ctgTestSetRspTableMeta();
STableMeta *tableMeta = NULL;
@ -1020,6 +1260,18 @@ TEST(tableMeta, normalTable) {
taosMemoryFree(tableMeta);
tableMeta = NULL;
catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
@ -1216,6 +1468,21 @@ TEST(tableMeta, superTableCase) {
}
}
tableMeta = NULL;
code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 0);
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
ctgTestSetRspCTableMeta();
tableMeta = NULL;
@ -2156,6 +2423,15 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_TRUE(code != 0);
int32_t dbVer = 0;
int64_t dbId = 0;
int32_t tbNum = 0;
code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum);
ASSERT_EQ(code, 0);
ASSERT_EQ(dbVer, ctgTestVgVersion);
ASSERT_EQ(dbId, ctgTestDbId);
ASSERT_EQ(tbNum, ctgTestVgNum / 2);
ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
ASSERT_EQ(code, 0);
@ -2415,6 +2691,264 @@ TEST(rentTest, allRent) {
catalogDestroy();
}
TEST(apiTest, catalogRefreshDBVgInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
code = catalogRefreshDBVgInfo(pCtg, mockPointer, ctgTestDbname);
ASSERT_EQ(code, 0);
catalogDestroy();
}
TEST(apiTest, catalogChkAuth_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_USERAUTH;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
bool pass = false;
code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass);
ASSERT_EQ(code, 0);
ASSERT_EQ(pass, true);
catalogDestroy();
}
TEST(apiTest, catalogRefreshGetTableCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
STableCfg *pCfg = NULL;
code = catalogRefreshGetTableCfg(pCtg, mockPointer, &n, &pCfg);
ASSERT_EQ(code, 0);
ASSERT_TRUE(NULL != pCfg);
ASSERT_EQ(pCfg->numOfColumns, ctgTestColNum);
tFreeSTableCfgRsp((STableCfgRsp *)pCfg);
taosMemoryFree(pCfg);
catalogDestroy();
}
TEST(apiTest, catalogGetTableIndex_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_TBLINDEX;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
SArray *pRes = NULL;
code = catalogGetTableIndex(pCtg, mockPointer, &n, &pRes);
ASSERT_EQ(code, 0);
ASSERT_TRUE(NULL != pRes);
ASSERT_EQ(taosArrayGetSize(pRes), ctgTestIndexNum);
taosArrayDestroyEx(pRes, tFreeSTableIndexInfo);
catalogDestroy();
}
TEST(apiTest, catalogGetDBCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_DBCFG;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
SDbCfgInfo cfgInfo = {0};
code = catalogGetDBCfg(pCtg, mockPointer, ctgTestDbname, &cfgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(cfgInfo.numOfVgroups, ctgTestVgNum);
catalogDestroy();
}
TEST(apiTest, catalogGetQnodeList_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_QNODELIST;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SArray *qnodeList = taosArrayInit(10, sizeof(SQueryNodeLoad));
code = catalogGetQnodeList(pCtg, mockPointer, qnodeList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize(qnodeList), ctgTestQnodeNum);
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad * pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i);
ASSERT_EQ(pLoad->addr.nodeId, i);
}
catalogDestroy();
}
TEST(apiTest, catalogGetUdfInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_UDF;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SFuncInfo funcInfo = {0};
code = catalogGetUdfInfo(pCtg, mockPointer, "func1", &funcInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(funcInfo.funcType, ctgTestFuncType);
catalogDestroy();
}
TEST(apiTest, catalogGetServerVersion_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_SVRVER;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
char* ver = NULL;
code = catalogGetServerVersion(pCtg, mockPointer, &ver);
ASSERT_EQ(code, 0);
ASSERT_TRUE(0 == strcmp(ver, "1.0"));
catalogDestroy();
}
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);

47
source/libs/executor/inc/executorimpl.h
View File

@ -174,12 +174,13 @@ typedef struct {
} SSchemaInfo;
typedef struct SExecTaskInfo {
STaskIdInfo id;
uint32_t status;
int32_t code;
STimeWindow window;
STaskCostInfo cost;
int64_t owner; // if it is in execution
STaskIdInfo id;
uint32_t status;
STimeWindow window;
STaskCostInfo cost;
int64_t owner; // if it is in execution
int32_t code;
int64_t version; // used for stream to record wal version
SStreamTaskInfo streamInfo;
SSchemaInfo schemaInfo;
@ -340,21 +341,23 @@ typedef struct STableScanInfo {
} STableScanInfo;
typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
int32_t tableStartIndex;
int32_t tableEndIndex;
bool hasGroupId;
uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT*
SReadHandle readHandle;
int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort
SArray* pSortInfo;
SSortHandle* pSortHandle;
SSDataBlock* pSortInputBlock;
int64_t startTs; // sort start time
SArray* sortSourceParams;
SLimitInfo limitInfo;
STableListInfo* tableListInfo;
int32_t tableStartIndex;
int32_t tableEndIndex;
bool hasGroupId;
uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT*
SArray* queryConds; // array of queryTableDataCond
STsdbReader* pReader;
SReadHandle readHandle;
int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort
SArray* pSortInfo;
SSortHandle* pSortHandle;
SSDataBlock* pSortInputBlock;
int64_t startTs; // sort start time
SArray* sortSourceParams;
SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder;
int64_t numOfRows;
SScanInfo scanInfo;
@ -1074,7 +1077,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
bool groupbyTbname(SNodeList* pGroupList);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey, bool groupSort);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort);
void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo);

2
source/libs/executor/src/cachescanoperator.c
View File

@ -106,7 +106,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pOperator->cost.openCost = 0;
return pOperator;
_error:
_error:
pTaskInfo->code = code;
destroyLastrowScanOperator(pInfo);
taosMemoryFree(pOperator);

8
source/libs/executor/src/executil.c
View File

@ -337,8 +337,8 @@ static EDealRes getColumn(SNode** pNode, void* pContext) {
taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
pSColumnNode->slotId = pData->index++;
SColumnInfo cInfo = {.colId = pSColumnNode->colId,
.type = pSColumnNode->node.resType.type,
.bytes = pSColumnNode->node.resType.bytes};
.type = pSColumnNode->node.resType.type,
.bytes = pSColumnNode->node.resType.bytes};
#if TAG_FILTER_DEBUG
qDebug("tagfilter build column info, slotId:%d, colId:%d, type:%d", pSColumnNode->slotId, cInfo.colId, cInfo.type);
#endif
@ -507,7 +507,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
// int64_t st2 = taosGetTimestampUs();
// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
end:
end:
taosHashCleanup(tags);
taosHashCleanup(ctx.colHash);
taosArrayDestroy(ctx.cInfoList);
@ -747,7 +747,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
// int64_t st2 = taosGetTimestampUs();
// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
end:
end:
taosMemoryFreeClear(keyBuf);
taosHashCleanup(tags);
taosHashCleanup(ctx.colHash);

2
source/libs/executor/src/executor.c
View File

@ -440,7 +440,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
_error:
_error:
// if failed to add ref for all tables in this query, abort current query
return code;
}

42
source/libs/executor/src/executorimpl.c
View File

@ -494,7 +494,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
// todo: refactor this
if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) {
pInput->pPTS = pInput->pData[j]; // in case of merge function, this is not always the ts column data.
// ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP);
// ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP);
}
ASSERT(pInput->pData[j] != NULL);
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
@ -815,7 +815,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
// abort current query execution.
if (pTaskInfo->owner != 0 &&
((taosGetTimestampSec() - pTaskInfo->cost.start / 1000) > 10 * getMaximumIdleDurationSec())
/*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
/*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
assert(pTaskInfo->cost.start != 0);
// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
@ -1963,7 +1963,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
if (pRsp->numOfRows == 0) {
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
@ -1990,10 +1990,10 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
if (pRsp->completed == 1) {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
" execId:%d"
" index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
", total:%.2f Kb,"
" completed:%d try next %d/%" PRIzu,
" execId:%d"
" index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
", total:%.2f Kb,"
" completed:%d try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks,
pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0,
completed + 1, i + 1, totalSources);
@ -2001,7 +2001,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
} else {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
" execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb",
" execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks,
pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0);
}
@ -2028,7 +2028,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
sched_yield();
}
_error:
_error:
pTaskInfo->code = code;
}
@ -2089,7 +2089,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
if (pRsp->numOfRows == 0) {
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 " try next",
", totalRows:%" PRIu64 " try next",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
pDataInfo->totalRows, pLoadInfo->totalRows);
@ -2106,7 +2106,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
if (pRsp->completed == 1) {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
totalSources);
@ -2115,7 +2115,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
pExchangeInfo->current += 1;
} else {
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
", totalBytes:%" PRIu64,
", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
pLoadInfo->totalRows, pLoadInfo->totalSize);
}
@ -2303,7 +2303,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL);
return pOperator;
_error:
_error:
if (pInfo != NULL) {
doDestroyExchangeOperatorInfo(pInfo);
}
@ -3040,7 +3040,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
return pOperator;
_error:
_error:
if (pInfo != NULL) {
destroyAggOperatorInfo(pInfo);
}
@ -3210,8 +3210,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
SInterval* pInterval =
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
: &((SIntervalAggOperatorInfo*)downstream->info)->interval;
? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
: &((SIntervalAggOperatorInfo*)downstream->info)->interval;
int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
int32_t type = convertFillType(pPhyFillNode->mode);
@ -3256,7 +3256,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
_error:
_error:
if (pInfo != NULL) {
destroyFillOperatorInfo(pInfo);
}
@ -3922,7 +3922,7 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32
*length = *(int32_t*)(*result);
}
_downstream:
_downstream:
for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
if (code != TDB_CODE_SUCCESS) {
@ -4042,7 +4042,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
return code;
_complete:
_complete:
taosMemoryFree(sql);
doDestroyTask(*pTaskInfo);
terrno = code;
@ -4181,8 +4181,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
int32_t size = 0;
void* pVal = NULL;
SWinKey key = {
.ts = *(TSKEY*)pPos->key,
.groupId = pPos->groupId,
.ts = *(TSKEY*)pPos->key,
.groupId = pPos->groupId,
};
int32_t code = streamStateGet(pState, &key, &pVal, &size);
ASSERT(code == 0);

1
source/libs/executor/src/projectoperator.c
View File

@ -159,7 +159,6 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S
// reset the value for a new group data
// existing rows that belongs to previous group.
// TODO refactor with doTableScan
pLimitInfo->numOfOutputRows = 0;
pLimitInfo->remainOffset = pLimitInfo->limit.offset;
}

52
source/libs/executor/src/scanoperator.c
View File

@ -621,12 +621,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
binfo.capacity = binfo.rows;
blockDataEnsureCapacity(pBlock, binfo.rows);
pBlock->info = binfo;
ASSERT(binfo.uid != 0);
uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
if (groupId) {
pBlock->info.groupId = *groupId;
}
ASSERT(binfo.uid != 0);
pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid);
uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
@ -765,6 +762,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
SSDataBlock* result = doTableScanGroup(pOperator);
if (result != NULL) {
ASSERT(result->info.uid != 0);
return result;
}
@ -822,7 +820,7 @@ static void destroyTableScanOperatorInfo(void* param) {
}
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo) {
SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -886,7 +884,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pOperator->cost.openCost = 0;
return pOperator;
_error:
_error:
if (pInfo != NULL) {
destroyTableScanOperatorInfo(pInfo);
}
@ -1034,7 +1032,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re
createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL);
return pOperator;
_error:
_error:
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
return NULL;
@ -1124,7 +1122,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
tsdbReaderClose(pReader);
qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
return pBlock->info.rows > 0 ? pBlock : NULL;
}
@ -1713,7 +1711,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
}
}
#if 0
} else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
} else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
qDebug("stream scan tsdb return %d rows", pResult->info.rows);
@ -1861,7 +1859,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
size_t total = taosArrayGetSize(pInfo->pBlockLists);
// TODO: refactor
FETCH_NEXT_BLOCK:
FETCH_NEXT_BLOCK:
if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
if (pInfo->validBlockIndex >= total) {
doClearBufferedBlocks(pInfo);
@ -1988,7 +1986,7 @@ FETCH_NEXT_BLOCK:
int32_t totBlockNum = taosArrayGetSize(pInfo->pBlockLists);
NEXT_SUBMIT_BLK:
NEXT_SUBMIT_BLK:
while (1) {
if (pInfo->tqReader->pMsg == NULL) {
if (pInfo->validBlockIndex >= totBlockNum) {
@ -2243,7 +2241,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL);
return pOperator;
_end:
_end:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
pTaskInfo->code = code;
@ -2428,7 +2426,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
return pOperator;
_error:
_error:
if (pColIds != NULL) {
taosArrayDestroy(pColIds);
}
@ -3052,11 +3050,11 @@ static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
if (func == NULL) return -1;
SMetaFltParam param = {.suid = 0,
.cid = 0,
.type = TSDB_DATA_TYPE_VARCHAR,
.val = pVal->datum.p,
.reverse = reverse,
.filterFunc = func};
.cid = 0,
.type = TSDB_DATA_TYPE_VARCHAR,
.val = pVal->datum.p,
.reverse = reverse,
.filterFunc = func};
return -1;
}
@ -3071,11 +3069,11 @@ static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
if (func == NULL) return -1;
SMetaFltParam param = {.suid = 0,
.cid = 0,
.type = TSDB_DATA_TYPE_BIGINT,
.val = &pVal->datum.i,
.reverse = reverse,
.filterFunc = func};
.cid = 0,
.type = TSDB_DATA_TYPE_BIGINT,
.val = &pVal->datum.i,
.reverse = reverse,
.filterFunc = func};
int32_t ret = metaFilterCreateTime(pMeta, &param, result);
return ret;
@ -3995,7 +3993,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
return pOperator;
_error:
_error:
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
@ -4134,7 +4132,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
return pOperator;
_error:
_error:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -4678,7 +4676,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pOperator->cost.openCost = 0;
return pOperator;
_error:
_error:
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);

19
source/libs/executor/src/sortoperator.c
View File

@ -38,7 +38,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols);
int32_t numOfOutputCols = 0;
int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
int32_t code =
extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
@ -62,8 +63,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
// there are headers, so pageSize = rowSize + header pInfo->sortBufSize = pInfo->bufPageSize * 16;
// TODO dynamic set the available sort buffer
pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo,
getExplainExecInfo);
pOperator->fpSet =
createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, getExplainExecInfo);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -126,7 +127,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
// ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
// ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
@ -316,7 +317,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
// ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
// ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
@ -592,7 +593,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataEnsureCapacity(p, capacity);
_retry:
_retry:
while (1) {
STupleHandle* pTupleHandle = NULL;
if (pInfo->groupSort) {
@ -647,13 +648,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
// ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID);
// ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID);
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
}
pInfo->limitInfo.numOfOutputRows += p->info.rows;
pDataBlock->info.rows = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
@ -735,7 +735,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
int32_t numOfOutputCols = 0;
code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID,
&pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

1
source/libs/executor/src/timewindowoperator.c
View File

@ -5384,6 +5384,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
maxTs = TMAX(maxTs, pBlock->info.window.ekey);
minTs = TMIN(minTs, pBlock->info.window.skey);
doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

2
source/libs/function/src/builtins.c
View File

@ -1651,7 +1651,7 @@ static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (numOfParams == 0 || numOfParams > 2) {
if (numOfParams > 2) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}

7
source/libs/function/src/builtinsimpl.c
View File

@ -380,7 +380,7 @@ typedef struct SGroupKeyInfo {
numOfElem += 1; \
pStddevRes->count -= 1; \
sumT -= plist[i]; \
pStddevRes->quadraticISum -= plist[i] * plist[i]; \
pStddevRes->quadraticISum -= (int64_t)(plist[i] * plist[i]); \
} \
} while (0)
@ -2526,8 +2526,9 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) {
// might be a race condition here that pHisto can be overwritten or setup function
// has not been called, need to relink the buffer pHisto points to.
buildHistogramInfo(pInfo);
qDebug("%s before add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__,
numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems);
qDebug("%s before add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p",
__FUNCTION__, numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto,
pInfo->pHisto->elems);
for (int32_t i = start; i < pInput->numOfRows + start; ++i) {
if (colDataIsNull_f(pCol->nullbitmap, i)) {
continue;

27
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -111,6 +111,8 @@ const char* nodesNodeName(ENodeType type) {
return "DropSuperTableStmt";
case QUERY_NODE_ALTER_TABLE_STMT:
return "AlterTableStmt";
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
return "AlterSuperTableStmt";
case QUERY_NODE_CREATE_USER_STMT:
return "CreateUserStmt";
case QUERY_NODE_ALTER_USER_STMT:
@ -669,7 +671,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanIgnoreGroupId, pNode->ignoreGroupId);
code = tjsonAddBoolToObject(pJson, jkProjectLogicPlanIgnoreGroupId, pNode->ignoreGroupId);
}
return code;
@ -2632,6 +2634,7 @@ static const char* jkSubplanRootNode = "RootNode";
static const char* jkSubplanDataSink = "DataSink";
static const char* jkSubplanTagCond = "TagCond";
static const char* jkSubplanTagIndexCond = "TagIndexCond";
static const char* jkSubplanShowRewrite = "ShowRewrite";
static int32_t subplanToJson(const void* pObj, SJson* pJson) {
const SSubplan* pNode = (const SSubplan*)pObj;
@ -2667,6 +2670,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite);
}
return code;
}
@ -2705,6 +2711,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite);
}
return code;
}
@ -2758,6 +2767,20 @@ static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToLogicAggNode(const SJson* pJson, void* pObj) {
SAggLogicNode* pNode = (SAggLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkAggLogicPlanGroupKeys, &pNode->pGroupKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkAggLogicPlanAggFuncs, &pNode->pAggFuncs);
}
return code;
}
static const char* jkDataTypeType = "Type";
static const char* jkDataTypePrecision = "Precision";
static const char* jkDataTypeScale = "Scale";
@ -4735,6 +4758,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDeleteStmt(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_SCAN:
return jsonToLogicScanNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_AGG:
return jsonToLogicAggNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return jsonToLogicProjectNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:

13
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -1969,7 +1969,12 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum { PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT, PHY_LAST_ROW_SCAN_CODE_IGNULL };
enum {
PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
PHY_LAST_ROW_SCAN_CODE_GROUP_SORT,
PHY_LAST_ROW_SCAN_CODE_IGNULL
};
static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@ -3433,6 +3438,9 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = queryNodeAddrInlineToMsg(&pNode->execNode, pEncoder);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->showRewrite);
}
return code;
}
@ -3479,6 +3487,9 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = msgToQueryNodeAddrInline(pDecoder, &pNode->execNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->showRewrite);
}
return code;
}

207
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -438,210 +438,3 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
return;
}
static EDealRes walkPhysiNode(SPhysiNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiPlan((SNode*)pNode->pOutputDataBlockDesc, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pNode->pConditions, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pNode->pChildren, order, walker, pContext);
}
return res;
}
static EDealRes walkScanPhysi(SScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiNode((SPhysiNode*)pScan, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pScan->pScanCols, order, walker, pContext);
}
return res;
}
static EDealRes walkTableScanPhysi(STableScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker,
void* pContext) {
EDealRes res = walkScanPhysi((SScanPhysiNode*)pScan, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pScan->pDynamicScanFuncs, order, walker, pContext);
}
return res;
}
static EDealRes walkWindowPhysi(SWinodwPhysiNode* pWindow, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiNode((SPhysiNode*)pWindow, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pWindow->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pWindow->pFuncs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pWindow->pTspk, order, walker, pContext);
}
return res;
}
static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = DEAL_RES_CONTINUE;
switch (nodeType(pNode)) {
case QUERY_NODE_NODE_LIST:
res = walkPhysiPlans(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
res = walkTableScanPhysi((STableScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
res = walkTableScanPhysi((STableScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pProject->pProjections, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode* pJoin = (SSortMergeJoinPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pJoin->pMergeCondition, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pJoin->pOnConditions, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pJoin->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
SAggPhysiNode* pAgg = (SAggPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pGroupKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pAggFuncs, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pExchange->pSrcEndPoints, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pSort = (SSortPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pSortKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: {
SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pNode;
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pState->pStateKey, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pPartitionKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
res = walkPhysiPlan((SNode*)(((SDataSinkNode*)pNode)->pInputDataBlockDesc), order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
res = walkPhysiPlan((SNode*)(((SDataSinkNode*)pNode)->pInputDataBlockDesc), order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_SUBPLAN: {
SSubplan* pSubplan = (SSubplan*)pNode;
res = walkPhysiPlans(pSubplan->pChildren, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan((SNode*)pSubplan->pNode, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan((SNode*)pSubplan->pDataSink, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN: {
SQueryPlan* pPlan = (SQueryPlan*)pNode;
if (NULL != pPlan->pSubplans) {
// only need to walk the top-level subplans, because they will recurse to all the subplans below
walkPhysiPlan(nodesListGetNode(pPlan->pSubplans, 0), order, walker, pContext);
}
break;
}
default:
res = dispatchExpr(pNode, order, walker, pContext);
break;
}
return res;
}
static EDealRes walkPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
return walkNode(pNode, order, walker, pContext, dispatchPhysiPlan);
}
static EDealRes walkPhysiPlans(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext) {
SNode* node;
FOREACH(node, pNodeList) {
EDealRes res = walkPhysiPlan(node, order, walker, pContext);
if (DEAL_RES_ERROR == res || DEAL_RES_END == res) {
return res;
}
}
return DEAL_RES_CONTINUE;
}
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext) {
(void)walkPhysiPlan(pNode, TRAVERSAL_PREORDER, walker, pContext);
}

1
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -328,6 +328,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_DROP_SUPER_TABLE_STMT:
return makeNode(type, sizeof(SDropSuperTableStmt));
case QUERY_NODE_ALTER_TABLE_STMT:
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
return makeNode(type, sizeof(SAlterTableStmt));
case QUERY_NODE_CREATE_USER_STMT:
return makeNode(type, sizeof(SCreateUserStmt));

31
source/libs/nodes/test/CMakeLists.txt
View File

@ -1,19 +1,26 @@
MESSAGE(STATUS "build nodes unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(nodesTest ${SOURCE_LIST})
TARGET_INCLUDE_DIRECTORIES(
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(nodesTest ${SOURCE_LIST})
TARGET_INCLUDE_DIRECTORIES(
nodesTest
PUBLIC "${TD_SOURCE_DIR}/include/nodes/"
PRIVATE "${TD_SOURCE_DIR}/source/nodes/inc"
)
TARGET_LINK_LIBRARIES(
)
TARGET_LINK_LIBRARIES(
nodesTest
PUBLIC os util common nodes gtest
)
PUBLIC os util common nodes qcom gtest
)
add_test(
NAME nodesTest
COMMAND nodesTest
)
ENDIF()

299
source/libs/nodes/test/nodesCloneTest.cpp Normal file
View File

@ -0,0 +1,299 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, AND/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include "nodes.h"
#include "plannodes.h"
#include "querynodes.h"
class NodesCloneTest : public testing::Test {
public:
void registerCheckFunc(const std::function<void(const SNode*, const SNode*)>& func) { checkFunc_ = func; }
void run(const SNode* pSrc) {
std::unique_ptr<SNode, void (*)(SNode*)> pDst(nodesCloneNode(pSrc), nodesDestroyNode);
checkFunc_(pSrc, pDst.get());
}
private:
std::function<void(const SNode*, const SNode*)> checkFunc_;
};
TEST_F(NodesCloneTest, tempTable) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STempTableNode* pSrcNode = (STempTableNode*)pSrc;
STempTableNode* pDstNode = (STempTableNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pSubquery), nodeType(pDstNode->pSubquery));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_TEMP_TABLE));
STempTableNode* pNode = (STempTableNode*)srcNode.get();
pNode->pSubquery = nodesMakeNode(QUERY_NODE_SELECT_STMT);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, joinTable) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SJoinTableNode* pSrcNode = (SJoinTableNode*)pSrc;
SJoinTableNode* pDstNode = (SJoinTableNode*)pDst;
ASSERT_EQ(pSrcNode->joinType, pDstNode->joinType);
ASSERT_EQ(nodeType(pSrcNode->pLeft), nodeType(pDstNode->pLeft));
ASSERT_EQ(nodeType(pSrcNode->pRight), nodeType(pDstNode->pRight));
if (NULL != pSrcNode->pOnCond) {
ASSERT_EQ(nodeType(pSrcNode->pOnCond), nodeType(pDstNode->pOnCond));
}
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_JOIN_TABLE));
SJoinTableNode* pNode = (SJoinTableNode*)srcNode.get();
pNode->joinType = JOIN_TYPE_INNER;
pNode->pLeft = nodesMakeNode(QUERY_NODE_REAL_TABLE);
pNode->pRight = nodesMakeNode(QUERY_NODE_REAL_TABLE);
pNode->pOnCond = nodesMakeNode(QUERY_NODE_OPERATOR);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, stateWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStateWindowNode* pSrcNode = (SStateWindowNode*)pSrc;
SStateWindowNode* pDstNode = (SStateWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pCol), nodeType(pDstNode->pCol));
ASSERT_EQ(nodeType(pSrcNode->pExpr), nodeType(pDstNode->pExpr));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_STATE_WINDOW));
SStateWindowNode* pNode = (SStateWindowNode*)srcNode.get();
pNode->pCol = nodesMakeNode(QUERY_NODE_COLUMN);
pNode->pExpr = nodesMakeNode(QUERY_NODE_OPERATOR);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, sessionWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SSessionWindowNode* pSrcNode = (SSessionWindowNode*)pSrc;
SSessionWindowNode* pDstNode = (SSessionWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pCol), nodeType(pDstNode->pCol));
ASSERT_EQ(nodeType(pSrcNode->pGap), nodeType(pDstNode->pGap));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_SESSION_WINDOW));
SSessionWindowNode* pNode = (SSessionWindowNode*)srcNode.get();
pNode->pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
pNode->pGap = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, intervalWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SIntervalWindowNode* pSrcNode = (SIntervalWindowNode*)pSrc;
SIntervalWindowNode* pDstNode = (SIntervalWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pInterval), nodeType(pDstNode->pInterval));
if (NULL != pSrcNode->pOffset) {
ASSERT_EQ(nodeType(pSrcNode->pOffset), nodeType(pDstNode->pOffset));
}
if (NULL != pSrcNode->pSliding) {
ASSERT_EQ(nodeType(pSrcNode->pSliding), nodeType(pDstNode->pSliding));
}
if (NULL != pSrcNode->pFill) {
ASSERT_EQ(nodeType(pSrcNode->pFill), nodeType(pDstNode->pFill));
}
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW));
SIntervalWindowNode* pNode = (SIntervalWindowNode*)srcNode.get();
pNode->pInterval = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pOffset = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pSliding = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pFill = nodesMakeNode(QUERY_NODE_FILL);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, fill) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SFillNode* pSrcNode = (SFillNode*)pSrc;
SFillNode* pDstNode = (SFillNode*)pDst;
ASSERT_EQ(pSrcNode->mode, pDstNode->mode);
if (NULL != pSrcNode->pValues) {
ASSERT_EQ(nodeType(pSrcNode->pValues), nodeType(pDstNode->pValues));
}
ASSERT_EQ(nodeType(pSrcNode->pWStartTs), nodeType(pDstNode->pWStartTs));
ASSERT_EQ(pSrcNode->timeRange.skey, pDstNode->timeRange.skey);
ASSERT_EQ(pSrcNode->timeRange.ekey, pDstNode->timeRange.ekey);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_FILL));
SFillNode* pNode = (SFillNode*)srcNode.get();
pNode->mode = FILL_MODE_VALUE;
pNode->pValues = nodesMakeNode(QUERY_NODE_NODE_LIST);
pNode->pWStartTs = nodesMakeNode(QUERY_NODE_COLUMN);
pNode->timeRange.skey = 1666756692907;
pNode->timeRange.ekey = 1666756699907;
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, logicSubplan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SLogicSubplan* pSrcNode = (SLogicSubplan*)pSrc;
SLogicSubplan* pDstNode = (SLogicSubplan*)pDst;
ASSERT_EQ(pSrcNode->subplanType, pDstNode->subplanType);
ASSERT_EQ(pSrcNode->level, pDstNode->level);
ASSERT_EQ(pSrcNode->splitFlag, pDstNode->splitFlag);
ASSERT_EQ(pSrcNode->numOfComputeNodes, pDstNode->numOfComputeNodes);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN));
SLogicSubplan* pNode = (SLogicSubplan*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiScan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc;
STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->uid, pDstNode->uid);
ASSERT_EQ(pSrcNode->suid, pDstNode->suid);
ASSERT_EQ(pSrcNode->tableType, pDstNode->tableType);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN));
STagScanPhysiNode* pNode = (STagScanPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiSystemTableScan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SSystemTableScanPhysiNode* pSrcNode = (SSystemTableScanPhysiNode*)pSrc;
SSystemTableScanPhysiNode* pDstNode = (SSystemTableScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->showRewrite, pDstNode->showRewrite);
ASSERT_EQ(pSrcNode->accountId, pDstNode->accountId);
ASSERT_EQ(pSrcNode->sysInfo, pDstNode->sysInfo);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN));
SSystemTableScanPhysiNode* pNode = (SSystemTableScanPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamSemiSessionWinodw) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamSemiSessionWinodwPhysiNode* pSrcNode = (SStreamSemiSessionWinodwPhysiNode*)pSrc;
SStreamSemiSessionWinodwPhysiNode* pDstNode = (SStreamSemiSessionWinodwPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->gap, pDstNode->gap);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION));
SStreamSemiSessionWinodwPhysiNode* pNode = (SStreamSemiSessionWinodwPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamFinalSessionWinodw) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamFinalSessionWinodwPhysiNode* pSrcNode = (SStreamFinalSessionWinodwPhysiNode*)pSrc;
SStreamFinalSessionWinodwPhysiNode* pDstNode = (SStreamFinalSessionWinodwPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->gap, pDstNode->gap);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION));
SStreamFinalSessionWinodwPhysiNode* pNode = (SStreamFinalSessionWinodwPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamPartition) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamPartitionPhysiNode* pSrcNode = (SStreamPartitionPhysiNode*)pSrc;
SStreamPartitionPhysiNode* pDstNode = (SStreamPartitionPhysiNode*)pDst;
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION));
SStreamPartitionPhysiNode* pNode = (SStreamPartitionPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiPartition) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SPartitionPhysiNode* pSrcNode = (SPartitionPhysiNode*)pSrc;
SPartitionPhysiNode* pDstNode = (SPartitionPhysiNode*)pDst;
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_PARTITION));
SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)srcNode.get();
return srcNode.get();
}());
}

0
source/libs/nodes/test/nodesTest.cpp → source/libs/nodes/test/nodesTestMain.cpp
View File

1
source/libs/parser/inc/sql.y
View File

@ -518,6 +518,7 @@ stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE.
stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; }
stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; }
stream_options(A) ::= stream_options(B) FILL_HISTORY NK_INTEGER(C). { ((SStreamOptions*)B)->fillHistory = taosStr2Int8(C.z, NULL, 10); A = B; }
subtable_opt(A) ::= . { A = NULL; }
subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }

1
source/libs/parser/src/parAstCreater.c
View File

@ -1702,6 +1702,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->triggerType = STREAM_TRIGGER_AT_ONCE;
pOptions->fillHistory = STREAM_DEFAULT_FILL_HISTORY;
pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED;
return (SNode*)pOptions;
}

1
source/libs/parser/src/parTokenizer.c
View File

@ -92,6 +92,7 @@ static SKeyword keywordTable[] = {
{"EVERY", TK_EVERY},
{"FILE", TK_FILE},
{"FILL", TK_FILL},
{"FILL_HISTORY", TK_FILL_HISTORY},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT},
{"FLUSH", TK_FLUSH},

1
source/libs/parser/src/parTranslater.c
View File

@ -5542,6 +5542,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
pReq->triggerType = pStmt->pOptions->triggerType;
pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0);
pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0);
pReq->fillHistory = pStmt->pOptions->fillHistory;
pReq->igExpired = pStmt->pOptions->ignoreExpired;
columnDefNodeToField(pStmt->pTags, &pReq->pTags);
pReq->numOfTags = LIST_LENGTH(pStmt->pTags);

5438
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

34
source/libs/parser/test/parInitialCTest.cpp
View File

@ -576,10 +576,10 @@ TEST_F(ParserInitialCTest, createStream) {
memset(&expect, 0, sizeof(SCMCreateStreamReq));
};
auto setCreateStreamReqFunc = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb,
int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE,
int64_t maxDelay = 0, int64_t watermark = 0,
int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED) {
auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb,
int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0,
int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED,
int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) {
snprintf(expect.name, sizeof(expect.name), "0.%s", pStream);
snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb);
snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb);
@ -588,6 +588,7 @@ TEST_F(ParserInitialCTest, createStream) {
expect.triggerType = triggerType;
expect.maxDelay = maxDelay;
expect.watermark = watermark;
expect.fillHistory = fillHistory;
expect.igExpired = igExpired;
};
@ -619,6 +620,7 @@ TEST_F(ParserInitialCTest, createStream) {
ASSERT_EQ(req.triggerType, expect.triggerType);
ASSERT_EQ(req.maxDelay, expect.maxDelay);
ASSERT_EQ(req.watermark, expect.watermark);
ASSERT_EQ(req.fillHistory, expect.fillHistory);
ASSERT_EQ(req.igExpired, expect.igExpired);
ASSERT_EQ(req.numOfTags, expect.numOfTags);
if (expect.numOfTags > 0) {
@ -636,24 +638,24 @@ TEST_F(ParserInitialCTest, createStream) {
tFreeSCMCreateStreamReq(&req);
});
setCreateStreamReqFunc("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1");
setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1");
run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)");
clearCreateStreamReq();
setCreateStreamReqFunc("s1", "test",
"create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 into st1 "
"as select count(*) from t1 interval(10s)",
"st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND,
0);
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 INTO st1 AS SELECT COUNT(*) "
setCreateStreamReq(
"s1", "test",
"create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st1 "
"as select count(*) from t1 interval(10s)",
"st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1);
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st1 AS "
"SELECT COUNT(*) "
"FROM t1 INTERVAL(10S)");
clearCreateStreamReq();
setCreateStreamReqFunc(
"s1", "test",
"create stream s1 into st3 tags(tname varchar(10), id int) subtable(concat('new-', tname)) as "
"select _wstart wstart, count(*) cnt from st1 partition by tbname tname, tag1 id interval(10s)",
"st3");
setCreateStreamReq("s1", "test",
"create stream s1 into st3 tags(tname varchar(10), id int) subtable(concat('new-', tname)) as "
"select _wstart wstart, count(*) cnt from st1 partition by tbname tname, tag1 id interval(10s)",
"st3");
addTag("tname", TSDB_DATA_TYPE_VARCHAR, 10 + VARSTR_HEADER_SIZE);
addTag("id", TSDB_DATA_TYPE_INT);
run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) "

4
source/libs/parser/test/parTestUtil.cpp
View File

@ -295,6 +295,10 @@ class ParserTestBaseImpl {
char* pStr = NULL;
int32_t len = 0;
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
// check toObject
SNode* pCopy = NULL;
DO_WITH_THROW(nodesStringToNode, pStr, &pCopy)
nodesDestroyNode(pCopy);
string str(pStr);
taosMemoryFreeClear(pStr);
return str;

37
source/libs/planner/src/planLogicCreater.c
View File

@ -339,29 +339,6 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
return code;
}
static int32_t createColumnByLastRow(SNodeList* pFuncs, SNodeList** pOutput) {
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pCols = NULL;
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
SFunctionNode* pLastRow = (SFunctionNode*)pFunc;
SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pLastRow->pParameterList, 0);
snprintf(pCol->colName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName);
snprintf(pCol->node.aliasName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName);
NODES_CLEAR_LIST(pLastRow->pParameterList);
code = nodesListMakeStrictAppend(&pCols, (SNode*)pCol);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pCols;
} else {
nodesDestroyList(pCols);
}
return code;
}
static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable,
SLogicNode** pLogicNode) {
return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
@ -491,20 +468,6 @@ static SNode* createGroupingSetNode(SNode* pExpr) {
return (SNode*)pGroupingSet;
}
static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOutput) {
SNodeList* pGroupKeys = NULL;
SNode* pPartKey = NULL;
FOREACH(pPartKey, pPartKeys) {
int32_t code = nodesListMakeStrictAppend(&pGroupKeys, createGroupingSetNode(pPartKey));
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pGroupKeys);
return code;
}
}
*pOutput = pGroupKeys;
return TSDB_CODE_SUCCESS;
}
static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP
: GROUP_ACTION_NONE;

48
source/libs/planner/src/planOptimizer.c
View File

@ -596,18 +596,6 @@ static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogi
return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond);
}
static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) {
return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToProject(SOptimizeContext* pCxt, SProjectLogicNode* pProject, SNode** pCond) {
return pushDownCondOptAppendCond(&pProject->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
return pushDownCondOptAppendCond(&pJoin->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
return pushDownCondOptAppendCond(&pChild->pConditions, pCond);
}
@ -1201,40 +1189,6 @@ static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) {
return true;
}
static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets,
SLogicNode** pOutput) {
SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
if (NULL == pMerge) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMerge->node.precision = pChild->precision;
pMerge->numOfChannels = 2;
pMerge->pMergeKeys = pMergeKeys;
pMerge->node.pTargets = pTargets;
pMerge->pInputs = nodesCloneList(pChild->pTargets);
if (NULL == pMerge->pInputs) {
nodesDestroyNode((SNode*)pMerge);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pOutput = (SLogicNode*)pMerge;
return TSDB_CODE_SUCCESS;
}
static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge,
SLogicNode* pSmaScan) {
int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan);
}
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, pInterval, pMerge);
pSmaScan->pParent = pMerge;
pInterval->pParent = pMerge;
}
return code;
}
static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols,
SLogicNode** pOutput) {
SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
@ -1770,7 +1724,7 @@ static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode**
return TSDB_CODE_OUT_OF_MEMORY;
}
pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i;
pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i;
pLimitNode->offset = NULL == pOffset ? 0 : ((SValueNode*)pOffset)->datum.i;
*pOutput = (SNode*)pLimitNode;
return TSDB_CODE_SUCCESS;
}

1
source/libs/planner/src/planPhysiCreater.c
View File

@ -584,6 +584,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
return TSDB_CODE_OUT_OF_MEMORY;
}
pSubplan->showRewrite = pScanLogicNode->showRewrite;
pScan->showRewrite = pScanLogicNode->showRewrite;
pScan->accountId = pCxt->pPlanCxt->acctId;
pScan->sysInfo = pCxt->pPlanCxt->sysInfo;

3
source/libs/planner/src/planSpliter.c
View File

@ -1419,7 +1419,8 @@ typedef struct SQnodeSplitInfo {
static bool qndSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode,
SQnodeSplitInfo* pInfo) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent &&
((SScanLogicNode*)pNode)->scanSeq[0] < 1 && ((SScanLogicNode*)pNode)->scanSeq[1] < 1) {
QUERY_NODE_LOGIC_PLAN_INTERP_FUNC != nodeType(pNode->pParent) && ((SScanLogicNode*)pNode)->scanSeq[0] <= 1 &&
((SScanLogicNode*)pNode)->scanSeq[1] <= 1) {
pInfo->pSplitNode = pNode;
pInfo->pSubplan = pSubplan;
return true;

9
source/libs/planner/src/planner.c
View File

@ -142,15 +142,6 @@ int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan) {
return nodesMsgToNode(pStr, len, (SNode**)pSubplan);
}
char* qQueryPlanToString(const SQueryPlan* pPlan) {
char* pStr = NULL;
int32_t len = 0;
if (TSDB_CODE_SUCCESS != nodesNodeToString((SNode*)pPlan, false, &pStr, &len)) {
return NULL;
}
return pStr;
}
SQueryPlan* qStringToQueryPlan(const char* pStr) {
SQueryPlan* pPlan = NULL;
if (TSDB_CODE_SUCCESS != nodesStringToNode(pStr, (SNode**)&pPlan)) {

5
source/libs/planner/test/planTestUtil.cpp
View File

@ -466,8 +466,13 @@ class PlannerTestBaseImpl {
char* pStr = NULL;
int32_t len = 0;
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
// check toObject
SNode* pCopy = NULL;
DO_WITH_THROW(nodesStringToNode, pStr, &pCopy)
nodesDestroyNode(pCopy);
string str(pStr);
taosMemoryFreeClear(pStr);
return str;
}

42
source/libs/stream/src/streamState.c
View File

@ -497,7 +497,7 @@ SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSess
if (pCur == NULL) return NULL;
tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL);
int32_t c = 0;
int32_t c = -2;
SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c);
if (c != 0) {
@ -727,3 +727,43 @@ _end:
streamStateFreeCur(pCur);
return res;
}
#if 0
char* streamStateSessionDump(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
return NULL;
}
pCur->number = pState->number;
if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) {
streamStateFreeCur(pCur);
return NULL;
}
tdbTbcMoveToFirst(pCur->pCur);
SSessionKey key = {0};
int32_t code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
return NULL;
}
int32_t size = 2048;
char* dumpBuf = taosMemoryCalloc(size, 1);
int64_t len = 0;
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
while (1) {
tdbTbcMoveToNext(pCur->pCur);
key = (SSessionKey){0};
code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
return dumpBuf;
}
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
}
return dumpBuf;
}
#endif

30
source/libs/wal/src/walMeta.c
View File

@ -72,7 +72,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
int64_t capacity = 0;
int64_t readSize = 0;
char* buf = NULL;
char* found = NULL;
int64_t found = -1;
bool firstTrial = pFileInfo->fileSize < fileSize;
// search for the valid last WAL entry, e.g. block by block
@ -114,18 +114,22 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
char* candidate = NULL;
char* haystack = buf;
int64_t pos = 0;
SWalCkHead* logContent = NULL;
while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) {
pos = candidate - buf;
// validate head
int64_t len = readSize - (candidate - buf);
int64_t len = readSize - pos;
if (len < walCkHeadSz) {
break;
}
SWalCkHead* logContent = (SWalCkHead*)candidate;
logContent = (SWalCkHead*)(buf + pos);
if (walValidHeadCksum(logContent) != 0) {
wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr);
haystack = candidate + 1;
offset + pos, fnameStr);
haystack = buf + pos + 1;
if (firstTrial) {
break;
} else {
@ -160,11 +164,13 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
break;
}
}
logContent = (SWalCkHead*)(buf + pos);
if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr);
haystack = candidate + 1;
offset + pos, fnameStr);
haystack = buf + pos + 1;
if (firstTrial) {
break;
} else {
@ -173,19 +179,19 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
// found one
found = candidate;
haystack = candidate + 1;
found = pos;
haystack = buf + pos + 1;
}
if (found || offset == 0) break;
if (found >= 0 || offset == 0) break;
// go backwards, e.g. by at most one WAL scan buf size
end = offset + walCkHeadSz - 1;
end = TMIN(offset + walCkHeadSz - 1, fileSize);
firstTrial = false;
}
// determine end of last entry
SWalCkHead* lastEntry = (SWalCkHead*)found;
SWalCkHead* lastEntry = (found >= 0) ? (SWalCkHead*)(buf + found) : NULL;
int64_t retVer = -1;
int64_t lastEntryBeginOffset = 0;
int64_t lastEntryEndOffset = 0;

2
source/os/src/osDir.c
View File

@ -158,7 +158,7 @@ int32_t taosMulMkDir(const char *dirname) {
#ifdef WINDOWS
code = _mkdir(temp, 0755);
#elif defined(DARWIN)
code = mkdir(dirname, 0777);
code = mkdir(temp, 0777);
#else
code = mkdir(temp, 0755);
#endif

2
tests/parallel_test/container_build.sh
View File

@ -52,7 +52,7 @@ fi
docker run \
-v $REP_MOUNT_PARAM \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT"
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT"
ret=$?
exit $ret

2
tests/pytest/util/dnodes.py
View File

@ -40,6 +40,7 @@ class TDSimClient:
"jniDebugFlag": "143",
"qDebugFlag": "143",
"supportVnodes": "1024",
"enableQueryHb": "1",
"telemetryReporting": "0",
}
@ -136,6 +137,7 @@ class TDDnode:
"wDebugFlag": "143",
"numOfLogLines": "100000000",
"statusInterval": "1",
"enableQueryHb": "1",
"supportVnodes": "1024",
"telemetryReporting": "0"
}

5
tests/script/api/makefile
View File

@ -17,6 +17,5 @@ exe:
clean:
rm $(ROOT)batchprepare
rm $(ROOT)stmtBatchTest
rm $(ROOT)stmtTest
rm $(ROOT)stmt_function
rm $(ROOT)stopquery
rm $(ROOT)dbTableRoute

13
tests/script/api/stopquery.c
View File

@ -36,7 +36,7 @@ int64_t st, et;
char hostName[128];
char dbName[128];
char tbName[128];
int32_t runTimes = 1000;
int32_t runTimes = 10;
typedef struct {
int id;
@ -657,7 +657,6 @@ int sqConCleanupAsyncQuery(bool fetch) {
void sqRunAllCase(void) {
#if 1
sqStopSyncQuery(false);
sqStopSyncQuery(true);
sqStopAsyncQuery(false);
@ -678,33 +677,33 @@ void sqRunAllCase(void) {
sqConCloseAsyncQuery(false);
sqConCloseAsyncQuery(true);
sqKillSyncQuery(false);
sqKillSyncQuery(true);
sqKillAsyncQuery(false);
sqKillAsyncQuery(true);
#if 0
/*
sqConKillSyncQuery(false);
sqConKillSyncQuery(true);
sqConKillAsyncQuery(false);
sqConKillAsyncQuery(true);
#endif
/*
sqConCleanupSyncQuery(false);
sqConCleanupSyncQuery(true);
sqConCleanupAsyncQuery(false);
sqConCleanupAsyncQuery(true);
*/
#endif
int32_t l = 5;
while (l) {
printf("%d\n", l--);
sleep(1);
}
printf("test done\n");
}
int main(int argc, char *argv[]) {
if (argc != 4) {
printf("usage: %s server-ip dbname tablename\n", argv[0]);

8
tests/script/jenkins/basic.txt
View File

@ -56,6 +56,8 @@
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
./test.sh -f tsim/dnode/vnode_clean.sim
./test.sh -f tsim/dnode/use_dropped_dnode.sim
./test.sh -f tsim/dnode/split_vgroup_replica1.sim
./test.sh -f tsim/dnode/split_vgroup_replica3.sim
# ---- import ----
./test.sh -f tsim/import/basic.sim
@ -227,7 +229,7 @@
./test.sh -f tsim/table/vgroup.sim
# ---- stream
./test.sh -f tsim/stream/basic0.sim -v
./test.sh -f tsim/stream/basic0.sim -g
./test.sh -f tsim/stream/basic1.sim
./test.sh -f tsim/stream/basic2.sim
./test.sh -f tsim/stream/drop_stream.sim
@ -241,7 +243,7 @@
./test.sh -f tsim/stream/triggerSession0.sim
./test.sh -f tsim/stream/partitionby.sim
./test.sh -f tsim/stream/partitionby1.sim
# unsupport ./test.sh -f tsim/stream/schedSnode.sim
./test.sh -f tsim/stream/schedSnode.sim
./test.sh -f tsim/stream/windowClose.sim
./test.sh -f tsim/stream/ignoreExpiredData.sim
./test.sh -f tsim/stream/sliding.sim
@ -446,5 +448,5 @@
./test.sh -f tsim/tag/tinyint.sim
./test.sh -f tsim/tag/drop_tag.sim
./test.sh -f tmp/monitor.sim
#======================b1-end===============

8
tests/script/test.sh
View File

@ -15,7 +15,7 @@ VALGRIND=0
UNIQUE=0
UNAME_BIN=`which uname`
OS_TYPE=`$UNAME_BIN`
while getopts "f:avum" arg
while getopts "f:agvum" arg
do
case $arg in
f)
@ -27,6 +27,9 @@ do
u)
UNIQUE=1
;;
g)
VALGRIND=2
;;
?)
echo "unknow argument"
;;
@ -125,6 +128,9 @@ if [ -n "$FILE_NAME" ]; then
FLAG="-v"
echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
elif [ $VALGRIND -eq 2 ]; then
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME -v
$PROGRAM -c $CFG_DIR -f $FILE_NAME -v
else
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
$PROGRAM -c $CFG_DIR -f $FILE_NAME

2
tests/script/tmp/monitor.sim
View File

@ -24,3 +24,5 @@ print =============== create drop qnode 1
sql create qnode on dnode 1
#sql create snode on dnode 1
sleep 10000
system sh/exec.sh -n dnode1 -s stop

87
tests/script/tsim/dnode/split_vgroup_replica1.sim Normal file
View File

@ -0,0 +1,87 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 1
system sh/exec.sh -n dnode3 -s start
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step2
endi
if $data(2)[4] != ready then
goto step2
endi
if $data(3)[4] != ready then
goto step2
endi
print =============== step3: create database
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step4: split
print split vgroup 2
sql split vgroup 2
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

67
tests/script/tsim/dnode/split_vgroup_replica3.sim Normal file
View File

@ -0,0 +1,67 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
#system sh/exec.sh -n dnode5 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 4 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 3
print =============== step3: split
print split vgroup 2
sql split vgroup 2
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT

1
tests/script/tsim/insert/basic.sim
View File

@ -50,6 +50,7 @@ print $data[1000][1000]
$a1 = 0
$a2 = 0
$a3 = $a1 % $a2
$a4 = $a1 / $a2
print $a3
$val = \\\1

7
tests/script/tsim/parser/limit1_stb.sim
View File

@ -39,7 +39,6 @@ endi
$val = $totalNum - 1
sql select * from $stb limit $totalNum offset 1
if $rows != $val then
print expect $val , actual: $rows
return -1
endi
if $data01 != 1 then
@ -493,9 +492,3 @@ sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8),
if $rows != 6 then
return -1
endi
sql select * from $stb partition by tbname limit 1
if $rows != 10 then
return -1
endi

1
tests/script/tsim/parser/limit_stb.sim
View File

@ -722,7 +722,6 @@ sql select bottom(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu limit 5 offse
if $rows != 0 then
return -1
endi
sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu limit 3 offset 5
if $rows != 0 then
return -1

58
tests/script/tsim/stream/partitionbyColumnInterval.sim
View File

@ -599,12 +599,62 @@ if $loop_count == 20 then
endi
if $rows != 4 then
print =====rows=$rows
print =====rows=$rows
print =====rows=$rows
#goto loop15
print ======rows=$rows
goto loop15
endi
sql insert into t1 values(1648791223001,11,2,3,1.0);
sql insert into t2 values(1648791223001,21,2,3,1.0);
sql insert into t3 values(1648791223001,31,2,3,1.0);
sql insert into t4 values(1648791223001,41,2,3,1.0);
sleep 300
sql delete from st where ts = 1648791223001;
$loop_count = 0
loop16:
sleep 50
sql select * from test.streamt5 order by c1, c2, c3;
$loop_count = $loop_count + 1
if $loop_count == 20 then
print ======rows=$rows
return -1
endi
if $rows != 4 then
goto loop16
endi
sql insert into t1 values(1648791223001,12,2,3,1.0);
sql insert into t2 values(1648791223001,22,2,3,1.0);
sql insert into t3 values(1648791223001,32,2,3,1.0);
sql insert into t4 values(1648791223001,42,2,3,1.0);
sleep 300
sql delete from st where ts = 1648791223001;
$loop_count = 0
loop17:
sleep 50
sql select * from test.streamt5 order by c1, c2, c3;
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 4 then
print ======rows=$rows
print ======rows=$rows
print ======rows=$rows
return 1
#goto loop17
endi
$loop_all = $loop_all + 1
print ============loop_all=$loop_all

36
tests/script/tsim/stream/schedSnode.sim
View File

@ -6,10 +6,14 @@ system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
sleep 50
sql create database test vgroups 2;
sql create database target vgroups 1;
sql create snode on dnode 1
sql create database test vgroups 1;
sql create database target vgroups 1;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
@ -72,23 +76,23 @@ if $data01 != 8 then
goto loop1
endi
if $data02 != 4 then
if $data02 != 6 then
print =====data02=$data02
goto loop1
endi
if $data03 != 4 then
print ======$data03
if $data03 != 52 then
print ======data03=$data03
goto loop1
endi
if $data04 != 52 then
print ======$data04
print ======data04=$data04
goto loop1
endi
if $data05 != 13 then
print ======$data05
print ======data05=$data05
goto loop1
endi
@ -104,17 +108,17 @@ if $data12 != 6 then
endi
if $data13 != 92 then
print ======$data13
print ======data13=$data13
return -1
endi
if $data14 != 22 then
print ======$data14
print ======data14=$data14
return -1
endi
if $data15 != 3 then
print ======$data15
print ======data15=$data15
return -1
endi
@ -130,17 +134,17 @@ if $data22 != 4 then
endi
if $data23 != 32 then
print ======$data23
print ======data23=$data23
return -1
endi
if $data24 != 12 then
print ======$data24
print ======data24=$data24
return -1
endi
if $data25 != 3 then
print ======$data25
print ======data25=$data25
return -1
endi
@ -156,17 +160,17 @@ if $data32 != 30 then
endi
if $data33 != 180 then
print ======$data33
print ======data33=$data33
return -1
endi
if $data34 != 42 then
print ======$data34
print ======data34=$data34
return -1
endi
if $data35 != 3 then
print ======$data35
print ======data35=$data35
return -1
endi

7
tests/system-test/0-others/taosdShell.py
View File

@ -134,7 +134,11 @@ class TDTestCase:
tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);")
#TD-19944 -Q=3
tdsqlN=tdCom.newTdSql()
tdsqlN.query("select * from source_db.stb")
tdsqlN.query("select * from db0.stb")
def run(self):
# tdSql.prepare()
@ -164,6 +168,7 @@ class TDTestCase:
startAction = " -s -c " + taosdCfgPath
tdLog.printNoPrefix("================================ parameter: %s"%startAction)
self.taosdCommandExe(startAction,taosdCmdRun)
os.system(" rm -rf sdb.json ")
startAction = " --help"
@ -198,7 +203,7 @@ class TDTestCase:
startAction=" -E taosdCaseTmp/.env"
tdLog.printNoPrefix("================================ parameter: %s"%startAction)
os.system(" mkdir -p taosdCaseTmp/.env ")
os.system(" mkdir -p taosdCaseTmp ")
os.system("echo \'TAOS_QUERY_POLICY=3\' > taosdCaseTmp/.env ")
self.taosdCommandStop(startAction,taosdCmdRun)
os.system(" rm -rf taosdCaseTmp/.env ")

14
tests/system-test/1-insert/block_wise.py
View File

@ -426,7 +426,17 @@ class TDTestCase:
tdDnodes.stop(1)
tdDnodes.start(1)
tdLog.printNoPrefix("==========step3:insert and flush in rollup database")
tdLog.printNoPrefix("==========step3: sleep 20s for catalogUpdateTableIndex")
tdSql.execute("create database db_s20")
tdSql.execute("use db_s20")
tdSql.execute(f"create stable stb1 (ts timestamp, c1 int) tags (t1 int) sma(c1);")
tdSql.execute("alter stable stb1 add column tinyint_col tinyint")
time.sleep(20)
tdSql.query("select count(*) from stb1")
tdSql.execute("drop database if exists db_s20 ")
tdLog.printNoPrefix("==========step4:insert and flush in rollup database")
tdSql.execute("create database db4 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("use db4")
self.__create_tb(rollup="first")
@ -435,7 +445,7 @@ class TDTestCase:
tdSql.execute(f'flush database db4')
tdLog.printNoPrefix("==========step4:after wal, all check again ")
tdLog.printNoPrefix("==========step5:after wal, all check again ")
tdSql.prepare()
self.__create_tb()
self.__insert_data()

1
tests/system-test/1-insert/database_pre_suf.py
View File

@ -184,6 +184,7 @@ class TDTestCase:
tdSql.query("select count(*) from hn_table_1_r;")
tdSql.checkData(0,0,num_random*n)
sleep(5)
# stream data check
tdSql.query("select start,wend,max_int from stream_max_stable_1 ;")
tdSql.checkRows(20)

1
tests/system-test/1-insert/keep_expired.py
View File

@ -29,6 +29,7 @@ class TDTestCase:
time.sleep(self.offset_time * 2)
tdSql.query(f'select * from {self.dbname}.{self.ctbname}')
tdSql.checkEqual(tdSql.queryRows, 0)
tdSql.execute(f'TRIM DATABASE {self.dbname}')
def stop(self):
tdSql.close()

Some files were not shown because too many files have changed in this diff Show More