xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/3_liaohj

This commit is contained in:
Haojun Liao 2022-10-28 19:43:47 +08:00 committed by GitHub
parent cb0adb0e9b d058475fac
commit 0035bb1966
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
113 changed files with 5792 additions and 4258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Jenkinsfile2
View File

@ -173,8 +173,8 @@ def pre_test_build_mac() {
''' '''
sh ''' sh '''
cd ${WK}/debug cd ${WK}/debug
cmake .. -DBUILD_TEST=true cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
make -j8 make -j10
''' '''
sh ''' sh '''
date date
@ -428,7 +428,7 @@ pipeline {
date date
rm -rf ${WKC}/debug rm -rf ${WKC}/debug
cd ${WKC}/tests/parallel_test cd ${WKC}/tests/parallel_test
time ./container_build.sh -w ${WKDIR} -t 8 -e time ./container_build.sh -w ${WKDIR} -t 10 -e
rm -f /tmp/cases.task rm -f /tmp/cases.task
./collect_cases.sh -e ./collect_cases.sh -e
''' '''

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG f20eb34 GIT_TAG 7321fbb
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

2
contrib/CMakeLists.txt
View File

@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED})
PREFIX "jemalloc" PREFIX "jemalloc"
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
BUILD_IN_SOURCE 1 BUILD_IN_SOURCE 1
CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto' CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
BUILD_COMMAND ${MAKE} BUILD_COMMAND ${MAKE}
) )
INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)

54
docs/en/14-reference/03-connector/05-go.mdx
View File

@ -353,7 +353,7 @@ The `af` package encapsulates TDengine advanced functions such as connection man
Closes the connection. Closes the connection.
#### Subscribe to #### Subscribe
* `func NewConsumer(conf *Config) (*Consumer, error)` * `func NewConsumer(conf *Config) (*Consumer, error)`
@ -387,7 +387,7 @@ Close consumer.
* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error` * `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error`
Write to influxDB line protocol. Write to InfluxDB line protocol.
* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error` * `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error`
@ -413,7 +413,7 @@ Close consumer.
* `func (stmt *InsertStmt) SetTableName(name string) error` * `func (stmt *InsertStmt) SetTableName(name string) error`
Bind the set table name parameter. Bind the table name parameter.
* `func (stmt *InsertStmt) SetSubTableName(name string) error` * `func (stmt *InsertStmt) SetSubTableName(name string) error`
@ -463,6 +463,54 @@ Close consumer.
For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go)
### parameter binding via WebSocket
* `func NewConnector(config *Config) (*Connector, error)`
Create a connection.
* `func (c *Connector) Init() (*Stmt, error)`
Initialize the parameters.
* `func (c *Connector) Close() error`
Close the connection.
* `func (s *Stmt) Prepare(sql string) error`
Parameter binding preprocessing SQL statement.
* `func (s *Stmt) SetTableName(name string) error`
Bind the table name parameter.
* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error`
Set tags.
* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error`
Parameter bind multiple rows of data.
* `func (s *Stmt) AddBatch() error`
Add to a parameter-bound batch.
* `func (s *Stmt) Exec() error`
Execute a parameter binding.
* `func (s *Stmt) GetAffectedRows() int`
Gets the number of affected rows inserted by the parameter binding.
* `func (s *Stmt) Close() error`
Closes the parameter binding.
For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go)
## API Reference ## API Reference
Full API see [driver-go documentation](https://pkg.go.dev/github.com/taosdata/driver-go/v3) Full API see [driver-go documentation](https://pkg.go.dev/github.com/taosdata/driver-go/v3)

52
docs/zh/08-connector/20-go.mdx
View File

@ -8,7 +8,7 @@ title: TDengine Go Connector
import Tabs from '@theme/Tabs'; import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem'; import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx" import Preparation from "./_preparation.mdx"
import GoInsert from "../07-develop/03-insert-data/_go_sql.mdx" import GoInsert from "../07-develop/03-insert-data/_go_sql.mdx"
import GoInfluxLine from "../07-develop/03-insert-data/_go_line.mdx" import GoInfluxLine from "../07-develop/03-insert-data/_go_line.mdx"
import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx" import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx"
@ -389,7 +389,7 @@ func main() {
* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error` * `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error`
写入 influxDB 行协议。 写入 InfluxDB 行协议。
* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error` * `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error`
@ -465,6 +465,54 @@ func main() {
完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) 完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go)
### 通过 WebSocket 进行参数绑定
* `func NewConnector(config *Config) (*Connector, error)`
创建连接。
* `func (c *Connector) Init() (*Stmt, error)`
初始化参数。
* `func (c *Connector) Close() error`
关闭连接。
* `func (s *Stmt) Prepare(sql string) error`
参数绑定预处理 SQL 语句。
* `func (s *Stmt) SetTableName(name string) error`
参数绑定设置表名。
* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error`
参数绑定设置标签。
* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error`
参数绑定多行数据。
* `func (s *Stmt) AddBatch() error`
添加到参数绑定批处理。
* `func (s *Stmt) Exec() error`
执行参数绑定。
* `func (s *Stmt) GetAffectedRows() int`
获取参数绑定插入受影响行数。
* `func (s *Stmt) Close() error`
结束参数绑定。
完整参数绑定示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go)
## API 参考 ## API 参考
全部 API 见 [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3) 全部 API 见 [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3)

2
docs/zh/08-connector/26-rust.mdx
View File

@ -7,7 +7,7 @@ title: TDengine Rust Connector
import Tabs from '@theme/Tabs'; import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem'; import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx" import Preparation from "./_preparation.mdx"
import RustInsert from "../07-develop/03-insert-data/_rust_sql.mdx" import RustInsert from "../07-develop/03-insert-data/_rust_sql.mdx"
import RustBind from "../07-develop/03-insert-data/_rust_stmt.mdx" import RustBind from "../07-develop/03-insert-data/_rust_stmt.mdx"
import RustQuery from "../07-develop/04-query-data/_rust.mdx" import RustQuery from "../07-develop/04-query-data/_rust.mdx"

2
docs/zh/08-connector/35-node.mdx
View File

@ -7,7 +7,7 @@ title: TDengine Node.js Connector
import Tabs from "@theme/Tabs"; import Tabs from "@theme/Tabs";
import TabItem from "@theme/TabItem"; import TabItem from "@theme/TabItem";
import Preparition from "./_preparition.mdx"; import Preparation from "./_preparation.mdx";
import NodeInsert from "../07-develop/03-insert-data/_js_sql.mdx"; import NodeInsert from "../07-develop/03-insert-data/_js_sql.mdx";
import NodeInfluxLine from "../07-develop/03-insert-data/_js_line.mdx"; import NodeInfluxLine from "../07-develop/03-insert-data/_js_line.mdx";
import NodeOpenTSDBTelnet from "../07-develop/03-insert-data/_js_opts_telnet.mdx"; import NodeOpenTSDBTelnet from "../07-develop/03-insert-data/_js_opts_telnet.mdx";

2
docs/zh/08-connector/40-csharp.mdx
View File

@ -7,7 +7,7 @@ title: C# Connector
import Tabs from '@theme/Tabs'; import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem'; import TabItem from '@theme/TabItem';
import Preparition from "./_preparition.mdx" import Preparation from "./_preparation.mdx"
import CSInsert from "../07-develop/03-insert-data/_cs_sql.mdx" import CSInsert from "../07-develop/03-insert-data/_cs_sql.mdx"
import CSInfluxLine from "../07-develop/03-insert-data/_cs_line.mdx" import CSInfluxLine from "../07-develop/03-insert-data/_cs_line.mdx"
import CSOpenTSDBTelnet from "../07-develop/03-insert-data/_cs_opts_telnet.mdx" import CSOpenTSDBTelnet from "../07-develop/03-insert-data/_cs_opts_telnet.mdx"

0
docs/zh/08-connector/_preparition.mdx → docs/zh/08-connector/_preparation.mdx
View File

2
docs/zh/12-taos-sql/24-show.md
View File

@ -201,7 +201,7 @@ SHOW TOPICS;
SHOW TRANSACTIONS; SHOW TRANSACTIONS;
``` ```
显示当前系统中正在执行的事务的信息 显示当前系统中正在执行的事务的信息(该事务仅针对除普通表以外的元数据级别)
## SHOW USERS ## SHOW USERS

8
include/common/tcommon.h
View File

@ -164,6 +164,14 @@ typedef enum EStreamType {
STREAM_FILL_OVER, STREAM_FILL_OVER,
} EStreamType; } EStreamType;
typedef struct {
SArray* pGroupList;
SArray* pTableList;
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
bool needSortTableByGroupId;
uint64_t suid;
} STableListInfo;
#pragma pack(push, 1) #pragma pack(push, 1)
typedef struct SColumnDataAgg { typedef struct SColumnDataAgg {
int16_t colId; int16_t colId;

4
include/common/tglobal.h
View File

@ -60,8 +60,8 @@ extern int32_t tsNumOfVnodeSyncThreads;
extern int32_t tsNumOfVnodeRsmaThreads; extern int32_t tsNumOfVnodeRsmaThreads;
extern int32_t tsNumOfQnodeQueryThreads; extern int32_t tsNumOfQnodeQueryThreads;
extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeSharedThreads; extern int32_t tsNumOfSnodeStreamThreads;
extern int32_t tsNumOfSnodeUniqueThreads; extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsRpcQueueMemoryAllowed; extern int64_t tsRpcQueueMemoryAllowed;
// monitor // monitor

3
include/common/tmsg.h
View File

@ -1726,6 +1726,9 @@ typedef struct {
#define STREAM_TRIGGER_WINDOW_CLOSE 2 #define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3 #define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_DEFAULT_IGNORE_EXPIRED 0 #define STREAM_DEFAULT_IGNORE_EXPIRED 0
#define STREAM_FILL_HISTORY_ON 1
#define STREAM_FILL_HISTORY_OFF 0
#define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF
typedef struct { typedef struct {
char name[TSDB_STREAM_FNAME_LEN]; char name[TSDB_STREAM_FNAME_LEN];

259
include/common/ttokendef.h
View File

@ -203,135 +203,136 @@
#define TK_WINDOW_CLOSE 185 #define TK_WINDOW_CLOSE 185
#define TK_IGNORE 186 #define TK_IGNORE 186
#define TK_EXPIRED 187 #define TK_EXPIRED 187
#define TK_SUBTABLE 188 #define TK_FILL_HISTORY 188
#define TK_KILL 189 #define TK_SUBTABLE 189
#define TK_CONNECTION 190 #define TK_KILL 190
#define TK_TRANSACTION 191 #define TK_CONNECTION 191
#define TK_BALANCE 192 #define TK_TRANSACTION 192
#define TK_VGROUP 193 #define TK_BALANCE 193
#define TK_MERGE 194 #define TK_VGROUP 194
#define TK_REDISTRIBUTE 195 #define TK_MERGE 195
#define TK_SPLIT 196 #define TK_REDISTRIBUTE 196
#define TK_DELETE 197 #define TK_SPLIT 197
#define TK_INSERT 198 #define TK_DELETE 198
#define TK_NULL 199 #define TK_INSERT 199
#define TK_NK_QUESTION 200 #define TK_NULL 200
#define TK_NK_ARROW 201 #define TK_NK_QUESTION 201
#define TK_ROWTS 202 #define TK_NK_ARROW 202
#define TK_TBNAME 203 #define TK_ROWTS 203
#define TK_QSTART 204 #define TK_TBNAME 204
#define TK_QEND 205 #define TK_QSTART 205
#define TK_QDURATION 206 #define TK_QEND 206
#define TK_WSTART 207 #define TK_QDURATION 207
#define TK_WEND 208 #define TK_WSTART 208
#define TK_WDURATION 209 #define TK_WEND 209
#define TK_IROWTS 210 #define TK_WDURATION 210
#define TK_QTAGS 211 #define TK_IROWTS 211
#define TK_CAST 212 #define TK_QTAGS 212
#define TK_NOW 213 #define TK_CAST 213
#define TK_TODAY 214 #define TK_NOW 214
#define TK_TIMEZONE 215 #define TK_TODAY 215
#define TK_CLIENT_VERSION 216 #define TK_TIMEZONE 216
#define TK_SERVER_VERSION 217 #define TK_CLIENT_VERSION 217
#define TK_SERVER_STATUS 218 #define TK_SERVER_VERSION 218
#define TK_CURRENT_USER 219 #define TK_SERVER_STATUS 219
#define TK_COUNT 220 #define TK_CURRENT_USER 220
#define TK_LAST_ROW 221 #define TK_COUNT 221
#define TK_CASE 222 #define TK_LAST_ROW 222
#define TK_END 223 #define TK_CASE 223
#define TK_WHEN 224 #define TK_END 224
#define TK_THEN 225 #define TK_WHEN 225
#define TK_ELSE 226 #define TK_THEN 226
#define TK_BETWEEN 227 #define TK_ELSE 227
#define TK_IS 228 #define TK_BETWEEN 228
#define TK_NK_LT 229 #define TK_IS 229
#define TK_NK_GT 230 #define TK_NK_LT 230
#define TK_NK_LE 231 #define TK_NK_GT 231
#define TK_NK_GE 232 #define TK_NK_LE 232
#define TK_NK_NE 233 #define TK_NK_GE 233
#define TK_MATCH 234 #define TK_NK_NE 234
#define TK_NMATCH 235 #define TK_MATCH 235
#define TK_CONTAINS 236 #define TK_NMATCH 236
#define TK_IN 237 #define TK_CONTAINS 237
#define TK_JOIN 238 #define TK_IN 238
#define TK_INNER 239 #define TK_JOIN 239
#define TK_SELECT 240 #define TK_INNER 240
#define TK_DISTINCT 241 #define TK_SELECT 241
#define TK_WHERE 242 #define TK_DISTINCT 242
#define TK_PARTITION 243 #define TK_WHERE 243
#define TK_BY 244 #define TK_PARTITION 244
#define TK_SESSION 245 #define TK_BY 245
#define TK_STATE_WINDOW 246 #define TK_SESSION 246
#define TK_SLIDING 247 #define TK_STATE_WINDOW 247
#define TK_FILL 248 #define TK_SLIDING 248
#define TK_VALUE 249 #define TK_FILL 249
#define TK_NONE 250 #define TK_VALUE 250
#define TK_PREV 251 #define TK_NONE 251
#define TK_LINEAR 252 #define TK_PREV 252
#define TK_NEXT 253 #define TK_LINEAR 253
#define TK_HAVING 254 #define TK_NEXT 254
#define TK_RANGE 255 #define TK_HAVING 255
#define TK_EVERY 256 #define TK_RANGE 256
#define TK_ORDER 257 #define TK_EVERY 257
#define TK_SLIMIT 258 #define TK_ORDER 258
#define TK_SOFFSET 259 #define TK_SLIMIT 259
#define TK_LIMIT 260 #define TK_SOFFSET 260
#define TK_OFFSET 261 #define TK_LIMIT 261
#define TK_ASC 262 #define TK_OFFSET 262
#define TK_NULLS 263 #define TK_ASC 263
#define TK_ABORT 264 #define TK_NULLS 264
#define TK_AFTER 265 #define TK_ABORT 265
#define TK_ATTACH 266 #define TK_AFTER 266
#define TK_BEFORE 267 #define TK_ATTACH 267
#define TK_BEGIN 268 #define TK_BEFORE 268
#define TK_BITAND 269 #define TK_BEGIN 269
#define TK_BITNOT 270 #define TK_BITAND 270
#define TK_BITOR 271 #define TK_BITNOT 271
#define TK_BLOCKS 272 #define TK_BITOR 272
#define TK_CHANGE 273 #define TK_BLOCKS 273
#define TK_COMMA 274 #define TK_CHANGE 274
#define TK_COMPACT 275 #define TK_COMMA 275
#define TK_CONCAT 276 #define TK_COMPACT 276
#define TK_CONFLICT 277 #define TK_CONCAT 277
#define TK_COPY 278 #define TK_CONFLICT 278
#define TK_DEFERRED 279 #define TK_COPY 279
#define TK_DELIMITERS 280 #define TK_DEFERRED 280
#define TK_DETACH 281 #define TK_DELIMITERS 281
#define TK_DIVIDE 282 #define TK_DETACH 282
#define TK_DOT 283 #define TK_DIVIDE 283
#define TK_EACH 284 #define TK_DOT 284
#define TK_FAIL 285 #define TK_EACH 285
#define TK_FILE 286 #define TK_FAIL 286
#define TK_FOR 287 #define TK_FILE 287
#define TK_GLOB 288 #define TK_FOR 288
#define TK_ID 289 #define TK_GLOB 289
#define TK_IMMEDIATE 290 #define TK_ID 290
#define TK_IMPORT 291 #define TK_IMMEDIATE 291
#define TK_INITIALLY 292 #define TK_IMPORT 292
#define TK_INSTEAD 293 #define TK_INITIALLY 293
#define TK_ISNULL 294 #define TK_INSTEAD 294
#define TK_KEY 295 #define TK_ISNULL 295
#define TK_MODULES 296 #define TK_KEY 296
#define TK_NK_BITNOT 297 #define TK_MODULES 297
#define TK_NK_SEMI 298 #define TK_NK_BITNOT 298
#define TK_NOTNULL 299 #define TK_NK_SEMI 299
#define TK_OF 300 #define TK_NOTNULL 300
#define TK_PLUS 301 #define TK_OF 301
#define TK_PRIVILEGE 302 #define TK_PLUS 302
#define TK_RAISE 303 #define TK_PRIVILEGE 303
#define TK_REPLACE 304 #define TK_RAISE 304
#define TK_RESTRICT 305 #define TK_REPLACE 305
#define TK_ROW 306 #define TK_RESTRICT 306
#define TK_SEMI 307 #define TK_ROW 307
#define TK_STAR 308 #define TK_SEMI 308
#define TK_STATEMENT 309 #define TK_STAR 309
#define TK_STRING 310 #define TK_STATEMENT 310
#define TK_TIMES 311 #define TK_STRING 311
#define TK_UPDATE 312 #define TK_TIMES 312
#define TK_VALUES 313 #define TK_UPDATE 313
#define TK_VARIABLE 314 #define TK_VALUES 314
#define TK_VIEW 315 #define TK_VARIABLE 315
#define TK_WAL 316 #define TK_VIEW 316
#define TK_WAL 317
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

4
include/dnode/snode/snode.h
View File

@ -68,8 +68,8 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
* @param pMsg The request message * @param pMsg The request message
* @param pRsp The response message * @param pRsp The response message
*/ */
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp);
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

8
include/libs/catalog/catalog.h
View File

@ -203,6 +203,14 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta);
int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta);
int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* pVgroup, bool* exists);
/** /**
* Force refresh DB's local cached vgroup info. * Force refresh DB's local cached vgroup info.
* @param pCtg (input, got with catalogGetHandle) * @param pCtg (input, got with catalogGetHandle)

1
include/libs/nodes/cmdnodes.h
View File

@ -374,6 +374,7 @@ typedef struct SStreamOptions {
int8_t triggerType; int8_t triggerType;
SNode* pDelay; SNode* pDelay;
SNode* pWatermark; SNode* pWatermark;
int8_t fillHistory;
int8_t ignoreExpired; int8_t ignoreExpired;
} SStreamOptions; } SStreamOptions;

3
include/libs/nodes/plannodes.h
View File

@ -567,6 +567,7 @@ typedef struct SSubplan {
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
SNode* pTagCond; SNode* pTagCond;
SNode* pTagIndexCond; SNode* pTagIndexCond;
bool showRewrite;
} SSubplan; } SSubplan;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
@ -585,8 +586,6 @@ typedef struct SQueryPlan {
SExplainInfo explainInfo; SExplainInfo explainInfo;
} SQueryPlan; } SQueryPlan;
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);
const char* dataOrderStr(EDataOrderLevel order); const char* dataOrderStr(EDataOrderLevel order);
#ifdef __cplusplus #ifdef __cplusplus

1
include/libs/planner/planner.h
View File

@ -61,7 +61,6 @@ int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);
int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen); int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan); int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan);
char* qQueryPlanToString(const SQueryPlan* pPlan);
SQueryPlan* qStringToQueryPlan(const char* pStr); SQueryPlan* qStringToQueryPlan(const char* pStr);
void qDestroyQueryPlan(SQueryPlan* pPlan); void qDestroyQueryPlan(SQueryPlan* pPlan);

2
include/libs/stream/streamState.h
View File

@ -99,6 +99,8 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
// char* streamStateSessionDump(SStreamState* pState);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

49
packaging/cfg/taos.cfg
View File

@ -5,42 +5,50 @@
# # # #
######################################################## ########################################################
# The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started ######### 0. Client only configurations #############
# The interval for TDengine CLI to send heartbeat to mnode
# shellActivityTimer 3
############### 1. Cluster End point ############################
# The end point of the first dnode in the cluster to be connected to when this dnode or a TDengine CLI `taos` is started
# firstEp hostname:6030 # firstEp hostname:6030
# The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started # The end point of the second dnode to be connected to if the firstEp is not available
# secondEp # secondEp
# The FQDN of the host where `taosd` will be started. It can be IP address
############### 2. Configuration Parameters of current dnode #####
# The FQDN of the host on which this dnode will be started. It can be IP address
# fqdn hostname # fqdn hostname
# The port for external access after `taosd` is started # The port for external access after this dnode is started
# serverPort 6030 # serverPort 6030
# The maximum number of connections a dnode can accept # The maximum number of connections a dnode can accept
# maxShellConns 5000 # maxShellConns 5000
# The directory for writing log files # The directory for writing log files, if you are using Windows platform please change to Windows path
# logDir /var/log/taos # logDir /var/log/taos
# All data files are stored in this directory # All data files are stored in this directory, if you are using Windows platform please change to Windows path
# dataDir /var/lib/taos # dataDir /var/lib/taos
# temporary file's directory # temporary file's directory, if you are using Windows platform please change to Windows path
# tempDir /tmp/ # tempDir /tmp/
# Switch for allowing TDengine to collect and report service usage information # Switch for allowing TDengine to collect and report service usage information
# telemetryReporting 1 # telemetryReporting 1
# The maximum number of vnodes supported by dnode # The maximum number of vnodes supported by this dnode
# supportVnodes 0 # supportVnodes 0
# The interval of dnode reporting status to mnode # The interval of this dnode reporting status to mnode, [1..10] seconds
# statusInterval 1 # statusInterval 1
# The interval for TDengine CLI to send heartbeat to mnode
# shellActivityTimer 3
# The minimum sliding window time, milli-second # The minimum sliding window time, milli-second
# minSlidingTime 10 # minSlidingTime 10
@ -66,13 +74,14 @@
# system time zone # system time zone
# timezone Asia/Shanghai (CST, +0800) # timezone Asia/Shanghai (CST, +0800)
# system time zone (for windows 10) # system time zone (for windows 10)
# timezone UTC-8 # timezone UTC-8
# system locale # system locale
# locale en_US.UTF-8 # locale en_US.UTF-8
# default system charset # system charset
# charset UTF-8 # charset UTF-8
# stop writing logs when the disk size of the log folder is less than this value # stop writing logs when the disk size of the log folder is less than this value
@ -81,7 +90,7 @@
# stop writing temporary files when the disk size of the tmp folder is less than this value # stop writing temporary files when the disk size of the tmp folder is less than this value
# minimalTmpDirGB 1.0 # minimalTmpDirGB 1.0
# if disk free space is less than this value, taosd service exit directly within startup process # if free disk space is less than this value, this dnode will fail to start
# minimalDataDirGB 2.0 # minimalDataDirGB 2.0
# enable/disable system monitor # enable/disable system monitor
@ -91,14 +100,18 @@
# max number of lines per log filters # max number of lines per log filters
# numOfLogLines 10000000 # numOfLogLines 10000000
# enable/disable async log # write log in async way: 1 - async, 0 - sync
# asyncLog 1 # asyncLog 1
# time of keeping log files, days # time period of keeping log files, in days
# logKeepDays 0 # logKeepDays 0
# The following parameters are used for debug purpose only.
# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR ############ 3. Debug Flag and levels #############################################
# The following parameters are used for debug purpose only by this dnode.
# debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
# Available debug levels are:
# 131: output warning and error # 131: output warning and error
# 135: output debug, warning and error # 135: output debug, warning and error
# 143: output trace, debug, warning and error to log # 143: output trace, debug, warning and error to log

38
source/common/src/tglobal.c
View File

@ -55,8 +55,8 @@ int32_t tsNumOfVnodeSyncThreads = 2;
int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfVnodeRsmaThreads = 2;
int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeQueryThreads = 4;
int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfQnodeFetchThreads = 1;
int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeStreamThreads = 4;
int32_t tsNumOfSnodeUniqueThreads = 2; int32_t tsNumOfSnodeWriteThreads = 1;
// monitor // monitor
bool tsEnableMonitor = true; bool tsEnableMonitor = true;
@ -133,7 +133,7 @@ int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
// stream scheduler // stream scheduler
bool tsSchedStreamToSnode = true; bool tsDeployOnSnode = true;
/* /*
* minimum scale for whole system, millisecond by default * minimum scale for whole system, millisecond by default
@ -390,13 +390,13 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
// tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
// if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; // if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
tsNumOfSnodeSharedThreads = tsNumOfCores / 4; tsNumOfSnodeStreamThreads = tsNumOfCores / 4;
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4); tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeSharedThreads, 2, 1024, 0) != 0) return -1; if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, 0) != 0) return -1;
tsNumOfSnodeUniqueThreads = tsNumOfCores / 4; tsNumOfSnodeWriteThreads = tsNumOfCores / 4;
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4); tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 2, 1024, 0) != 0) return -1; if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, 0) != 0) return -1;
tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
@ -542,17 +542,17 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads"); pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfSnodeSharedThreads = numOfCores / 4; tsNumOfSnodeStreamThreads = numOfCores / 4;
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4); tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
pItem->i32 = tsNumOfSnodeSharedThreads; pItem->i32 = tsNumOfSnodeStreamThreads;
pItem->stype = stype; pItem->stype = stype;
} }
pItem = cfgGetItem(tsCfg, "numOfSnodeUniqueThreads"); pItem = cfgGetItem(tsCfg, "numOfSnodeUniqueThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfSnodeUniqueThreads = numOfCores / 4; tsNumOfSnodeWriteThreads = numOfCores / 4;
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4); tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
pItem->i32 = tsNumOfSnodeUniqueThreads; pItem->i32 = tsNumOfSnodeWriteThreads;
pItem->stype = stype; pItem->stype = stype;
} }
@ -696,8 +696,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; // tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@ -946,9 +946,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
*/ */
} else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) {
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
} else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) {
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
} else if (strcasecmp("numOfLogLines", name) == 0) { } else if (strcasecmp("numOfLogLines", name) == 0) {
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
} }

1
source/common/src/tmsg.c
View File

@ -4996,6 +4996,7 @@ int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStrea
} }
void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
taosArrayDestroy(pReq->pTags);
taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->ast); taosMemoryFreeClear(pReq->ast);
} }

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -124,6 +124,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

13
source/dnode/mgmt/mgmt_snode/inc/smInt.h
View File

@ -30,9 +30,9 @@ typedef struct SSnodeMgmt {
SMsgCb msgCb; SMsgCb msgCb;
const char *path; const char *path;
const char *name; const char *name;
int8_t uniqueWorkerInUse; int8_t writeWorkerInUse;
SArray *uniqueWorkers; // SArray<SMultiWorker*> SArray *writeWroker; // SArray<SMultiWorker*>
SSingleWorker sharedWorker; SSingleWorker streamWorker;
} SSnodeMgmt; } SSnodeMgmt;
// smHandle.c // smHandle.c
@ -43,10 +43,11 @@ int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg);
// smWorker.c // smWorker.c
int32_t smStartWorker(SSnodeMgmt *pMgmt); int32_t smStartWorker(SSnodeMgmt *pMgmt);
void smStopWorker(SSnodeMgmt *pMgmt); void smStopWorker(SSnodeMgmt *pMgmt);
int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pMsg);
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

14
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -69,13 +69,13 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0; code = 0;
_OVER: _OVER:

1
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -45,6 +45,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->name = pInput->name; pMgmt->name = pInput->name;
pMgmt->msgCb = pInput->msgCb; pMgmt->msgCb = pInput->msgCb;
pMgmt->msgCb.mgmt = pMgmt; pMgmt->msgCb.mgmt = pMgmt;
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)smPutMsgToQueue;
SSnodeOpt option = {0}; SSnodeOpt option = {0};
smInitOption(pMgmt, &option); smInitOption(pMgmt, &option);

120
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -26,18 +26,24 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) {
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SSnodeMgmt *pMgmt = pInfo->ahandle; SSnodeMgmt *pMgmt = pInfo->ahandle;
for (int32_t i = 0; i < numOfMsgs; i++) { for (int32_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = NULL; SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
const STraceId *trace = &pMsg->info.traceId;
dTrace("msg:%p, get from snode-unique queue", pMsg); dTrace("msg:%p, get from snode-write queue", pMsg);
if (sndProcessUMsg(pMgmt->pSnode, pMsg) < 0) { int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL);
ASSERT(0); if (code < 0) {
dGError("snd, msg:%p failed to process write since %s", pMsg, terrstr(code));
if (pMsg->info.handle != NULL) {
tmsgSendRsp(pMsg);
} }
} else {
smSendRsp(pMsg, 0); smSendRsp(pMsg, 0);
}
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
@ -45,13 +51,15 @@ static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t num
} }
} }
static void smProcessSharedQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = pInfo->ahandle; SSnodeMgmt *pMgmt = pInfo->ahandle;
const STraceId *trace = &pMsg->info.traceId;
dTrace("msg:%p, get from snode-shared queue", pMsg); dTrace("msg:%p, get from snode-stream queue", pMsg);
if (sndProcessSMsg(pMgmt->pSnode, pMsg) < 0) { int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
if (code < 0) {
dGError("snd, msg:%p failed to process stream since %s", pMsg, terrstr(code));
smSendRsp(pMsg, terrno); smSendRsp(pMsg, terrno);
ASSERT(0);
} }
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
@ -60,44 +68,44 @@ static void smProcessSharedQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t smStartWorker(SSnodeMgmt *pMgmt) {
pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(SMultiWorker *)); pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->uniqueWorkers == NULL) { if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
for (int32_t i = 0; i < tsNumOfSnodeUniqueThreads; i++) { for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker)); SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pUniqueWorker == NULL) { if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
SMultiWorkerCfg cfg = { SMultiWorkerCfg cfg = {
.max = 1, .max = 1,
.name = "snode-unique", .name = "snode-write",
.fp = smProcessUniqueQueue, .fp = smProcessWriteQueue,
.param = pMgmt, .param = pMgmt,
}; };
if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) { if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr()); dError("failed to start snode-unique worker since %s", terrstr());
return -1; return -1;
} }
if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) { if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
} }
SSingleWorkerCfg cfg = { SSingleWorkerCfg cfg = {
.min = tsNumOfSnodeSharedThreads, .min = tsNumOfSnodeStreamThreads,
.max = tsNumOfSnodeSharedThreads, .max = tsNumOfSnodeStreamThreads,
.name = "snode-shared", .name = "snode-stream",
.fp = (FItem)smProcessSharedQueue, .fp = (FItem)smProcessStreamQueue,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->sharedWorker, &cfg)) { if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) {
dError("failed to start snode shared-worker since %s", terrstr()); dError("failed to start snode shared-worker since %s", terrstr());
return -1; return -1;
} }
@ -107,30 +115,50 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
} }
void smStopWorker(SSnodeMgmt *pMgmt) { void smStopWorker(SSnodeMgmt *pMgmt) {
for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { for (int32_t i = 0; i < taosArrayGetSize(pMgmt->writeWroker); i++) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, i);
tMultiWorkerCleanup(pWorker); tMultiWorkerCleanup(pWorker);
} }
taosArrayDestroy(pMgmt->uniqueWorkers); taosArrayDestroy(pMgmt->writeWroker);
tSingleWorkerCleanup(&pMgmt->sharedWorker); tSingleWorkerCleanup(&pMgmt->streamWorker);
dDebug("snode workers are closed"); dDebug("snode workers are closed");
} }
static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SMsgHead *pHead = pMsg->pCont; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
pHead->vgId = htonl(pHead->vgId); if (pMsg == NULL) {
return pHead->vgId % tsNumOfSnodeUniqueThreads; rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
} }
static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { SSnode *pSnode = pMgmt->pSnode;
/*SMsgHead *pHead = pMsg->pCont;*/ if (pSnode == NULL) {
/*pHead->workerType = htonl(pHead->workerType);*/ dError("snode: msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pMsg, terrstr(),
/*return pHead->workerType;*/ TMSG_INFO(pMsg->msgType), qtype);
return -1;
}
SMsgHead *pHead = pRpc->pCont;
pHead->contLen = htonl(pHead->contLen);
pHead->vgId = SNODE_HANDLE;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
switch (qtype) {
case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break;
case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break;
default:
ASSERT(0);
}
return 0; return 0;
} }
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
@ -141,9 +169,8 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t index = smGetSWIdFromMsg(pMsg); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
@ -154,19 +181,14 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->sharedWorker; SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); if (pMsg->msgType == TDMT_STREAM_TASK_DISPATCH) {
return 0; sndEnqueueStreamDispatch(pMgmt->pSnode, pMsg);
}
int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t workerType = smGetSWTypeFromMsg(pMsg);
if (workerType == SND_WORKER_TYPE__SHARED) {
return smPutNodeMsgToSharedQueue(pMgmt, pMsg);
} else { } else {
return smPutNodeMsgToUniqueQueue(pMgmt, pMsg); taosWriteQitem(pWorker->queue, pMsg);
} }
return 0;
} }

1
source/dnode/mgmt/test/sut/inc/server.h
View File

@ -20,6 +20,7 @@ class TestServer {
public: public:
bool Start(); bool Start();
void Stop(); void Stop();
bool runnning;
private: private:
TdThread threadId; TdThread threadId;

18
source/dnode/mgmt/test/sut/src/server.cpp
View File

@ -16,8 +16,18 @@
#include "sut.h" #include "sut.h"
void* serverLoop(void* param) { void* serverLoop(void* param) {
dmInit(); TestServer* server = (TestServer*)param;
dmRun(); server->runnning = false;
if (dmInit() != 0) {
return NULL;
}
server->runnning = true;
if (dmRun() != 0) {
return NULL;
}
dmCleanup(); dmCleanup();
return NULL; return NULL;
} }
@ -26,10 +36,10 @@ bool TestServer::Start() {
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
taosThreadCreate(&threadId, &thAttr, serverLoop, NULL); taosThreadCreate(&threadId, &thAttr, serverLoop, this);
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
taosMsleep(2100); taosMsleep(2100);
return true; return runnning;
} }
void TestServer::Stop() { void TestServer::Stop() {

5
source/dnode/mgmt/test/sut/src/sut.cpp
View File

@ -53,7 +53,10 @@ void Testbase::Init(const char* path, int16_t port) {
taosMkDir(path); taosMkDir(path);
InitLog(TD_TMP_DIR_PATH "td"); InitLog(TD_TMP_DIR_PATH "td");
server.Start(); if (!server.Start()) {
printf("failed to start server, exit\n");
exit(0);
};
client.Init("root", "taosdata"); client.Init("root", "taosdata");
showRsp = NULL; showRsp = NULL;
} }

2
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -80,7 +80,7 @@ typedef struct {
} SProfileMgmt; } SProfileMgmt;
typedef struct { typedef struct {
SRWLatch lock; TdThreadMutex lock;
char email[TSDB_FQDN_LEN]; char email[TSDB_FQDN_LEN];
} STelemMgmt; } STelemMgmt;

6
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -32,7 +32,7 @@
#include "tname.h" #include "tname.h"
#include "tuuid.h" #include "tuuid.h"
extern bool tsSchedStreamToSnode; extern bool tsDeployOnSnode;
static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
int32_t childId = taosArrayGetSize(pArray); int32_t childId = taosArrayGetSize(pArray);
@ -190,7 +190,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan,
pTask->nodeId = SNODE_HANDLE; pTask->nodeId = SNODE_HANDLE;
pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode); pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode);
plan->execNode.nodeId = 0; plan->execNode.nodeId = SNODE_HANDLE;
plan->execNode.epSet = pTask->epSet; plan->execNode.epSet = pTask->epSet;
if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
@ -373,7 +373,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return -1; return -1;
} }
if (tsSchedStreamToSnode) { if (tsDeployOnSnode) {
SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode);
if (pSnode == NULL) { if (pSnode == NULL) {
SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid);

12
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -127,8 +127,10 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) {
STelemMgmt* pMgmt = &pMnode->telemMgmt; STelemMgmt* pMgmt = &pMnode->telemMgmt;
if (!tsEnableTelem) return 0; if (!tsEnableTelem) return 0;
taosWLockLatch(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
char* pCont = mndBuildTelemetryReport(pMnode); char* pCont = mndBuildTelemetryReport(pMnode);
taosThreadMutexUnlock(&pMgmt->lock);
if (pCont != NULL) { if (pCont != NULL) {
if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) { if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) {
mError("failed to send telemetry report"); mError("failed to send telemetry report");
@ -137,18 +139,20 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) {
} }
taosMemoryFree(pCont); taosMemoryFree(pCont);
} }
taosWUnLockLatch(&pMgmt->lock);
return 0; return 0;
} }
int32_t mndInitTelem(SMnode* pMnode) { int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt; STelemMgmt* pMgmt = &pMnode->telemMgmt;
taosInitRWLatch(&pMgmt->lock); taosThreadMutexInit(&pMgmt->lock, NULL);
taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
return 0; return 0;
} }
void mndCleanupTelem(SMnode* pMnode) {} void mndCleanupTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
taosThreadMutexDestroy(&pMgmt->lock);
}

89
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -60,7 +60,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg);
mndSetMsgHandle(pMnode, TDMT_MND_MERGE_VGROUP, mndProcessSplitVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg);
mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups);
@ -617,35 +617,6 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
return epset; return epset;
} }
static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
SVgObj *pVgroup = pObj;
int64_t uid = *(int64_t *)p1;
int8_t *pReplica = p2;
int32_t *pNumOfVgroups = p3;
if (pVgroup->dbUid == uid) {
*pReplica = TMAX(*pReplica, pVgroup->replica);
(*pNumOfVgroups)++;
}
return true;
}
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
*pReplica = 1;
*pNumOfVgroups = 0;
sdbTraverse(pSdb, SDB_VGROUP, mndGetVgroupMaxReplicaFp, &pDb->uid, pReplica, pNumOfVgroups);
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -1670,35 +1641,81 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
if (newVg1.replica == 1) { if (newVg1.replica == 1) {
if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER; if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, -1) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
} else if (newVg1.replica == 3) { } else if (newVg1.replica == 3) {
SVnodeGid del1 = {0}; SVnodeGid del1 = {0};
if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER; if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, -1) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
} else { } else {
goto _OVER; goto _OVER;
} }
mInfo("vgId:%d, vgroup info after adjust replica, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
SVgObj newVg2 = {0}; SVgObj newVg2 = {0};
memcpy(&newVg1, &newVg2, sizeof(SVgObj)); memcpy(&newVg2, &newVg1, sizeof(SVgObj));
newVg1.replica = 1; newVg1.replica = 1;
newVg1.hashEnd = (newVg1.hashBegin + newVg1.hashEnd) / 2; newVg1.hashEnd = newVg1.hashBegin / 2 + newVg1.hashEnd / 2;
memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));
newVg2.replica = 1; newVg2.replica = 1;
newVg2.hashBegin = newVg1.hashEnd + 1; newVg2.hashBegin = newVg1.hashEnd + 1;
memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid)); memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid));
memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); memset(&newVg2.vnodeGid[1], 0, sizeof(SVnodeGid));
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId,
newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER;
// adjust vgroup #if 0
// adjust vgroup replica
if (pDb->cfg.replications != newVg1.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
}
if (pDb->cfg.replications != newVg2.replica) {
if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
}
#endif
{
pRaw = mndVgroupActionEncode(&newVg1);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
{
pRaw = mndVgroupActionEncode(&newVg2);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId,
newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);

82
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -367,6 +367,80 @@ bool sdbTraverseFail(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, in
return false; return false;
} }
#ifndef WINDOWS
TEST_F(MndTestSdb, 00_API) {
SMnode mnode = {0};
SSdbOpt opt = {0};
opt.pMnode = &mnode;
opt.path = TD_TMP_DIR_PATH "mnode_test_sdb";
taosRemoveDir(opt.path);
SSdb *pSdb = sdbInit(&opt);
SSdbTable table = {.sdbType = SDB_USER, .keyType = SDB_KEY_BINARY};
sdbSetTable(pSdb, table);
// sdbRow.c
SSdbRow *pRow1 = sdbAllocRow(-128);
ASSERT_EQ(pRow1 == NULL, 1);
void *pRow2 = sdbGetRowObj(NULL);
ASSERT_EQ(pRow2 == NULL, 1);
//sdbRaw.c
SStrObj strObj;
SSdbRaw *pRaw1 = NULL;
strSetDefault(&strObj, 1);
pRaw1 = strEncode(&strObj);
int32_t id = sdbGetIdFromRaw(pSdb, pRaw1);
ASSERT_EQ(id, -2);
SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128);
ASSERT_EQ(pRaw2 == NULL, 1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), -1);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), -1);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), -1);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), -1);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), -1);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), -1);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), -1);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), -1);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), -1);
ASSERT_EQ(sdbGetRawTotalSize(NULL), -1);
// sdbHash.c
EXPECT_STREQ(sdbTableName((ESdbType)100), "undefine");
EXPECT_STREQ(sdbStatusName((ESdbStatus)100), "undefine");
ASSERT_EQ(sdbGetTableVer(pSdb, (ESdbType)100), -1);
SSdbRaw *pRaw3 = sdbAllocRaw((ESdbType)-12, 1, 128);
ASSERT_NE(sdbWriteWithoutFree(pSdb, pRaw3), 0);
pSdb->hashObjs[1] = NULL;
SSdbRaw *pRaw4 = sdbAllocRaw((ESdbType)1, 1, 128);
ASSERT_NE(sdbWriteWithoutFree(pSdb, pRaw4), 0);
}
#endif
TEST_F(MndTestSdb, 01_Write_Str) { TEST_F(MndTestSdb, 01_Write_Str) {
void *pIter = NULL; void *pIter = NULL;
int32_t num = 0; int32_t num = 0;
@ -885,8 +959,8 @@ TEST_F(MndTestSdb, 01_Read_Str) {
ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); ASSERT_EQ(sdbWrite(pSdb, pRaw), 0);
pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key);
ASSERT_EQ(pI32Obj, nullptr); ASSERT_EQ(pI32Obj, nullptr);
int32_t code = terrno; // int32_t code = terrno;
ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_DROPPING); // ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_DROPPING);
} }
{ {
@ -899,8 +973,8 @@ TEST_F(MndTestSdb, 01_Read_Str) {
ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); ASSERT_EQ(sdbWrite(pSdb, pRaw), 0);
pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key);
ASSERT_EQ(pI32Obj, nullptr); ASSERT_EQ(pI32Obj, nullptr);
int32_t code = terrno; // int32_t code = terrno;
ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING); // ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING);
} }
{ {

3
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -654,15 +654,12 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
} }
if (config > 0) { if (config > 0) {
ASSERT(pSdb->commitConfig == config);
pSdb->commitConfig = config; pSdb->commitConfig = config;
} }
if (term > 0) { if (term > 0) {
ASSERT(pSdb->commitTerm == term);
pSdb->commitTerm = term; pSdb->commitTerm = term;
} }
if (index > 0) { if (index > 0) {
ASSERT(pSdb->commitIndex == index);
pSdb->commitIndex = index; pSdb->commitIndex = index;
} }

4
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -241,9 +241,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) {
SdbDecodeFp decodeFp = pSdb->decodeFps[pRaw->type]; SdbDecodeFp decodeFp = pSdb->decodeFps[pRaw->type];
SSdbRow *pRow = (*decodeFp)(pRaw); SSdbRow *pRow = (*decodeFp)(pRaw);
if (pRow == NULL) { if (pRow == NULL) return terrno;
return terrno;
}
pRow->type = pRaw->type; pRow->type = pRaw->type;

2
source/dnode/snode/inc/sndInt.h
View File

@ -31,6 +31,8 @@ extern "C" {
#endif #endif
typedef struct SSnode { typedef struct SSnode {
char* path;
SStreamMeta* pMeta;
SMsgCb msgCb; SMsgCb msgCb;
} SSnode; } SSnode;

328
source/dnode/snode/src/snode.c
View File

@ -15,197 +15,197 @@
#include "executor.h" #include "executor.h"
#include "sndInt.h" #include "sndInt.h"
#include "tstream.h"
#include "tuuid.h" #include "tuuid.h"
/*SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { return NULL; }*/
/*void sndClose(SSnode *pSnode) {}*/ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) {
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } char *msgStr = pMsg->pCont;
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamDispatchReq req;
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
if (tDecodeStreamDispatchReq(&decoder, &req) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
tDecoderClear(&decoder);
goto FAIL;
}
tDecoderClear(&decoder);
int32_t taskId = req.taskId;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessDispatchReq(pTask, &req, &rsp, false);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
return;
}
FAIL:
if (pMsg->info.handle == NULL) return;
SRpcMsg rsp = {
.code = code,
.info = pMsg->info,
};
tmsgSendRsp(&rsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->inputQueue = streamQueueOpen();
pTask->outputQueue = streamQueueOpen();
if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) {
return -1;
}
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
pTask->pMsgCb = &pSnode->msgCb;
pTask->startVer = ver;
pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
if (pTask->pState == NULL) {
return -1;
}
SReadHandle mgHandle = {
.vnode = NULL,
.numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
.pStateBackend = pTask->pState,
};
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
ASSERT(pTask->exec.executor);
return 0;
}
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) { if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pSnode->path = strdup(path);
if (pSnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
#if 0
pSnode->pMeta = sndMetaNew(); pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE);
if (pSnode->pMeta == NULL) { if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
return pSnode;
FAIL:
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode); taosMemoryFree(pSnode);
return NULL; return NULL;
} }
#endif
return pSnode;
}
void sndClose(SSnode *pSnode) { void sndClose(SSnode *pSnode) {
/*sndMetaDelete(pSnode->pMeta);*/ streamMetaClose(pSnode->pMeta);
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode); taosMemoryFree(pSnode);
} }
int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; }
#if 0 int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
SStreamMeta *sndMetaNew() { int32_t code;
SStreamMeta *pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
if (pMeta == NULL) {
return NULL;
}
pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pMeta->pHash == NULL) {
taosMemoryFree(pMeta);
return NULL;
}
return pMeta;
}
void sndMetaDelete(SStreamMeta *pMeta) {
taosHashCleanup(pMeta->pHash);
taosMemoryFree(pMeta);
}
int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) {
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *));
}
SStreamTask *sndMetaGetTask(SStreamMeta *pMeta, int32_t taskId) {
return taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
}
int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) {
SStreamTask *pTask = taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
if (pTask == NULL) {
return -1;
}
taosMemoryFree(pTask->exec.qmsg);
// TODO:free executor
taosMemoryFree(pTask);
return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t));
}
static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
// 1.deserialize msg and build task
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) { if (pTask == NULL) {
return -1; return -1;
} }
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t *)msg, msgLen); tDecoderInit(&decoder, (uint8_t *)msg, msgLen);
if (tDecodeSStreamTask(&decoder, pTask) < 0) { code = tDecodeSStreamTask(&decoder, pTask);
ASSERT(0); if (code < 0) {
tDecoderClear(&decoder);
taosMemoryFree(pTask);
return -1;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
pTask->execStatus = TASK_EXEC_STATUS__IDLE; ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
pTask->inputQueue = streamQueueOpen(); // 2.save task
pTask->outputQueue = streamQueueOpen(); code = streamMetaAddTask(pSnode->pMeta, -1, pTask);
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; if (code < 0) {
pTask->outputStatus = TASK_INPUT_STATUS__NORMAL;
if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) goto FAIL;
pTask->pMsgCb = &pNode->msgCb;
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
ASSERT(pTask->exec.executor);
streamSetupTrigger(pTask);
qInfo("deploy stream: stream id %" PRId64 " task id %d child id %d on snode", pTask->streamId, pTask->taskId,
pTask->selfChildId);
taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));
return 0;
FAIL:
if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
if (pTask) taosMemoryFree(pTask);
return -1; return -1;
} }
static int32_t sndProcessTaskRunReq(SSnode *pNode, SRpcMsg *pMsg) { // 3.go through recover steps to fill history
SStreamMeta *pMeta = pNode->pMeta; if (pTask->fillHistory) {
SStreamTaskRunReq *pReq = pMsg->pCont; streamSetParamForRecover(pTask);
int32_t taskId = pReq->taskId; streamAggRecoverPrepare(pTask);
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t)); }
streamProcessRunReq(pTask);
return 0; return 0;
} }
static int32_t sndProcessTaskDispatchReq(SSnode *pNode, SRpcMsg *pMsg) { int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
SStreamMeta *pMeta = pNode->pMeta; SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
return streamMetaRemoveTask(pSnode->pMeta, pReq->taskId);
}
int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) {
SStreamTaskRunReq *pReq = pMsg->pCont;
int32_t taskId = pReq->taskId;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
streamProcessRunReq(pTask);
return 0;
} else {
return -1;
}
}
int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) {
char *msgStr = pMsg->pCont; char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
SStreamDispatchReq req; SStreamDispatchReq req;
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen); tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen);
tDecodeStreamDispatchReq(&decoder, &req); tDecodeStreamDispatchReq(&decoder, &req);
int32_t taskId = req.taskId; int32_t taskId = req.taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
SRpcMsg rsp = { SRpcMsg rsp = {
.info = pMsg->info, .info = pMsg->info,
.code = 0, .code = 0,
}; };
streamProcessDispatchReq(pTask, &req, &rsp, true); streamProcessDispatchReq(pTask, &req, &rsp, exec);
return 0;
} else {
return -1;
}
return 0; return 0;
} }
static int32_t sndProcessTaskRecoverReq(SSnode *pNode, SRpcMsg *pMsg) { int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamTaskRecoverReq *pReq = pMsg->pCont;
int32_t taskId = pReq->taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessRecoverReq(pTask, pReq, pMsg);
return 0;
}
static int32_t sndProcessTaskDispatchRsp(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t taskId = pRsp->taskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessDispatchRsp(pTask, pRsp);
return 0;
}
static int32_t sndProcessTaskRecoverRsp(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
SStreamTaskRecoverRsp *pRsp = pMsg->pCont;
int32_t taskId = pRsp->rspTaskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
streamProcessRecoverRsp(pTask, pRsp);
return 0;
}
static int32_t sndProcessTaskDropReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msg = pMsg->pCont;
int32_t msgLen = pMsg->contLen;
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
int32_t code = taosHashRemove(pMeta->pHash, &pReq->taskId, sizeof(int32_t));
ASSERT(code == 0);
if (code == 0) {
// sendrsp
}
return code;
}
static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msgStr = pMsg->pCont; char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
@ -213,53 +213,64 @@ static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen); tDecoderInit(&decoder, msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req); tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder);
int32_t taskId = req.dstTaskId; int32_t taskId = req.dstTaskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t)); SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { if (pTask) {
return 0;
}
SRpcMsg rsp = { SRpcMsg rsp = {
.info = pMsg->info, .info = pMsg->info,
.code = 0, .code = 0,
}; };
streamProcessRetrieveReq(pTask, &req, &rsp); streamProcessRetrieveReq(pTask, &req, &rsp);
tDeleteStreamRetrieveReq(&req);
return 0;
} else {
return -1;
}
}
int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) {
SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t taskId = pRsp->taskId;
SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
if (pTask) {
streamProcessDispatchRsp(pTask, pRsp);
return 0;
} else {
return -1;
}
return 0; return 0;
} }
static int32_t sndProcessTaskRetrieveRsp(SSnode *pNode, SRpcMsg *pMsg) { int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) {
// //
return 0; return 0;
} }
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
// stream deploy void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
// stream stop/resume int32_t len = pMsg->contLen - sizeof(SMsgHead);
// operator exec
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_STREAM_TASK_DEPLOY: case TDMT_STREAM_TASK_DEPLOY:
return sndProcessTaskDeployReq(pSnode, pMsg); return sndProcessTaskDeployReq(pSnode, pReq, len);
case TDMT_STREAM_TASK_DROP: case TDMT_STREAM_TASK_DROP:
return sndProcessTaskDropReq(pSnode, pMsg); return sndProcessTaskDropReq(pSnode, pReq, len);
default: default:
ASSERT(0); ASSERT(0);
} }
return 0; return 0;
} }
int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_STREAM_TASK_RUN: case TDMT_STREAM_TASK_RUN:
return sndProcessTaskRunReq(pSnode, pMsg); return sndProcessTaskRunReq(pSnode, pMsg);
case TDMT_STREAM_TASK_DISPATCH: case TDMT_STREAM_TASK_DISPATCH:
return sndProcessTaskDispatchReq(pSnode, pMsg); return sndProcessTaskDispatchReq(pSnode, pMsg, true);
case TDMT_STREAM_TASK_RECOVER:
return sndProcessTaskRecoverReq(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE: case TDMT_STREAM_RETRIEVE:
return sndProcessTaskRecoverReq(pSnode, pMsg); return sndProcessTaskRetrieveReq(pSnode, pMsg);
case TDMT_STREAM_TASK_DISPATCH_RSP: case TDMT_STREAM_TASK_DISPATCH_RSP:
return sndProcessTaskDispatchRsp(pSnode, pMsg); return sndProcessTaskDispatchRsp(pSnode, pMsg);
case TDMT_STREAM_TASK_RECOVER_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE_RSP: case TDMT_STREAM_RETRIEVE_RSP:
return sndProcessTaskRetrieveRsp(pSnode, pMsg); return sndProcessTaskRetrieveRsp(pSnode, pMsg);
default: default:
@ -267,4 +278,3 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
} }
return 0; return 0;
} }
#endif

13
source/dnode/vnode/inc/vnode.h
View File

@ -152,23 +152,26 @@ typedef struct STsdbReader STsdbReader;
#define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST_ROW 0x4
#define CACHESCAN_RETRIEVE_LAST 0x8 #define CACHESCAN_RETRIEVE_LAST 0x8
int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num); int32_t tsdbSetTableId(STsdbReader *pReader, int64_t uid);
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTableList, STsdbReader **ppReader,
STsdbReader **ppReader, const char *idstr); const char *idstr);
void tsdbReaderClose(STsdbReader *pReader); void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader);
bool tsdbTableNextDataBlock(STsdbReader *pReader, int64_t uid);
void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
bool tsdbIsAscendingOrder(STsdbReader *pReader);
void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIdx(SMeta *pMeta);
void *tsdbGetIvtIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta);
uint64_t getReaderMaxVersion(STsdbReader *pReader); uint64_t getReaderMaxVersion(STsdbReader *pReader);
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, uint64_t suid,
int32_t numOfCols, uint64_t suid, void** pReader); void **pReader);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader); void *tsdbCacherowsReaderClose(void *pReader);
int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);

5
source/dnode/vnode/src/inc/tsdb.h
View File

@ -716,10 +716,7 @@ typedef struct SCacheRowsReader {
int32_t numOfCols; int32_t numOfCols;
int32_t type; int32_t type;
int32_t tableIndex; // currently returned result tables int32_t tableIndex; // currently returned result tables
SArray *pTableList; // table id list
STableKeyInfo *pTableList; // table id list
int32_t numOfTables;
SSttBlockLoadInfo *pLoadInfo; SSttBlockLoadInfo *pLoadInfo;
STsdbReadSnap *pReadSnap; STsdbReadSnap *pReadSnap;
SDataFReader *pDataFReader; SDataFReader *pDataFReader;

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -162,7 +162,10 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
SSubmitBlkRsp* pRsp); SSubmitBlkRsp* pRsp);
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
void* pMemRef);
int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list);
// tq // tq
int tqInit(); int tqInit();

3
source/dnode/vnode/src/tq/tq.c
View File

@ -1272,7 +1272,6 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
} }
int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
//
SStreamTaskRunReq* pReq = pMsg->pCont; SStreamTaskRunReq* pReq = pMsg->pCont;
int32_t taskId = pReq->taskId; int32_t taskId = pReq->taskId;
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
@ -1285,7 +1284,6 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
} }
int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
ASSERT(0);
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
@ -1349,7 +1347,6 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
return streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); return streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
} }

2
source/dnode/vnode/src/tq/tqSink.c
View File

@ -42,7 +42,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl
if (metaGetTableEntryByName(&mr, name) < 0) { if (metaGetTableEntryByName(&mr, name) < 0) {
metaReaderClear(&mr); metaReaderClear(&mr);
taosMemoryFree(name); taosMemoryFree(name);
return -1; continue;
} }
int64_t uid = mr.me.uid; int64_t uid = mr.me.uid;

19
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -97,9 +97,10 @@ static void saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea
} }
} }
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList, int32_t numOfCols, uint64_t suid,
uint64_t suid, void** pReader) { void** pReader) {
*pReader = NULL; *pReader = NULL;
SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader)); SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader));
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -110,15 +111,14 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
p->numOfCols = numOfCols; p->numOfCols = numOfCols;
p->suid = suid; p->suid = suid;
if (numOfTables == 0) { if (taosArrayGetSize(pTableIdList) == 0) {
*pReader = p; *pReader = p;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pTableIdList)[0]; STableKeyInfo* pKeyInfo = taosArrayGet(pTableIdList, 0);
p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1, 1); p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1, 1);
p->pTableList = pTableIdList; p->pTableList = pTableIdList;
p->numOfTables = numOfTables;
p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES); p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES);
if (p->transferBuf == NULL) { if (p->transferBuf == NULL) {
@ -205,6 +205,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
SLRUCache* lruCache = pr->pVnode->pTsdb->lruCache; SLRUCache* lruCache = pr->pVnode->pTsdb->lruCache;
LRUHandle* h = NULL; LRUHandle* h = NULL;
SArray* pRow = NULL; SArray* pRow = NULL;
size_t numOfTables = taosArrayGetSize(pr->pTableList);
bool hasRes = false; bool hasRes = false;
SArray* pLastCols = NULL; SArray* pLastCols = NULL;
@ -242,8 +243,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
// retrieve the only one last row of all tables in the uid list. // retrieve the only one last row of all tables in the uid list.
if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
for (int32_t i = 0; i < pr->numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
STableKeyInfo* pKeyInfo = &pr->pTableList[i]; STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i);
code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h); code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -307,8 +308,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
} else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { for (int32_t i = pr->tableIndex; i < numOfTables; ++i) {
STableKeyInfo* pKeyInfo = &pr->pTableList[i]; STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pr->pTableList, i);
code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h); code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;

2
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -290,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
// only apply to the child tables, ordinary tables will not incur this filter procedure. // only apply to the child tables, ordinary tables will not incur this filter procedure.
size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
if (size > 1) { if (size >= 1) {
SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);

41
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -270,7 +270,10 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
} }
} }
static void clearBlockScanInfo(STableBlockScanInfo* p) { static void destroyBlockScanInfo(SHashObj* pTableMap) {
STableBlockScanInfo* p = NULL;
while ((p = taosHashIterate(pTableMap, p)) != NULL) {
p->iterInit = false; p->iterInit = false;
p->iiter.hasVal = false; p->iiter.hasVal = false;
@ -287,12 +290,6 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) {
tMapDataClear(&p->mapData); tMapDataClear(&p->mapData);
} }
static void destroyBlockScanInfo(SHashObj* pTableMap) {
STableBlockScanInfo* p = NULL;
while ((p = taosHashIterate(pTableMap, p)) != NULL) {
clearBlockScanInfo(p);
}
taosHashCleanup(pTableMap); taosHashCleanup(pTableMap);
} }
@ -3455,23 +3452,13 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// TODO refactor: with createDataBlockScanInfo // todo refactor, use arraylist instead
int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num) { int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) {
ASSERT(pReader != NULL); ASSERT(pReader != NULL);
STableBlockScanInfo* p = NULL;
while ((p = taosHashIterate(pReader->status.pTableMap, p)) != NULL) {
clearBlockScanInfo(p);
}
taosHashClear(pReader->status.pTableMap); taosHashClear(pReader->status.pTableMap);
STableKeyInfo* pList = (STableKeyInfo*) pTableList; STableBlockScanInfo info = {.lastKey = 0, .uid = uid};
for(int32_t i = 0; i < num; ++i) {
STableBlockScanInfo info = {.lastKey = 0, .uid = pList[i].uid};
taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
}
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
@ -3507,8 +3494,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
} }
// ====================================== EXPOSED APIs ====================================== // ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader,
STsdbReader** ppReader, const char* idstr) { const char* idstr) {
STimeWindow window = pCond->twindows; STimeWindow window = pCond->twindows;
if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
pCond->twindows.skey += 1; pCond->twindows.skey += 1;
@ -3567,8 +3554,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
if (pReader->pSchema == NULL) { if (pReader->pSchema == NULL) {
tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
} }
} else if (numOfTables > 0) { } else if (taosArrayGetSize(pTableList) > 0) {
STableKeyInfo* pKey = pTableList; STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
if (pReader->pSchema == NULL) { if (pReader->pSchema == NULL) {
tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
@ -3577,7 +3564,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
STsdbReader* p = pReader->innerReader[0] != NULL ? pReader->innerReader[0] : pReader; STsdbReader* p = pReader->innerReader[0] != NULL ? pReader->innerReader[0] : pReader;
pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables); int32_t numOfTables = taosArrayGetSize(pTableList);
pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList->pData, numOfTables);
if (pReader->status.pTableMap == NULL) { if (pReader->status.pTableMap == NULL) {
tsdbReaderClose(pReader); tsdbReaderClose(pReader);
*ppReader = NULL; *ppReader = NULL;
@ -3787,7 +3775,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
return false; return false;
} }
bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { bool tsdbTableNextDataBlock(STsdbReader* pReader, int64_t uid) {
STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
if (pBlockScanInfo == NULL) { // no data block for the table of given uid if (pBlockScanInfo == NULL) { // no data block for the table of given uid
return false; return false;
@ -4186,3 +4174,4 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) {
} }
tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
} }
bool tsdbIsAscendingOrder(STsdbReader* pReader) { return ASCENDING_TRAVERSE(pReader->order); }

9
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -385,9 +385,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return tqProcessPollReq(pVnode->pTq, pMsg); return tqProcessPollReq(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_RUN: case TDMT_STREAM_TASK_RUN:
return tqProcessTaskRunReq(pVnode->pTq, pMsg); return tqProcessTaskRunReq(pVnode->pTq, pMsg);
#if 0
case TDMT_STREAM_TASK_DISPATCH: case TDMT_STREAM_TASK_DISPATCH:
// return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, pInfo->workerId != 0);
return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true);
#endif
/*case TDMT_STREAM_TASK_RECOVER:*/ /*case TDMT_STREAM_TASK_RECOVER:*/
/*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/ /*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/
case TDMT_STREAM_RETRIEVE: case TDMT_STREAM_RETRIEVE:
@ -1163,8 +1164,10 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i); SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i);
int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts); int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts);
if (code) { if (code < 0) {
// TODO terrno = code;
vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64,
TD_VID(pVnode), terrstr(), deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts);
} }
} }
taosArrayDestroy(deleteReq.deleteReqs); taosArrayDestroy(deleteReq.deleteReqs);

536
source/libs/catalog/test/catalogTests.cpp
View File

@ -56,6 +56,13 @@ enum {
CTGT_RSP_STBMETA, CTGT_RSP_STBMETA,
CTGT_RSP_MSTBMETA, CTGT_RSP_MSTBMETA,
CTGT_RSP_INDEXINFO_E, CTGT_RSP_INDEXINFO_E,
CTGT_RSP_USERAUTH,
CTGT_RSP_TBLCFG,
CTGT_RSP_TBLINDEX,
CTGT_RSP_DBCFG,
CTGT_RSP_QNODELIST,
CTGT_RSP_UDF,
CTGT_RSP_SVRVER,
CTGT_RSP_TBMETA_NOT_EXIST, CTGT_RSP_TBMETA_NOT_EXIST,
}; };
@ -71,6 +78,10 @@ int32_t ctgTestVgVersion = 1;
int32_t ctgTestVgNum = 10; int32_t ctgTestVgNum = 10;
int32_t ctgTestColNum = 2; int32_t ctgTestColNum = 2;
int32_t ctgTestTagNum = 1; int32_t ctgTestTagNum = 1;
int32_t ctgTestQnodeNum = 3;
int32_t ctgTestIndexNum = 3;
int32_t ctgTestFuncNum = 2;
int32_t ctgTestFuncType = 3;
int32_t ctgTestSVersion = 1; int32_t ctgTestSVersion = 1;
int32_t ctgTestTVersion = 1; int32_t ctgTestTVersion = 1;
int32_t ctgTestSuid = 2; int32_t ctgTestSuid = 2;
@ -82,6 +93,7 @@ char *ctgTestDbname = "1.db1";
char *ctgTestTablename = "table1"; char *ctgTestTablename = "table1";
char *ctgTestCTablename = "ctable1"; char *ctgTestCTablename = "ctable1";
char *ctgTestSTablename = "stable1"; char *ctgTestSTablename = "stable1";
char *ctgTestUsername = "user1";
char *ctgTestCurrentCTableName = NULL; char *ctgTestCurrentCTableName = NULL;
char *ctgTestCurrentTableName = NULL; char *ctgTestCurrentTableName = NULL;
char *ctgTestCurrentSTableName = NULL; char *ctgTestCurrentSTableName = NULL;
@ -311,6 +323,7 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
strcpy(addr->fqdn, "a0"); strcpy(addr->fqdn, "a0");
addr->port = n + 22; addr->port = n + 22;
} }
vg.numOfTable = i % 2;
taosArrayPush(usedbRsp.pVgroupInfos, &vg); taosArrayPush(usedbRsp.pVgroupInfos, &vg);
} }
@ -531,6 +544,197 @@ void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMs
} }
void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SGetUserAuthRsp userRsp = {0};
strcpy(userRsp.user, ctgTestUsername);
userRsp.version = 1;
userRsp.superAuth = 1;
int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSGetUserAuthRsp(pReq, contLen, &userRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
}
void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableCfgRsp tblRsp = {0};
strcpy(tblRsp.tbName, ctgTestTablename);
tblRsp.numOfColumns = ctgTestColNum;
tblRsp.pSchemas = (SSchema *)taosMemoryMalloc((tblRsp.numOfTags + tblRsp.numOfColumns) * sizeof(SSchema));
SSchema *s = NULL;
s = &tblRsp.pSchemas[0];
s->type = TSDB_DATA_TYPE_TIMESTAMP;
s->colId = 1;
s->bytes = 8;
strcpy(s->name, "ts");
s = &tblRsp.pSchemas[1];
s->type = TSDB_DATA_TYPE_INT;
s->colId = 2;
s->bytes = 4;
strcpy(s->name, "col1");
int32_t contLen = tSerializeSTableCfgRsp(NULL, 0, &tblRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSTableCfgRsp(pReq, contLen, &tblRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
tFreeSTableCfgRsp(&tblRsp);
}
void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
STableIndexRsp tblRsp = {0};
strcpy(tblRsp.tbName, ctgTestSTablename);
tblRsp.pIndex = taosArrayInit(ctgTestIndexNum, sizeof(STableIndexInfo));
STableIndexInfo info = {0};
for (int32_t i = 0; i < ctgTestIndexNum; ++i) {
info.interval = 1 + i;
info.expr = (char*)taosMemoryCalloc(1, 10);
taosArrayPush(tblRsp.pIndex, &info);
}
int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSTableIndexRsp(pReq, contLen, &tblRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
tFreeSTableIndexRsp(&tblRsp);
}
void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
static int32_t idx = 1;
SDbCfgRsp dbRsp = {0};
dbRsp.numOfVgroups = ctgTestVgNum;
int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp);
void *pReq = rpcMallocCont(contLen);
tSerializeSDbCfgRsp(pReq, contLen, &dbRsp);
pRsp->code = 0;
pRsp->contLen = contLen;
pRsp->pCont = pReq;
}
void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SQnodeListRsp qlistRsp = {0};
qlistRsp.qnodeList = taosArrayInit(10, sizeof(SQueryNodeLoad));
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad nodeLoad = {0};
nodeLoad.addr.nodeId = i;
(void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad);
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSQnodeListRsp(pReq, rspLen, &qlistRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
tFreeSQnodeListRsp(&qlistRsp);
}
void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SRetrieveFuncRsp funcRsp = {0};
funcRsp.numOfFuncs = 1;
funcRsp.pFuncInfos = taosArrayInit(1, sizeof(SFuncInfo));
SFuncInfo funcInfo = {0};
strcpy(funcInfo.name, "func1");
funcInfo.funcType = ctgTestFuncType;
(void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo);
int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSRetrieveFuncRsp(pReq, rspLen, &funcRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
tFreeSRetrieveFuncRsp(&funcRsp);
}
void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
rpcFreeCont(pMsg->pCont);
SServerVerRsp verRsp = {0};
strcpy(verRsp.ver, "1.0");
int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp);
void *pReq = rpcMallocCont(rspLen);
tSerializeSServerVerRsp(pReq, rspLen, &verRsp);
pRsp->code = 0;
pRsp->contLen = rspLen;
pRsp->pCont = pReq;
}
void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (pMsg->msgType) {
case TDMT_MND_USE_DB:
ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_VND_TABLE_CFG:
case TDMT_MND_TABLE_CFG:
ctgTestRspTableCfg(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_GET_TABLE_INDEX:
ctgTestRspTableIndex(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_GET_DB_CFG:
ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_QNODE_LIST:
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_RETRIEVE_FUNC:
ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp);
break;
case TDMT_MND_SERVER_VERSION:
ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
break;
default:
break;
}
return;
}
void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (ctgTestRspFunc[ctgTestRspIdx]) { switch (ctgTestRspFunc[ctgTestRspIdx]) {
case CTGT_RSP_VGINFO: case CTGT_RSP_VGINFO:
@ -551,10 +755,32 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp
case CTGT_RSP_INDEXINFO_E: case CTGT_RSP_INDEXINFO_E:
ctgTestRspErrIndexInfo(shandle, pEpSet, pMsg, pRsp); ctgTestRspErrIndexInfo(shandle, pEpSet, pMsg, pRsp);
break; break;
case CTGT_RSP_USERAUTH:
ctgTestRspUserAuth(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBLCFG:
ctgTestRspTableCfg(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_TBMETA_NOT_EXIST: case CTGT_RSP_TBMETA_NOT_EXIST:
ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp); ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp);
break; break;
case CTGT_RSP_TBLINDEX:
ctgTestRspTableIndex(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_DBCFG:
ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_QNODELIST:
ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_UDF:
ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp);
break;
case CTGT_RSP_SVRVER:
ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
break;
default: default:
ctgTestRspAuto(shandle, pEpSet, pMsg, pRsp);
break; break;
} }
@ -976,9 +1202,23 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3); ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n != 1) {
taosMsleep(50); taosMsleep(50);
} else {
break;
} }
}
memset(&vgInfo, 0, sizeof(vgInfo));
bool exists = false;
code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
ASSERT_EQ(exists, true);
ctgTestSetRspTableMeta(); ctgTestSetRspTableMeta();
@ -1020,6 +1260,18 @@ TEST(tableMeta, normalTable) {
taosMemoryFree(tableMeta); taosMemoryFree(tableMeta);
tableMeta = NULL;
catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
SDbVgVersion *dbs = NULL; SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL; SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
@ -1216,6 +1468,21 @@ TEST(tableMeta, superTableCase) {
} }
} }
tableMeta = NULL;
code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 0);
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->uid, ctgTestSuid);
ASSERT_EQ(tableMeta->suid, ctgTestSuid);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
taosMemoryFree(tableMeta);
ctgTestSetRspCTableMeta(); ctgTestSetRspCTableMeta();
tableMeta = NULL; tableMeta = NULL;
@ -2156,6 +2423,15 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList); code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList);
ASSERT_TRUE(code != 0); ASSERT_TRUE(code != 0);
int32_t dbVer = 0;
int64_t dbId = 0;
int32_t tbNum = 0;
code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum);
ASSERT_EQ(code, 0);
ASSERT_EQ(dbVer, ctgTestVgVersion);
ASSERT_EQ(dbId, ctgTestDbId);
ASSERT_EQ(tbNum, ctgTestVgNum / 2);
ctgTestBuildDBVgroup(&dbVgroup); ctgTestBuildDBVgroup(&dbVgroup);
code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup); code = catalogUpdateDBVgInfo(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -2415,6 +2691,264 @@ TEST(rentTest, allRent) {
catalogDestroy(); catalogDestroy();
} }
TEST(apiTest, catalogRefreshDBVgInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
code = catalogRefreshDBVgInfo(pCtg, mockPointer, ctgTestDbname);
ASSERT_EQ(code, 0);
catalogDestroy();
}
TEST(apiTest, catalogChkAuth_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_USERAUTH;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
bool pass = false;
code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass);
ASSERT_EQ(code, 0);
ASSERT_EQ(pass, true);
catalogDestroy();
}
TEST(apiTest, catalogRefreshGetTableCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
STableCfg *pCfg = NULL;
code = catalogRefreshGetTableCfg(pCtg, mockPointer, &n, &pCfg);
ASSERT_EQ(code, 0);
ASSERT_TRUE(NULL != pCfg);
ASSERT_EQ(pCfg->numOfColumns, ctgTestColNum);
tFreeSTableCfgRsp((STableCfgRsp *)pCfg);
taosMemoryFree(pCfg);
catalogDestroy();
}
TEST(apiTest, catalogGetTableIndex_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_TBLINDEX;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
SArray *pRes = NULL;
code = catalogGetTableIndex(pCtg, mockPointer, &n, &pRes);
ASSERT_EQ(code, 0);
ASSERT_TRUE(NULL != pRes);
ASSERT_EQ(taosArrayGetSize(pRes), ctgTestIndexNum);
taosArrayDestroyEx(pRes, tFreeSTableIndexInfo);
catalogDestroy();
}
TEST(apiTest, catalogGetDBCfg_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_DBCFG;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
SDbCfgInfo cfgInfo = {0};
code = catalogGetDBCfg(pCtg, mockPointer, ctgTestDbname, &cfgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(cfgInfo.numOfVgroups, ctgTestVgNum);
catalogDestroy();
}
TEST(apiTest, catalogGetQnodeList_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_QNODELIST;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SArray *qnodeList = taosArrayInit(10, sizeof(SQueryNodeLoad));
code = catalogGetQnodeList(pCtg, mockPointer, qnodeList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize(qnodeList), ctgTestQnodeNum);
for (int32_t i = 0; i < ctgTestQnodeNum; ++i) {
SQueryNodeLoad * pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i);
ASSERT_EQ(pLoad->addr.nodeId, i);
}
catalogDestroy();
}
TEST(apiTest, catalogGetUdfInfo_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_UDF;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SFuncInfo funcInfo = {0};
code = catalogGetUdfInfo(pCtg, mockPointer, "func1", &funcInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(funcInfo.funcType, ctgTestFuncType);
catalogDestroy();
}
TEST(apiTest, catalogGetServerVersion_test) {
struct SCatalog *pCtg = NULL;
SRequestConnInfo connInfo = {0};
SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
ctgTestInitLogFile();
memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
ctgTestRspIdx = 0;
ctgTestRspFunc[0] = CTGT_RSP_SVRVER;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
char* ver = NULL;
code = catalogGetServerVersion(pCtg, mockPointer, &ver);
ASSERT_EQ(code, 0);
ASSERT_TRUE(0 == strcmp(ver, "1.0"));
catalogDestroy();
}
int main(int argc, char **argv) { int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv); testing::InitGoogleTest(&argc, argv);

21
source/libs/executor/inc/executil.h
View File

@ -28,7 +28,7 @@
do { \ do { \
ASSERT((_c) != -1); \ ASSERT((_c) != -1); \
longjmp((_obj), (_c)); \ longjmp((_obj), (_c)); \
} while (0) } while (0);
#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \
do { \ do { \
@ -95,25 +95,6 @@ typedef struct SColMatchInfo {
int32_t matchType; // determinate the source according to col id or slot id int32_t matchType; // determinate the source according to col id or slot id
} SColMatchInfo; } SColMatchInfo;
// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly
// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups
typedef struct STableListInfo {
bool oneTableForEachGroup;
int32_t numOfOuputGroups; // the data block will be generated one by one
int32_t* groupOffset; // keep the offset value for each group in the tableList
SArray* pTableList;
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
uint64_t suid;
} STableListInfo;
void destroyTableList(STableListInfo* pTableList);
int32_t getNumOfOutputGroups(const STableListInfo* pTableList);
bool oneTableForEachGroup(const STableListInfo* pTableList);
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num);
uint64_t getTotalTables(const STableListInfo* pTableList);
struct SqlFunctionCtx; struct SqlFunctionCtx;
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);

9
source/libs/executor/inc/executorimpl.h
View File

@ -176,10 +176,11 @@ typedef struct {
typedef struct SExecTaskInfo { typedef struct SExecTaskInfo {
STaskIdInfo id; STaskIdInfo id;
uint32_t status; uint32_t status;
int32_t code;
STimeWindow window; STimeWindow window;
STaskCostInfo cost; STaskCostInfo cost;
int64_t owner; // if it is in execution int64_t owner; // if it is in execution
int32_t code;
int64_t version; // used for stream to record wal version int64_t version; // used for stream to record wal version
SStreamTaskInfo streamInfo; SStreamTaskInfo streamInfo;
SSchemaInfo schemaInfo; SSchemaInfo schemaInfo;
@ -346,6 +347,8 @@ typedef struct STableMergeScanInfo {
bool hasGroupId; bool hasGroupId;
uint64_t groupId; uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT* SArray* dataReaders; // array of tsdbReaderT*
SArray* queryConds; // array of queryTableDataCond
STsdbReader* pReader;
SReadHandle readHandle; SReadHandle readHandle;
int32_t bufPageSize; int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort uint32_t sortBufSize; // max buffer size for in-memory sort
@ -355,8 +358,6 @@ typedef struct STableMergeScanInfo {
int64_t startTs; // sort start time int64_t startTs; // sort start time
SArray* sortSourceParams; SArray* sortSourceParams;
SLimitInfo limitInfo; SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder; SFileBlockLoadRecorder readRecorder;
int64_t numOfRows; int64_t numOfRows;
SScanInfo scanInfo; SScanInfo scanInfo;
@ -1076,7 +1077,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
bool groupbyTbname(SNodeList* pGroupList); bool groupbyTbname(SNodeList* pGroupList);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey, bool groupSort); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);

44
source/libs/executor/src/cachescanoperator.c
View File

@ -48,10 +48,6 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
int32_t numOfCols = 0; int32_t numOfCols = 0;
code = code =
extractColMatchInfo(pScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); extractColMatchInfo(pScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
removeRedundantTsCol(pScanNode, &pInfo->matchInfo); removeRedundantTsCol(pScanNode, &pInfo->matchInfo);
code = extractCacheScanSlotId(pInfo->matchInfo.pList, pTaskInfo, &pInfo->pSlotIds); code = extractCacheScanSlotId(pInfo->matchInfo.pList, pTaskInfo, &pInfo->pSlotIds);
@ -65,15 +61,11 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
// partition by tbname, todo opt perf // partition by tbname
if (oneTableForEachGroup(pTableList) || (getTotalTables(pTableList) == 1)) { if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) {
pInfo->retrieveType = pInfo->retrieveType =
CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW); CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList,
STableKeyInfo* pList = taosArrayGet(pTableList->pTableList, 0);
size_t num = taosArrayGetSize(pTableList->pTableList);
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader); taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -175,7 +167,16 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
} }
} }
pInfo->pRes->info.groupId = getTableGroupId(pTableList, pInfo->pRes->info.uid); if (pTableList->map != NULL) {
int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t));
if (groupId != NULL) {
pInfo->pRes->info.groupId = *groupId;
}
} else {
ASSERT(taosArrayGetSize(pTableList->pTableList) == 1);
STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0);
pInfo->pRes->info.groupId = pKeyInfo->groupId;
}
pInfo->indexOfBufferedRes += 1; pInfo->indexOfBufferedRes += 1;
return pInfo->pRes; return pInfo->pRes;
@ -184,25 +185,18 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
} else { } else {
size_t totalGroups = getNumOfOutputGroups(pTableList); size_t totalGroups = taosArrayGetSize(pTableList->pGroupList);
while (pInfo->currentGroupIndex < totalGroups) { while (pInfo->currentGroupIndex < totalGroups) {
SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex);
STableKeyInfo* pList = NULL; tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList,
int32_t num = 0;
int32_t code = getTablesOfGroup(pTableList, pInfo->currentGroupIndex, &pList, &num);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader); taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader);
taosArrayClear(pInfo->pUidList); taosArrayClear(pInfo->pUidList);
code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList); int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); longjmp(pTaskInfo->env, code);
} }
pInfo->currentGroupIndex += 1; pInfo->currentGroupIndex += 1;
@ -212,7 +206,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
if (pInfo->pseudoExprSup.numOfExprs > 0) { if (pInfo->pseudoExprSup.numOfExprs > 0) {
SExprSupp* pSup = &pInfo->pseudoExprSup; SExprSupp* pSup = &pInfo->pseudoExprSup;
STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pTableList)[0]; STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0);
pInfo->pRes->info.groupId = pKeyInfo->groupId; pInfo->pRes->info.groupId = pKeyInfo->groupId;
if (taosArrayGetSize(pInfo->pUidList) > 0) { if (taosArrayGetSize(pInfo->pUidList) > 0) {

95
source/libs/executor/src/executil.c
View File

@ -544,7 +544,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
ctx.index = 0; ctx.index = 0;
ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
if (ctx.cInfoList == NULL) { if (ctx.cInfoList == NULL) {
@ -607,7 +606,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
} else { } else {
void* tag = taosHashGet(tags, uid, sizeof(int64_t)); void* tag = taosHashGet(tags, uid, sizeof(int64_t));
ASSERT(tag); ASSERT(tag);
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId; tagVal.cid = pColInfo->info.colId;
const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
@ -638,7 +636,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
} }
} }
} }
pResBlock->info.rows = rows; pResBlock->info.rows = rows;
// int64_t st1 = taosGetTimestampUs(); // int64_t st1 = taosGetTimestampUs();
@ -664,12 +661,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
} }
break; break;
} }
default: default:
code = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
goto end; goto end;
} }
if (nodeType(pNode) == QUERY_NODE_COLUMN) { if (nodeType(pNode) == QUERY_NODE_COLUMN) {
SColumnNode* pSColumnNode = (SColumnNode*)pNode; SColumnNode* pSColumnNode = (SColumnNode*)pNode;
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, pSColumnNode->slotId); SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, pSColumnNode->slotId);
@ -679,12 +674,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
} else { } else {
code = scalarCalculate(pNode, pBlockList, &output); code = scalarCalculate(pNode, pBlockList, &output);
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
releaseColInfoData(output.columnData); releaseColInfoData(output.columnData);
goto end; goto end;
} }
taosArrayPush(groupData, &output.columnData); taosArrayPush(groupData, &output.columnData);
} }
@ -703,7 +696,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
for (int i = 0; i < rows; i++) { for (int i = 0; i < rows; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
@ -825,86 +817,38 @@ static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) {
taosArrayDestroy(validUid); taosArrayDestroy(validUid);
return 0; return 0;
} }
static int32_t nameComparFn(const void* p1, const void* p2) {
const char* pName1 = *(const char**) p1;
const char* pName2 = *(const char**) p2;
int32_t ret = strcmp(pName1, pName2);
if (ret == 0) {
return 0;
} else {
return (ret > 0)? 1:-1;
}
}
static SArray* getTableNameList(const SNodeListNode* pList) {
int32_t len = LIST_LENGTH(pList->pNodeList);
SListCell* cell = pList->pNodeList->pHead;
SArray* pTbList = taosArrayInit(len, POINTER_BYTES);
for (int i = 0; i < pList->pNodeList->length; i++) {
SValueNode* valueNode = (SValueNode*) cell->pNode;
if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) {
terrno = TSDB_CODE_INVALID_PARA;
taosArrayDestroy(pTbList);
return NULL;
}
char* name = varDataVal(valueNode->datum.p);
taosArrayPush(pTbList, &name);
cell = cell->pNext;
}
size_t numOfTables = taosArrayGetSize(pTbList);
// order the name
taosArraySort(pTbList, nameComparFn);
// remove the duplicates
SArray* pNewList = taosArrayInit(taosArrayGetSize(pTbList), sizeof(void*));
taosArrayPush(pNewList, taosArrayGet(pTbList, 0));
for (int32_t i = 1; i < numOfTables; ++i) {
char** name = taosArrayGetLast(pNewList);
char** nameInOldList = taosArrayGet(pTbList, i);
if (strcmp(*name, *nameInOldList) == 0) {
continue;
}
taosArrayPush(pNewList, nameInOldList);
}
taosArrayDestroy(pTbList);
return pNewList;
}
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) { static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
return -1; return -1;
} }
SOperatorNode* pNode = (SOperatorNode*)pTagCond; SOperatorNode* pNode = (SOperatorNode*)pTagCond;
if (pNode->opType != OP_TYPE_IN) { if (pNode->opType != OP_TYPE_IN) {
return -1; return -1;
} }
if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_COLUMN && if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_COLUMN &&
((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME) && ((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME) &&
(pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) { (pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) {
SNodeListNode* pList = (SNodeListNode*)pNode->pRight; SNodeListNode* pList = (SNodeListNode*)pNode->pRight;
int32_t len = LIST_LENGTH(pList->pNodeList); int32_t len = LIST_LENGTH(pList->pNodeList);
if (len <= 0) { if (len <= 0) return -1;
SListCell* cell = pList->pNodeList->pHead;
SArray* pTbList = taosArrayInit(len, sizeof(void*));
for (int i = 0; i < pList->pNodeList->length; i++) {
SValueNode* valueNode = (SValueNode*)cell->pNode;
if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) {
taosArrayDestroy(pTbList);
return -1; return -1;
} }
char* name = varDataVal(valueNode->datum.p);
taosArrayPush(pTbList, &name);
cell = cell->pNext;
}
SArray* pTbList = getTableNameList(pList); for (int i = 0; i < taosArrayGetSize(pTbList); i++) {
int32_t numOfTables = taosArrayGetSize(pTbList);
for (int i = 0; i < numOfTables; i++) {
char* name = taosArrayGetP(pTbList, i); char* name = taosArrayGetP(pTbList, i);
uint64_t uid = 0; uint64_t uid = 0;
if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
ETableType tbType = TSDB_TABLE_MAX; ETableType tbType = TSDB_TABLE_MAX;
@ -919,14 +863,11 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
terrno = 0; terrno = 0;
} }
} }
taosArrayDestroy(pTbList); taosArrayDestroy(pTbList);
return 0; return 0;
} }
return -1; return -1;
} }
int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
STableListInfo* pListInfo) { STableListInfo* pListInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1005,6 +946,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
} }
taosArrayDestroy(res); taosArrayDestroy(res);
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
if (pListInfo->pGroupList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// put into list as default group, remove it if grouping sorting is required later
taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
return code; return code;
} }

44
source/libs/executor/src/executor.c
View File

@ -293,7 +293,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
} }
if (pListInfo->map == NULL) {
pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
}
// traverse to the stream scanner node to add this table id // traverse to the stream scanner node to add this table id
SOperatorInfo* pInfo = pTaskInfo->pRoot; SOperatorInfo* pInfo = pTaskInfo->pRoot;
@ -305,10 +307,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
SStreamScanInfo* pScanInfo = pInfo->info; SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id if (isAdd) { // add new table id
SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
int32_t numOfQualifiedTables = taosArrayGetSize(qa);
qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables);
qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(qa); taosArrayDestroy(qa);
@ -328,9 +328,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
} }
} }
STableListInfo* pTableListInfo = &pTaskInfo->tableqinfoList; for (int32_t i = 0; i < taosArrayGetSize(qa); ++i) {
for (int32_t i = 0; i < numOfQualifiedTables; ++i) {
uint64_t* uid = taosArrayGet(qa, i); uint64_t* uid = taosArrayGet(qa, i);
STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0};
@ -360,7 +358,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
if (!exists) { if (!exists) {
#endif #endif
addTableIntoTableList(pTableListInfo, keyInfo.uid, keyInfo.groupId); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo);
taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
} }
if (keyBuf != NULL) { if (keyBuf != NULL) {
@ -936,7 +935,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
/*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
/*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
int32_t numOfTables = getTotalTables(&pTaskInfo->tableqinfoList); int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
#ifndef NDEBUG #ifndef NDEBUG
qDebug("switch to next table %" PRId64 " (cursor %d), %" PRId64 " rows returned", uid, qDebug("switch to next table %" PRId64 " (cursor %d), %" PRId64 " rows returned", uid,
@ -945,7 +944,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
#endif #endif
bool found = false; bool found = false;
for (int32_t i = 0; i < numOfTables; i++) { for (int32_t i = 0; i < tableSz; i++) {
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
if (pTableInfo->uid == uid) { if (pTableInfo->uid == uid) {
found = true; found = true;
@ -958,17 +957,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
ASSERT(found); ASSERT(found);
if (pTableScanInfo->dataReader == NULL) { if (pTableScanInfo->dataReader == NULL) {
STableKeyInfo* pList = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond,
int32_t num = getTotalTables(&pTaskInfo->tableqinfoList); pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 ||
pTableScanInfo->dataReader == NULL) {
if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num,
&pTableScanInfo->dataReader, NULL) < 0 || pTableScanInfo->dataReader == NULL) {
ASSERT(0); ASSERT(0);
} }
} }
STableKeyInfo tki = {.uid = uid}; tsdbSetTableId(pTableScanInfo->dataReader, uid);
tsdbSetTableList(pTableScanInfo->dataReader, &tki, 1);
int64_t oldSkey = pTableScanInfo->cond.twindows.skey; int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
pTableScanInfo->cond.twindows.skey = ts + 1; pTableScanInfo->cond.twindows.skey = ts + 1;
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
@ -976,7 +972,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
ts, pTableScanInfo->currentTable, numOfTables); ts, pTableScanInfo->currentTable, tableSz);
/*}*/ /*}*/
} else { } else {
ASSERT(0); ASSERT(0);
@ -998,15 +994,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
STableListInfo* pListInfo = &pTaskInfo->tableqinfoList; taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList,
pListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(pListInfo->pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
STableKeyInfo* pList = taosArrayGet(pListInfo->pTableList, 0);
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, taosArrayGetSize(pListInfo->pTableList),
&pInfo->dataReader, NULL); &pInfo->dataReader, NULL);
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);

164
source/libs/executor/src/executorimpl.c
View File

@ -1739,6 +1739,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void doDestroyTableList(STableListInfo* pTableqinfoList);
typedef struct SFetchRspHandleWrapper { typedef struct SFetchRspHandleWrapper {
uint32_t exchangeId; uint32_t exchangeId;
int32_t sourceIndex; int32_t sourceIndex;
@ -3364,116 +3366,62 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) {
static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); } static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); }
static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) {
STableKeyInfo* pInfo1 = (STableKeyInfo*) p1;
STableKeyInfo* pInfo2 = (STableKeyInfo*) p2;
if (pInfo1->groupId == pInfo2->groupId) {
return 0;
} else {
return pInfo1->groupId < pInfo2->groupId? -1:1;
}
}
static int32_t sortTableGroup(STableListInfo* pTableListInfo) { static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
int32_t code = TSDB_CODE_SUCCESS; taosArrayClear(pTableListInfo->pGroupList);
taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn);
int32_t size = getTotalTables(pTableListInfo);
SArray* pList = taosArrayInit(4, sizeof(int32_t));
STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0);
uint64_t gid = pInfo->groupId;
int32_t start = 0;
taosArrayPush(pList, &start);
for(int32_t i = 1; i < size; ++i) {
pInfo = taosArrayGet(pTableListInfo->pTableList, i);
if (pInfo->groupId != gid) {
taosArrayPush(pList, &i);
gid = pInfo->groupId;
}
}
pTableListInfo->numOfOuputGroups = taosArrayGetSize(pList);
pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
taosArrayDestroy(pList);
# if 0
SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t)); SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t));
if (sortSupport == NULL) { if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
}
size_t num = taosArrayGetSize(pTableListInfo->pTableList);
for (int32_t i = 0; i < num; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ); int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
if (index == -1) { if (index == -1) {
void* p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); void* p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
if (tGroup == NULL) { if (tGroup == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (taosArrayPush(tGroup, info) == NULL) { if (taosArrayPush(tGroup, info) == NULL) {
qError("taos push info array error"); qError("taos push info array error");
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_QRY_APP_ERROR;
} }
if (p == NULL) { if (p == NULL) {
if (taosArrayPush(sortSupport, groupId) == NULL) { if (taosArrayPush(sortSupport, groupId) == NULL) {
qError("taos push support array error"); qError("taos push support array error");
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_QRY_APP_ERROR;
} }
if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) == NULL) { if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) == NULL) {
qError("taos push group array error"); qError("taos push group array error");
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_QRY_APP_ERROR;
} }
} else { } else {
int32_t pos = TARRAY_ELEM_IDX(sortSupport, p); int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
if (taosArrayInsert(sortSupport, pos, groupId) == NULL) { if (taosArrayInsert(sortSupport, pos, groupId) == NULL) {
qError("taos insert support array error"); qError("taos insert support array error");
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_QRY_APP_ERROR;
} }
if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) { if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) {
qError("taos insert group array error"); qError("taos insert group array error");
code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(sortSupport);
goto _error; return TSDB_CODE_QRY_APP_ERROR;
} }
} }
} else { } else {
SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index); SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
if (taosArrayPush(tGroup, info) == NULL) { if (taosArrayPush(tGroup, info) == NULL) {
qError("taos push uid array error"); qError("taos push uid array error");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
}
}
taosArrayDestroy(sortSupport); taosArrayDestroy(sortSupport);
#endif return TSDB_CODE_QRY_APP_ERROR;
}
}
}
taosArrayDestroy(sortSupport);
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
_error:
// taosArrayDestroy(sortSupport);
return code;
} }
bool groupbyTbname(SNodeList* pGroupList) { bool groupbyTbname(SNodeList* pGroupList) {
@ -3489,44 +3437,38 @@ bool groupbyTbname(SNodeList* pGroupList) {
return bytbname; return bytbname;
} }
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) { int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) {
int32_t code = TSDB_CODE_SUCCESS;
if (group == NULL) { if (group == NULL) {
return code; return TDB_CODE_SUCCESS;
} }
pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (pTableListInfo->map == NULL) { if (pTableListInfo->map == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return code;
} }
bool assignUid = groupbyTbname(group); bool assignUid = groupbyTbname(group);
size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
if (assignUid) { // in case of group/partition by tbname, the group id is equalled to the uid of table if (assignUid) {
for (int32_t i = 0; i < numOfTables; i++) { for (int32_t i = 0; i < numOfTables; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
info->groupId = info->uid; info->groupId = info->uid;
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t));
} }
pTableListInfo->oneTableForEachGroup = true;
if (groupSort) {
pTableListInfo->numOfOuputGroups = numOfTables;
}
} else { } else {
code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo); int32_t code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
if (groupSort) {
code = sortTableGroup(pTableListInfo);
}
} }
return code; if (pTableListInfo->needSortTableByGroupId) {
return sortTableGroup(pTableListInfo);
}
return TDB_CODE_SUCCESS;
} }
static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) {
@ -3563,12 +3505,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
// NOTE: this is an patch to fix the physical plan
// TODO remove it later
if (pTableScanNode->scan.node.pLimit != NULL) {
pTableScanNode->groupSort = true;
}
int32_t code = int32_t code =
createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
@ -3627,10 +3563,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t sz = taosArrayGetSize(pTableListInfo->pTableList); int32_t sz = taosArrayGetSize(pTableListInfo->pTableList);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
STableKeyInfo* pKeyInfo = taosArrayGet(pTableListInfo->pTableList, i); STableKeyInfo* pKeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
qDebug("creating stream task: add table uid:%" PRIu64, pKeyInfo->uid); qDebug("creating stream task: add table %" PRId64, pKeyInfo->uid);
} }
qDebug("table in hashmap, %d", (int32_t) getTotalTables(pTableListInfo));
#endif #endif
} }
@ -3665,20 +3599,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} }
SQueryTableDataCond cond = {0}; SQueryTableDataCond cond = {0};
int32_t code = initTableblockDistQueryCond(pBlockNode->suid, &cond); int32_t code = initTableblockDistQueryCond(pBlockNode->suid, &cond);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
} }
size_t num = getTotalTables(pTableListInfo);
void* pList = NULL;
if (num > 0) {
pList = taosArrayGet(pTableListInfo->pTableList, 0);
}
STsdbReader* pReader = NULL; STsdbReader* pReader = NULL;
tsdbReaderOpen(pHandle->vnode, &cond, pList, num, &pReader, ""); tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, "");
cleanupQueryTableDataCond(&cond); cleanupQueryTableDataCond(&cond);
pOperator = createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo); pOperator = createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo);
@ -3713,7 +3640,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} }
size_t size = LIST_LENGTH(pPhyNode->pChildren); size_t size = LIST_LENGTH(pPhyNode->pChildren);
SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
@ -4049,10 +3975,28 @@ _complete:
return code; return code;
} }
void doDestroyTableList(STableListInfo* pTableqinfoList) {
taosArrayDestroy(pTableqinfoList->pTableList);
taosHashCleanup(pTableqinfoList->map);
if (pTableqinfoList->needSortTableByGroupId) {
for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) {
SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
if (tmp == pTableqinfoList->pTableList) {
continue;
}
taosArrayDestroy(tmp);
}
}
taosArrayDestroy(pTableqinfoList->pGroupList);
pTableqinfoList->pTableList = NULL;
pTableqinfoList->map = NULL;
}
void doDestroyTask(SExecTaskInfo* pTaskInfo) { void doDestroyTask(SExecTaskInfo* pTaskInfo) {
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
destroyTableList(&pTaskInfo->tableqinfoList); doDestroyTableList(&pTaskInfo->tableqinfoList);
destroyOperatorInfo(pTaskInfo->pRoot); destroyOperatorInfo(pTaskInfo->pRoot);
cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); cleanupTableSchemaInfo(&pTaskInfo->schemaInfo);
cleanupStreamInfo(&pTaskInfo->streamInfo); cleanupStreamInfo(&pTaskInfo->streamInfo);

1
source/libs/executor/src/projectoperator.c
View File

@ -159,7 +159,6 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S
// reset the value for a new group data // reset the value for a new group data
// existing rows that belongs to previous group. // existing rows that belongs to previous group.
// TODO refactor with doTableScan
pLimitInfo->numOfOutputRows = 0; pLimitInfo->numOfOutputRows = 0;
pLimitInfo->remainOffset = pLimitInfo->limit.offset; pLimitInfo->remainOffset = pLimitInfo->limit.offset;
} }

576
source/libs/executor/src/scanoperator.c
View File

@ -42,7 +42,7 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S
static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns",
"ttl", "stable_name", "vgroup_id', 'uid", "type"}; "ttl", "stable_name", "vgroup_id', 'uid", "type"};
static char* SYSTABLE_IDX_EXCEPT[] = {"db_name", "vgroup_id"}; static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"};
typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
typedef int32_t (*__sys_check)(SNode* cond); typedef int32_t (*__sys_check)(SNode* cond);
@ -363,7 +363,8 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
if (pLimitInfo->remainOffset >= pBlock->info.rows) { if (pLimitInfo->remainOffset >= pBlock->info.rows) {
pLimitInfo->remainOffset -= pBlock->info.rows; pLimitInfo->remainOffset -= pBlock->info.rows;
pBlock->info.rows = 0; pBlock->info.rows = 0;
qDebug("current block ignore due to offset, current:%"PRId64", %s", pLimitInfo->remainOffset, GET_TASKID(pTaskInfo)); qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset,
GET_TASKID(pTaskInfo));
} else { } else {
blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
pLimitInfo->remainOffset = 0; pLimitInfo->remainOffset = 0;
@ -377,6 +378,8 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
blockDataKeepFirstNRows(pBlock, keep); blockDataKeepFirstNRows(pBlock, keep);
qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo)); qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo));
// setTaskStatus(pTaskInfo, TASK_COMPLETED);
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
} }
} }
@ -677,7 +680,10 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN; pTableScanInfo->scanFlag = REPEAT_SCAN;
qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); qDebug(
"%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks "
"due to query func required",
GET_TASKID(pTaskInfo));
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
@ -704,7 +710,8 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN; pTableScanInfo->scanFlag = REPEAT_SCAN;
qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); qDebug("%s start to repeat descending order scan data blocks due to query func required",
GET_TASKID(pTaskInfo));
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
} }
} }
@ -717,7 +724,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info; STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// scan table one by one sequentially // if scan table by table
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
@ -734,65 +741,58 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
} }
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable); STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
tsdbSetTableList(pInfo->dataReader, pTableInfo, 1); tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables,
pInfo->currentTable, pTaskInfo->id.str); pInfo->currentTable, pTaskInfo->id.str);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond); tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
} }
} else { // scan table group by group sequentially }
if (pInfo->currentGroupId == -1) { if (pInfo->currentGroupId == -1) {
if ((++pInfo->currentGroupId) >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)) { pInfo->currentGroupId++;
doSetOperatorCompleted(pOperator); if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL; return NULL;
} }
int32_t num = 0;
STableKeyInfo* pList = NULL;
getTablesOfGroup(&pTaskInfo->tableqinfoList, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->dataReader == NULL);
int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, (STsdbReader**)&pInfo->dataReader,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
}
SSDataBlock* result = doTableScanGroup(pOperator); SSDataBlock* result = doTableScanGroup(pOperator);
if (result != NULL) { if (result != NULL) {
ASSERT(result->info.uid != 0); ASSERT(result->info.uid != 0);
return result; return result;
} }
if ((++pInfo->currentGroupId) >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)) {
doSetOperatorCompleted(pOperator); int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
return NULL;
}
}
SSDataBlock* result = doTableScanGroup(pOperator);
if (result) {
return result;
}
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL; return NULL;
} }
// reset value for the next group data output
pOperator->status = OP_OPENED;
pInfo->limitInfo.numOfOutputRows = 0;
pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset;
int32_t num = 0;
STableKeyInfo* pList = NULL;
getTablesOfGroup(&pTaskInfo->tableqinfoList, pInfo->currentGroupId, &pList, &num);
tsdbSetTableList(pInfo->dataReader, pList, num);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond); tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
result = doTableScanGroup(pOperator); result = doTableScanGroup(pOperator);
if (result != NULL) { if (result) {
return result; return result;
} }
doSetOperatorCompleted(pOperator); setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL; return NULL;
} }
}
static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder)); SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder));
@ -837,6 +837,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
} }
initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -1076,55 +1077,39 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
pTableScanInfo->cond.twindows = *pWin; pTableScanInfo->cond.twindows = *pWin;
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1; pTableScanInfo->currentGroupId = -1;
tsdbReaderClose(pTableScanInfo->dataReader); }
pTableScanInfo->dataReader = NULL;
static void freeArray(void* array) { taosArrayDestroy(array); }
static void resetTableScanOperator(SOperatorInfo* pTableScanOp) {
STableScanInfo* pTableScanInfo = pTableScanOp->info;
pTableScanInfo->cond.startVersion = -1;
pTableScanInfo->cond.endVersion = -1;
SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList;
taosArrayClearP(gpTbls, freeArray);
taosArrayPush(gpTbls, &allTbls);
STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
resetTableScanInfo(pTableScanOp->info, &win);
} }
static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
int64_t maxVersion) { int64_t maxVersion) {
SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
taosArrayClear(gpTbls);
STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(tbls, &tblInfo);
taosArrayPush(gpTbls, &tbls);
STimeWindow win = {.skey = startTs, .ekey = endTs};
STableScanInfo* pTableScanInfo = pTableScanOp->info; STableScanInfo* pTableScanInfo = pTableScanOp->info;
SQueryTableDataCond cond = pTableScanInfo->cond; pTableScanInfo->cond.startVersion = -1;
pTableScanInfo->cond.endVersion = maxVersion;
cond.startVersion = -1; resetTableScanInfo(pTableScanOp->info, &win);
cond.endVersion = maxVersion; SSDataBlock* pRes = doTableScan(pTableScanOp);
cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; resetTableScanOperator(pTableScanOp);
return pRes;
SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
blockDataCleanup(pBlock);
STsdbReader* pReader = NULL;
int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
bool hasBlock = tsdbNextDataBlock(pReader);
if (hasBlock) {
SDataBlockInfo binfo = {0};
tsdbRetrieveDataBlockInfo(pReader, &binfo);
SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
blockDataEnsureCapacity(pBlock, binfo.rows);
pBlock->info.window = binfo.window;
pBlock->info.uid = binfo.uid;
pBlock->info.rows = binfo.rows;
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
}
tsdbReaderClose(pReader);
qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
return pBlock->info.rows > 0 ? pBlock : NULL;
} }
static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
@ -1315,6 +1300,9 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
SSessionKey startWin = {0}; SSessionKey startWin = {0};
getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin); getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin);
if (IS_INVALID_SESSION_WIN_KEY(startWin)) { if (IS_INVALID_SESSION_WIN_KEY(startWin)) {
// char* tmp = streamStateSessionDump(pInfo->windowSup.pStreamAggSup->pState);
// qInfo("%s", tmp);
// taosMemoryFree(tmp);
// window has been closed. // window has been closed.
continue; continue;
} }
@ -2347,14 +2335,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pTSInfo->cond.endVersion = pHandle->version; pTSInfo->cond.endVersion = pHandle->version;
} }
STableKeyInfo* pList = NULL; SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
int32_t num = 0;
getTablesOfGroup(&pTaskInfo->tableqinfoList, 0, &pList, &num);
if (pHandle->initTableReader) { if (pHandle->initTableReader) {
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
pTSInfo->dataReader = NULL; pTSInfo->dataReader = NULL;
if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) { if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, tableList, &pTSInfo->dataReader, NULL) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _error; goto _error;
} }
@ -2382,7 +2367,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
// set the extract column id to streamHandle // set the extract column id to streamHandle
tqReaderSetColIdList(pInfo->tqReader, pColIds); tqReaderSetColIdList(pInfo->tqReader, pColIds);
SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
if (code != 0) { if (code != 0) {
taosArrayDestroy(tableIdList); taosArrayDestroy(tableIdList);
goto _error; goto _error;
@ -3267,18 +3252,80 @@ static int tableUidCompare(const void* a, const void* b) {
} }
return u1 < u2 ? -1 : 1; return u1 < u2 ? -1 : 1;
} }
typedef struct MergeIndex {
int idx;
int len;
} MergeIndex;
static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) {
uint64_t v;
int32_t m;
while (s <= e) {
m = s + (e - s) / 2;
v = *(uint64_t*)taosArrayGet(arr, m);
if (v >= k) {
e = m - 1;
} else {
s = m + 1;
}
}
return s;
}
void optSysIntersection(SArray* in, SArray* out) {
int32_t sz = (int32_t)taosArrayGetSize(in);
if (sz <= 0) {
return;
}
MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray* t = taosArrayGetP(in, i);
mi[i].len = (int32_t)taosArrayGetSize(t);
mi[i].idx = 0;
}
SArray* base = taosArrayGetP(in, 0);
for (int i = 0; i < taosArrayGetSize(base); i++) {
uint64_t tgt = *(uint64_t*)taosArrayGet(base, i);
bool has = true;
for (int j = 1; j < taosArrayGetSize(in); j++) {
SArray* oth = taosArrayGetP(in, j);
int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
if (mid >= 0 && mid < mi[j].len) {
uint64_t val = *(uint64_t*)taosArrayGet(oth, mid);
has = (val == tgt ? true : false);
mi[j].idx = mid;
} else {
has = false;
}
}
if (has == true) {
taosArrayPush(out, &tgt);
}
}
taosMemoryFreeClear(mi);
}
static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) {
// TODO, find comm mem from mRslt // TODO, find comm mem from mRslt
for (int i = 0; i < taosArrayGetSize(mRslt); i++) { for (int i = 0; i < taosArrayGetSize(mRslt); i++) {
SArray* aRslt = taosArrayGetP(mRslt, i); SArray* arslt = taosArrayGetP(mRslt, i);
taosArrayAddAll(rslt, aRslt); taosArraySort(arslt, tableUidCompare);
}
optSysIntersection(mRslt, rslt);
return 0;
}
static int32_t optSysSpecialColumn(SNode* cond) {
SOperatorNode* pOper = (SOperatorNode*)cond;
SColumnNode* pCol = (SColumnNode*)pOper->pLeft;
for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) {
if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) {
return 1;
}
} }
taosArraySort(rslt, tableUidCompare);
taosArrayRemoveDuplicate(rslt, tableUidCompare, NULL);
return 0; return 0;
} }
static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
int ret = -1; int ret = -1;
if (nodeType(cond) == QUERY_NODE_OPERATOR) { if (nodeType(cond) == QUERY_NODE_OPERATOR) {
@ -3302,7 +3349,6 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
SNodeList* pList = (SNodeList*)pNode->pParameterList; SNodeList* pList = (SNodeList*)pNode->pParameterList;
int32_t len = LIST_LENGTH(pList); int32_t len = LIST_LENGTH(pList);
if (len <= 0) return ret;
bool hasIdx = false; bool hasIdx = false;
bool hasRslt = true; bool hasRslt = true;
@ -3313,12 +3359,16 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
if (cell == NULL) break; if (cell == NULL) break;
SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); SArray* aRslt = taosArrayInit(16, sizeof(int64_t));
ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); ret = optSysTabFilteImpl(arg, cell->pNode, aRslt);
if (ret == 0) { if (ret == 0) {
// has index // has index
hasIdx = true; hasIdx = true;
if (optSysSpecialColumn(cell->pNode) == 0) {
taosArrayPush(mRslt, &aRslt); taosArrayPush(mRslt, &aRslt);
} else {
// db_name/vgroup not result
taosArrayDestroy(aRslt);
}
} else if (ret == -2) { } else if (ret == -2) {
// current vg // current vg
hasIdx = true; hasIdx = true;
@ -4103,9 +4153,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
int32_t code = int32_t code =
extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -4163,8 +4210,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pTableListInfo->numOfOuputGroups = 1; pTableListInfo->needSortTableByGroupId = groupSort;
code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags, groupSort); code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -4178,15 +4225,144 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, const char* idstr) { int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, const char* idstr) {
for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
STableKeyInfo* pList = taosArrayGet(pTableListInfo->pTableList, i); SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
STsdbReader* pReader = NULL; STsdbReader* pReader = NULL;
tsdbReaderOpen(pHandle->vnode, pQueryCond, pList, 1, &pReader, idstr); tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, &pReader, idstr);
taosArrayPush(arrayReader, &pReader); taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(subTableList);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
STsdbReader** ppReader, const char* idstr) {
STsdbReader* pReader = NULL;
SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
}
int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, &pReader, idstr);
if (code != 0) {
taosArrayDestroy(subTableList);
return code;
}
*ppReader = pReader;
return TSDB_CODE_SUCCESS;
}
static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
SSDataBlock* pBlock, uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STableMergeScanInfo* pInfo = pOperator->info;
uint64_t uid = pBlock->info.uid;
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
pCost->totalBlocks += 1;
pCost->totalRows += pBlock->info.rows;
*status = pInfo->dataBlockLoadFlag;
if (pTableScanInfo->pFilterNode != NULL ||
overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
(*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
}
SDataBlockInfo* pBlockInfo = &pBlock->info;
taosMemoryFreeClear(pBlock->pBlockAgg);
if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL;
}
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
pCost->loadBlockStatis += 1;
bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL;
STsdbReader* reader = pTableScanInfo->pReader;
tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) {
pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
if (!pColMatchInfo->needOutput) {
continue;
}
pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
}
return TSDB_CODE_SUCCESS;
} else { // failed to load the block sma data, data block statistics does not exist, load data block instead
*status = FUNC_DATA_REQUIRED_DATA_LOAD;
}
}
ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
STsdbReader* reader = pTableScanInfo->pReader;
SArray* pCols = tsdbRetrieveDataBlock(reader, NULL);
if (pCols == NULL) {
return terrno;
}
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
// currently only the tbname pseudo column
if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampMs();
doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
if (pBlock->info.rows == 0) {
pCost->filterOutBlocks += 1;
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
} else {
qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
}
}
return TSDB_CODE_SUCCESS;
}
// todo refactor // todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@ -4270,7 +4446,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); STsdbReader* reader = pTableScanInfo->pReader; // taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
SArray* pCols = tsdbRetrieveDataBlock(reader, NULL); SArray* pCols = tsdbRetrieveDataBlock(reader, NULL);
if (pCols == NULL) { if (pCols == NULL) {
return terrno; return terrno;
@ -4309,9 +4485,143 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
typedef struct STableMergeScanSortSourceParam { typedef struct STableMergeScanSortSourceParam {
SOperatorInfo* pOperator; SOperatorInfo* pOperator;
int32_t readerIdx; int32_t readerIdx;
int64_t uid;
SSDataBlock* inputBlock; SSDataBlock* inputBlock;
} STableMergeScanSortSourceParam; } STableMergeScanSortSourceParam;
static SSDataBlock* getTableDataBlockTemp(void* param) {
STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator;
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t readIdx = source->readerIdx;
SSDataBlock* pBlock = source->inputBlock;
STableMergeScanInfo* pTableScanInfo = pOperator->info;
SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
blockDataCleanup(pBlock);
int64_t st = taosGetTimestampUs();
SArray* subTable = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subTable, taosArrayGet(pInfo->tableListInfo->pTableList, readIdx + pInfo->tableStartIndex));
SReadHandle* pHandle = &pInfo->readHandle;
tsdbReaderOpen(pHandle->vnode, pQueryCond, subTable, &pInfo->pReader, GET_TASKID(pTaskInfo));
taosArrayDestroy(subTable);
STsdbReader* reader = pInfo->pReader;
while (tsdbNextDataBlock(reader)) {
if (isTaskKilled(pOperator->pTaskInfo)) {
T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
// process this data block based on the probabilities
bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
if (!processThisBlock) {
continue;
}
blockDataCleanup(pBlock);
SDataBlockInfo binfo = pBlock->info;
tsdbRetrieveDataBlockInfo(reader, &binfo);
blockDataEnsureCapacity(pBlock, binfo.rows);
pBlock->info.type = binfo.type;
pBlock->info.uid = binfo.uid;
pBlock->info.window = binfo.window;
pBlock->info.rows = binfo.rows;
if (tsdbIsAscendingOrder(pInfo->pReader)) {
pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
} else {
pQueryCond->twindows.ekey = pBlock->info.window.skey - 1;
}
uint32_t status = 0;
int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readIdx, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pOperator->pTaskInfo->env, code);
}
// current block is filter out according to filter condition, continue load the next block
if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
continue;
}
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
if (groupId) {
pBlock->info.groupId = *groupId;
}
pOperator->resultInfo.totalRows += pBlock->info.rows; // pTableScanInfo->readRecorder.totalRows;
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
tsdbReaderClose(pInfo->pReader);
pInfo->pReader = NULL;
return pBlock;
}
tsdbReaderClose(pInfo->pReader);
pInfo->pReader = NULL;
return NULL;
}
static SSDataBlock* getTableDataBlock2(void* param) {
STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator;
int64_t uid = source->uid;
SSDataBlock* pBlock = source->inputBlock;
STableMergeScanInfo* pTableScanInfo = pOperator->info;
int64_t st = taosGetTimestampUs();
blockDataCleanup(pBlock);
STsdbReader* reader = pTableScanInfo->pReader;
while (tsdbTableNextDataBlock(reader, uid)) {
if (isTaskKilled(pOperator->pTaskInfo)) {
T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
// process this data block based on the probabilities
bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
if (!processThisBlock) {
continue;
}
blockDataCleanup(pBlock);
SDataBlockInfo binfo = pBlock->info;
tsdbRetrieveDataBlockInfo(reader, &binfo);
blockDataEnsureCapacity(pBlock, binfo.rows);
pBlock->info.type = binfo.type;
pBlock->info.uid = binfo.uid;
pBlock->info.window = binfo.window;
pBlock->info.rows = binfo.rows;
uint32_t status = 0;
int32_t code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pOperator->pTaskInfo->env, code);
}
// current block is filter out according to filter condition, continue load the next block
if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
continue;
}
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
if (groupId) {
pBlock->info.groupId = *groupId;
}
pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
return pBlock;
}
return NULL;
}
static SSDataBlock* getTableDataBlock(void* param) { static SSDataBlock* getTableDataBlock(void* param) {
STableMergeScanSortSourceParam* source = param; STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator; SOperatorInfo* pOperator = source->pOperator;
@ -4347,7 +4657,6 @@ static SSDataBlock* getTableDataBlock(void* param) {
uint32_t status = 0; uint32_t status = 0;
int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status);
// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pOperator->pTaskInfo->env, code); T_LONG_JMP(pOperator->pTaskInfo->env, code);
} }
@ -4390,6 +4699,14 @@ SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
return pList; return pList;
} }
int32_t dumpSQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
for (int i = 0; i < src->numOfCols; i++) {
dst->colList[i] = src->colList[i];
}
return 0;
}
int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info; STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -4410,10 +4727,9 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t tableEndIdx = pInfo->tableEndIndex; int32_t tableEndIdx = pInfo->tableEndIndex;
STableListInfo* tableListInfo = pInfo->tableListInfo; STableListInfo* tableListInfo = pInfo->tableListInfo;
pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
pInfo->dataReaders, GET_TASKID(pTaskInfo));
// pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
pInfo->pReader = NULL;
// todo the total available buffer should be determined by total capacity of buffer of this task. // todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result // the additional one is reserved for merge result
pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
@ -4421,18 +4737,27 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str); pInfo->pSortInputBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL); tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL);
// one table has one data block
int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond));
for (int32_t i = 0; i < numOfTable; ++i) {
STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i + tableStartIdx);
size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
for (int32_t i = 0; i < numReaders; ++i) {
STableMergeScanSortSourceParam param = {0}; STableMergeScanSortSourceParam param = {0};
param.readerIdx = i; param.readerIdx = i;
param.pOperator = pOperator; param.pOperator = pOperator;
param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
taosArrayPush(pInfo->sortSourceParams, &param); taosArrayPush(pInfo->sortSourceParams, &param);
SQueryTableDataCond cond;
dumpSQueryTableCond(&pInfo->cond, &cond);
taosArrayPush(pInfo->queryConds, &cond);
} }
for (int32_t i = 0; i < numReaders; ++i) { for (int32_t i = 0; i < numOfTable; ++i) {
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
ps->param = param; ps->param = param;
@ -4452,7 +4777,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info; STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
size_t numReaders = taosArrayGetSize(pInfo->dataReaders); int32_t numOfTable = taosArrayGetSize(pInfo->queryConds);
SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
@ -4461,7 +4786,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
for (int32_t i = 0; i < numReaders; ++i) { for (int32_t i = 0; i < numOfTable; ++i) {
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
blockDataDestroy(param->inputBlock); blockDataDestroy(param->inputBlock);
} }
@ -4469,12 +4794,13 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
tsortDestroySortHandle(pInfo->pSortHandle); tsortDestroySortHandle(pInfo->pSortHandle);
for (int32_t i = 0; i < numReaders; ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) {
STsdbReader* reader = taosArrayGetP(pInfo->dataReaders, i); SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i);
tsdbReaderClose(reader); taosMemoryFree(cond->colList);
} }
taosArrayDestroy(pInfo->dataReaders); taosArrayDestroy(pInfo->queryConds);
pInfo->dataReaders = NULL; pInfo->queryConds = NULL;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -4558,13 +4884,23 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
void destroyTableMergeScanOperatorInfo(void* param) { void destroyTableMergeScanOperatorInfo(void* param) {
STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
cleanupQueryTableDataCond(&pTableScanInfo->cond); cleanupQueryTableDataCond(&pTableScanInfo->cond);
int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds);
for (int32_t i = 0; i < numOfTable; i++) {
STableMergeScanSortSourceParam* param = taosArrayGet(pTableScanInfo->sortSourceParams, i);
blockDataDestroy(param->inputBlock);
}
taosArrayDestroy(pTableScanInfo->sortSourceParams); taosArrayDestroy(pTableScanInfo->sortSourceParams);
for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) { tsdbReaderClose(pTableScanInfo->pReader);
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, i); pTableScanInfo->pReader = NULL;
tsdbReaderClose(reader);
for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) {
SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
taosMemoryFree(pCond->colList);
} }
taosArrayDestroy(pTableScanInfo->dataReaders); taosArrayDestroy(pTableScanInfo->queryConds);
if (pTableScanInfo->matchInfo.pList != NULL) { if (pTableScanInfo->matchInfo.pList != NULL) {
taosArrayDestroy(pTableScanInfo->matchInfo.pList); taosArrayDestroy(pTableScanInfo->matchInfo.pList);

11
source/libs/executor/src/sortoperator.c
View File

@ -38,7 +38,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols);
int32_t numOfOutputCols = 0; int32_t numOfOutputCols = 0;
int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); int32_t code =
extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
@ -62,8 +63,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
// there are headers, so pageSize = rowSize + header pInfo->sortBufSize = pInfo->bufPageSize * 16; // there are headers, so pageSize = rowSize + header pInfo->sortBufSize = pInfo->bufPageSize * 16;
// TODO dynamic set the available sort buffer // TODO dynamic set the available sort buffer
pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, pOperator->fpSet =
getExplainExecInfo); createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, getExplainExecInfo);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -653,7 +654,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
} }
pInfo->limitInfo.numOfOutputRows += p->info.rows; pInfo->limitInfo.numOfOutputRows += p->info.rows;
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId; pDataBlock->info.groupId = pInfo->groupId;
@ -735,7 +735,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
int32_t numOfOutputCols = 0; int32_t numOfOutputCols = 0;
code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID,
&pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }

2
source/libs/function/src/builtins.c
View File

@ -1651,7 +1651,7 @@ static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (numOfParams == 0 || numOfParams > 2) { if (numOfParams > 2) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
} }

7
source/libs/function/src/builtinsimpl.c
View File

@ -380,7 +380,7 @@ typedef struct SGroupKeyInfo {
numOfElem += 1; \ numOfElem += 1; \
pStddevRes->count -= 1; \ pStddevRes->count -= 1; \
sumT -= plist[i]; \ sumT -= plist[i]; \
pStddevRes->quadraticISum -= plist[i] * plist[i]; \ pStddevRes->quadraticISum -= (int64_t)(plist[i] * plist[i]); \
} \ } \
} while (0) } while (0)
@ -2526,8 +2526,9 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) {
// might be a race condition here that pHisto can be overwritten or setup function // might be a race condition here that pHisto can be overwritten or setup function
// has not been called, need to relink the buffer pHisto points to. // has not been called, need to relink the buffer pHisto points to.
buildHistogramInfo(pInfo); buildHistogramInfo(pInfo);
qDebug("%s before add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, qDebug("%s before add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p",
numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); __FUNCTION__, numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto,
pInfo->pHisto->elems);
for (int32_t i = start; i < pInput->numOfRows + start; ++i) { for (int32_t i = start; i < pInput->numOfRows + start; ++i) {
if (colDataIsNull_f(pCol->nullbitmap, i)) { if (colDataIsNull_f(pCol->nullbitmap, i)) {
continue; continue;

27
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -111,6 +111,8 @@ const char* nodesNodeName(ENodeType type) {
return "DropSuperTableStmt"; return "DropSuperTableStmt";
case QUERY_NODE_ALTER_TABLE_STMT: case QUERY_NODE_ALTER_TABLE_STMT:
return "AlterTableStmt"; return "AlterTableStmt";
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
return "AlterSuperTableStmt";
case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_CREATE_USER_STMT:
return "CreateUserStmt"; return "CreateUserStmt";
case QUERY_NODE_ALTER_USER_STMT: case QUERY_NODE_ALTER_USER_STMT:
@ -669,7 +671,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections); code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanIgnoreGroupId, pNode->ignoreGroupId); code = tjsonAddBoolToObject(pJson, jkProjectLogicPlanIgnoreGroupId, pNode->ignoreGroupId);
} }
return code; return code;
@ -2632,6 +2634,7 @@ static const char* jkSubplanRootNode = "RootNode";
static const char* jkSubplanDataSink = "DataSink"; static const char* jkSubplanDataSink = "DataSink";
static const char* jkSubplanTagCond = "TagCond"; static const char* jkSubplanTagCond = "TagCond";
static const char* jkSubplanTagIndexCond = "TagIndexCond"; static const char* jkSubplanTagIndexCond = "TagIndexCond";
static const char* jkSubplanShowRewrite = "ShowRewrite";
static int32_t subplanToJson(const void* pObj, SJson* pJson) { static int32_t subplanToJson(const void* pObj, SJson* pJson) {
const SSubplan* pNode = (const SSubplan*)pObj; const SSubplan* pNode = (const SSubplan*)pObj;
@ -2667,6 +2670,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond); code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite);
}
return code; return code;
} }
@ -2705,6 +2711,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond); code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite);
}
return code; return code;
} }
@ -2758,6 +2767,20 @@ static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToLogicAggNode(const SJson* pJson, void* pObj) {
SAggLogicNode* pNode = (SAggLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkAggLogicPlanGroupKeys, &pNode->pGroupKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkAggLogicPlanAggFuncs, &pNode->pAggFuncs);
}
return code;
}
static const char* jkDataTypeType = "Type"; static const char* jkDataTypeType = "Type";
static const char* jkDataTypePrecision = "Precision"; static const char* jkDataTypePrecision = "Precision";
static const char* jkDataTypeScale = "Scale"; static const char* jkDataTypeScale = "Scale";
@ -4735,6 +4758,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDeleteStmt(pJson, pObj); return jsonToDeleteStmt(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return jsonToLogicScanNode(pJson, pObj); return jsonToLogicScanNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_AGG:
return jsonToLogicAggNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_PROJECT: case QUERY_NODE_LOGIC_PLAN_PROJECT:
return jsonToLogicProjectNode(pJson, pObj); return jsonToLogicProjectNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:

13
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -1969,7 +1969,12 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) {
return code; return code;
} }
enum { PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT, PHY_LAST_ROW_SCAN_CODE_IGNULL }; enum {
PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
PHY_LAST_ROW_SCAN_CODE_GROUP_SORT,
PHY_LAST_ROW_SCAN_CODE_IGNULL
};
static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@ -3433,6 +3438,9 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = queryNodeAddrInlineToMsg(&pNode->execNode, pEncoder); code = queryNodeAddrInlineToMsg(&pNode->execNode, pEncoder);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->showRewrite);
}
return code; return code;
} }
@ -3479,6 +3487,9 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = msgToQueryNodeAddrInline(pDecoder, &pNode->execNode); code = msgToQueryNodeAddrInline(pDecoder, &pNode->execNode);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->showRewrite);
}
return code; return code;
} }

207
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -438,210 +438,3 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
return; return;
} }
static EDealRes walkPhysiNode(SPhysiNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiPlan((SNode*)pNode->pOutputDataBlockDesc, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pNode->pConditions, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pNode->pChildren, order, walker, pContext);
}
return res;
}
static EDealRes walkScanPhysi(SScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiNode((SPhysiNode*)pScan, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pScan->pScanCols, order, walker, pContext);
}
return res;
}
static EDealRes walkTableScanPhysi(STableScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker,
void* pContext) {
EDealRes res = walkScanPhysi((SScanPhysiNode*)pScan, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pScan->pDynamicScanFuncs, order, walker, pContext);
}
return res;
}
static EDealRes walkWindowPhysi(SWinodwPhysiNode* pWindow, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = walkPhysiNode((SPhysiNode*)pWindow, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pWindow->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pWindow->pFuncs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pWindow->pTspk, order, walker, pContext);
}
return res;
}
static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
EDealRes res = DEAL_RES_CONTINUE;
switch (nodeType(pNode)) {
case QUERY_NODE_NODE_LIST:
res = walkPhysiPlans(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
res = walkTableScanPhysi((STableScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
res = walkTableScanPhysi((STableScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pProject->pProjections, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode* pJoin = (SSortMergeJoinPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pJoin->pMergeCondition, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pJoin->pOnConditions, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pJoin->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
SAggPhysiNode* pAgg = (SAggPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pGroupKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pAgg->pAggFuncs, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pExchange->pSrcEndPoints, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pSort = (SSortPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pSortKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSort->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: {
SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pNode;
res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan(pState->pStateKey, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pExprs, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pPartitionKeys, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pPart->pTargets, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
res = walkPhysiPlan((SNode*)(((SDataSinkNode*)pNode)->pInputDataBlockDesc), order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
res = walkPhysiPlan((SNode*)(((SDataSinkNode*)pNode)->pInputDataBlockDesc), order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_SUBPLAN: {
SSubplan* pSubplan = (SSubplan*)pNode;
res = walkPhysiPlans(pSubplan->pChildren, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan((SNode*)pSubplan->pNode, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan((SNode*)pSubplan->pDataSink, order, walker, pContext);
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN: {
SQueryPlan* pPlan = (SQueryPlan*)pNode;
if (NULL != pPlan->pSubplans) {
// only need to walk the top-level subplans, because they will recurse to all the subplans below
walkPhysiPlan(nodesListGetNode(pPlan->pSubplans, 0), order, walker, pContext);
}
break;
}
default:
res = dispatchExpr(pNode, order, walker, pContext);
break;
}
return res;
}
static EDealRes walkPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext) {
return walkNode(pNode, order, walker, pContext, dispatchPhysiPlan);
}
static EDealRes walkPhysiPlans(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext) {
SNode* node;
FOREACH(node, pNodeList) {
EDealRes res = walkPhysiPlan(node, order, walker, pContext);
if (DEAL_RES_ERROR == res || DEAL_RES_END == res) {
return res;
}
}
return DEAL_RES_CONTINUE;
}
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext) {
(void)walkPhysiPlan(pNode, TRAVERSAL_PREORDER, walker, pContext);
}

1
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -328,6 +328,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_DROP_SUPER_TABLE_STMT: case QUERY_NODE_DROP_SUPER_TABLE_STMT:
return makeNode(type, sizeof(SDropSuperTableStmt)); return makeNode(type, sizeof(SDropSuperTableStmt));
case QUERY_NODE_ALTER_TABLE_STMT: case QUERY_NODE_ALTER_TABLE_STMT:
case QUERY_NODE_ALTER_SUPER_TABLE_STMT:
return makeNode(type, sizeof(SAlterTableStmt)); return makeNode(type, sizeof(SAlterTableStmt));
case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_CREATE_USER_STMT:
return makeNode(type, sizeof(SCreateUserStmt)); return makeNode(type, sizeof(SCreateUserStmt));

9
source/libs/nodes/test/CMakeLists.txt
View File

@ -1,6 +1,7 @@
MESSAGE(STATUS "build nodes unit test") MESSAGE(STATUS "build nodes unit test")
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11 # GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11) SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
@ -15,5 +16,11 @@ TARGET_INCLUDE_DIRECTORIES(
TARGET_LINK_LIBRARIES( TARGET_LINK_LIBRARIES(
nodesTest nodesTest
PUBLIC os util common nodes gtest PUBLIC os util common nodes qcom gtest
) )
add_test(
NAME nodesTest
COMMAND nodesTest
)
ENDIF()

299
source/libs/nodes/test/nodesCloneTest.cpp Normal file
View File

@ -0,0 +1,299 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, AND/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include "nodes.h"
#include "plannodes.h"
#include "querynodes.h"
class NodesCloneTest : public testing::Test {
public:
void registerCheckFunc(const std::function<void(const SNode*, const SNode*)>& func) { checkFunc_ = func; }
void run(const SNode* pSrc) {
std::unique_ptr<SNode, void (*)(SNode*)> pDst(nodesCloneNode(pSrc), nodesDestroyNode);
checkFunc_(pSrc, pDst.get());
}
private:
std::function<void(const SNode*, const SNode*)> checkFunc_;
};
TEST_F(NodesCloneTest, tempTable) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STempTableNode* pSrcNode = (STempTableNode*)pSrc;
STempTableNode* pDstNode = (STempTableNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pSubquery), nodeType(pDstNode->pSubquery));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_TEMP_TABLE));
STempTableNode* pNode = (STempTableNode*)srcNode.get();
pNode->pSubquery = nodesMakeNode(QUERY_NODE_SELECT_STMT);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, joinTable) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SJoinTableNode* pSrcNode = (SJoinTableNode*)pSrc;
SJoinTableNode* pDstNode = (SJoinTableNode*)pDst;
ASSERT_EQ(pSrcNode->joinType, pDstNode->joinType);
ASSERT_EQ(nodeType(pSrcNode->pLeft), nodeType(pDstNode->pLeft));
ASSERT_EQ(nodeType(pSrcNode->pRight), nodeType(pDstNode->pRight));
if (NULL != pSrcNode->pOnCond) {
ASSERT_EQ(nodeType(pSrcNode->pOnCond), nodeType(pDstNode->pOnCond));
}
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_JOIN_TABLE));
SJoinTableNode* pNode = (SJoinTableNode*)srcNode.get();
pNode->joinType = JOIN_TYPE_INNER;
pNode->pLeft = nodesMakeNode(QUERY_NODE_REAL_TABLE);
pNode->pRight = nodesMakeNode(QUERY_NODE_REAL_TABLE);
pNode->pOnCond = nodesMakeNode(QUERY_NODE_OPERATOR);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, stateWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStateWindowNode* pSrcNode = (SStateWindowNode*)pSrc;
SStateWindowNode* pDstNode = (SStateWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pCol), nodeType(pDstNode->pCol));
ASSERT_EQ(nodeType(pSrcNode->pExpr), nodeType(pDstNode->pExpr));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_STATE_WINDOW));
SStateWindowNode* pNode = (SStateWindowNode*)srcNode.get();
pNode->pCol = nodesMakeNode(QUERY_NODE_COLUMN);
pNode->pExpr = nodesMakeNode(QUERY_NODE_OPERATOR);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, sessionWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SSessionWindowNode* pSrcNode = (SSessionWindowNode*)pSrc;
SSessionWindowNode* pDstNode = (SSessionWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pCol), nodeType(pDstNode->pCol));
ASSERT_EQ(nodeType(pSrcNode->pGap), nodeType(pDstNode->pGap));
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_SESSION_WINDOW));
SSessionWindowNode* pNode = (SSessionWindowNode*)srcNode.get();
pNode->pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
pNode->pGap = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, intervalWindow) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SIntervalWindowNode* pSrcNode = (SIntervalWindowNode*)pSrc;
SIntervalWindowNode* pDstNode = (SIntervalWindowNode*)pDst;
ASSERT_EQ(nodeType(pSrcNode->pInterval), nodeType(pDstNode->pInterval));
if (NULL != pSrcNode->pOffset) {
ASSERT_EQ(nodeType(pSrcNode->pOffset), nodeType(pDstNode->pOffset));
}
if (NULL != pSrcNode->pSliding) {
ASSERT_EQ(nodeType(pSrcNode->pSliding), nodeType(pDstNode->pSliding));
}
if (NULL != pSrcNode->pFill) {
ASSERT_EQ(nodeType(pSrcNode->pFill), nodeType(pDstNode->pFill));
}
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW));
SIntervalWindowNode* pNode = (SIntervalWindowNode*)srcNode.get();
pNode->pInterval = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pOffset = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pSliding = nodesMakeNode(QUERY_NODE_VALUE);
pNode->pFill = nodesMakeNode(QUERY_NODE_FILL);
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, fill) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SFillNode* pSrcNode = (SFillNode*)pSrc;
SFillNode* pDstNode = (SFillNode*)pDst;
ASSERT_EQ(pSrcNode->mode, pDstNode->mode);
if (NULL != pSrcNode->pValues) {
ASSERT_EQ(nodeType(pSrcNode->pValues), nodeType(pDstNode->pValues));
}
ASSERT_EQ(nodeType(pSrcNode->pWStartTs), nodeType(pDstNode->pWStartTs));
ASSERT_EQ(pSrcNode->timeRange.skey, pDstNode->timeRange.skey);
ASSERT_EQ(pSrcNode->timeRange.ekey, pDstNode->timeRange.ekey);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_FILL));
SFillNode* pNode = (SFillNode*)srcNode.get();
pNode->mode = FILL_MODE_VALUE;
pNode->pValues = nodesMakeNode(QUERY_NODE_NODE_LIST);
pNode->pWStartTs = nodesMakeNode(QUERY_NODE_COLUMN);
pNode->timeRange.skey = 1666756692907;
pNode->timeRange.ekey = 1666756699907;
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, logicSubplan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SLogicSubplan* pSrcNode = (SLogicSubplan*)pSrc;
SLogicSubplan* pDstNode = (SLogicSubplan*)pDst;
ASSERT_EQ(pSrcNode->subplanType, pDstNode->subplanType);
ASSERT_EQ(pSrcNode->level, pDstNode->level);
ASSERT_EQ(pSrcNode->splitFlag, pDstNode->splitFlag);
ASSERT_EQ(pSrcNode->numOfComputeNodes, pDstNode->numOfComputeNodes);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN));
SLogicSubplan* pNode = (SLogicSubplan*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiScan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
STagScanPhysiNode* pSrcNode = (STagScanPhysiNode*)pSrc;
STagScanPhysiNode* pDstNode = (STagScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->uid, pDstNode->uid);
ASSERT_EQ(pSrcNode->suid, pDstNode->suid);
ASSERT_EQ(pSrcNode->tableType, pDstNode->tableType);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN));
STagScanPhysiNode* pNode = (STagScanPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiSystemTableScan) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SSystemTableScanPhysiNode* pSrcNode = (SSystemTableScanPhysiNode*)pSrc;
SSystemTableScanPhysiNode* pDstNode = (SSystemTableScanPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->showRewrite, pDstNode->showRewrite);
ASSERT_EQ(pSrcNode->accountId, pDstNode->accountId);
ASSERT_EQ(pSrcNode->sysInfo, pDstNode->sysInfo);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN));
SSystemTableScanPhysiNode* pNode = (SSystemTableScanPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamSemiSessionWinodw) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamSemiSessionWinodwPhysiNode* pSrcNode = (SStreamSemiSessionWinodwPhysiNode*)pSrc;
SStreamSemiSessionWinodwPhysiNode* pDstNode = (SStreamSemiSessionWinodwPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->gap, pDstNode->gap);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION));
SStreamSemiSessionWinodwPhysiNode* pNode = (SStreamSemiSessionWinodwPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamFinalSessionWinodw) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamFinalSessionWinodwPhysiNode* pSrcNode = (SStreamFinalSessionWinodwPhysiNode*)pSrc;
SStreamFinalSessionWinodwPhysiNode* pDstNode = (SStreamFinalSessionWinodwPhysiNode*)pDst;
ASSERT_EQ(pSrcNode->gap, pDstNode->gap);
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION));
SStreamFinalSessionWinodwPhysiNode* pNode = (SStreamFinalSessionWinodwPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiStreamPartition) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SStreamPartitionPhysiNode* pSrcNode = (SStreamPartitionPhysiNode*)pSrc;
SStreamPartitionPhysiNode* pDstNode = (SStreamPartitionPhysiNode*)pDst;
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION));
SStreamPartitionPhysiNode* pNode = (SStreamPartitionPhysiNode*)srcNode.get();
return srcNode.get();
}());
}
TEST_F(NodesCloneTest, physiPartition) {
registerCheckFunc([](const SNode* pSrc, const SNode* pDst) {
ASSERT_EQ(nodeType(pSrc), nodeType(pDst));
SPartitionPhysiNode* pSrcNode = (SPartitionPhysiNode*)pSrc;
SPartitionPhysiNode* pDstNode = (SPartitionPhysiNode*)pDst;
});
std::unique_ptr<SNode, void (*)(SNode*)> srcNode(nullptr, nodesDestroyNode);
run([&]() {
srcNode.reset(nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_PARTITION));
SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)srcNode.get();
return srcNode.get();
}());
}

0
source/libs/nodes/test/nodesTest.cpp → source/libs/nodes/test/nodesTestMain.cpp
View File

1
source/libs/parser/inc/sql.y
View File

@ -518,6 +518,7 @@ stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE.
stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; } stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; }
stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; } stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; }
stream_options(A) ::= stream_options(B) FILL_HISTORY NK_INTEGER(C). { ((SStreamOptions*)B)->fillHistory = taosStr2Int8(C.z, NULL, 10); A = B; }
subtable_opt(A) ::= . { A = NULL; } subtable_opt(A) ::= . { A = NULL; }
subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }

1
source/libs/parser/src/parAstCreater.c
View File

@ -1702,6 +1702,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS); SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->triggerType = STREAM_TRIGGER_AT_ONCE; pOptions->triggerType = STREAM_TRIGGER_AT_ONCE;
pOptions->fillHistory = STREAM_DEFAULT_FILL_HISTORY;
pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED; pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED;
return (SNode*)pOptions; return (SNode*)pOptions;
} }

1
source/libs/parser/src/parTokenizer.c
View File

@ -92,6 +92,7 @@ static SKeyword keywordTable[] = {
{"EVERY", TK_EVERY}, {"EVERY", TK_EVERY},
{"FILE", TK_FILE}, {"FILE", TK_FILE},
{"FILL", TK_FILL}, {"FILL", TK_FILL},
{"FILL_HISTORY", TK_FILL_HISTORY},
{"FIRST", TK_FIRST}, {"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT}, {"FLOAT", TK_FLOAT},
{"FLUSH", TK_FLUSH}, {"FLUSH", TK_FLUSH},

1
source/libs/parser/src/parTranslater.c
View File

@ -5542,6 +5542,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
pReq->triggerType = pStmt->pOptions->triggerType; pReq->triggerType = pStmt->pOptions->triggerType;
pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0); pReq->maxDelay = (NULL != pStmt->pOptions->pDelay ? ((SValueNode*)pStmt->pOptions->pDelay)->datum.i : 0);
pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0);
pReq->fillHistory = pStmt->pOptions->fillHistory;
pReq->igExpired = pStmt->pOptions->ignoreExpired; pReq->igExpired = pStmt->pOptions->ignoreExpired;
columnDefNodeToField(pStmt->pTags, &pReq->pTags); columnDefNodeToField(pStmt->pTags, &pReq->pTags);
pReq->numOfTags = LIST_LENGTH(pStmt->pTags); pReq->numOfTags = LIST_LENGTH(pStmt->pTags);

5438
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

26
source/libs/parser/test/parInitialCTest.cpp
View File

@ -576,10 +576,10 @@ TEST_F(ParserInitialCTest, createStream) {
memset(&expect, 0, sizeof(SCMCreateStreamReq)); memset(&expect, 0, sizeof(SCMCreateStreamReq));
}; };
auto setCreateStreamReqFunc = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb, auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb,
int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0,
int64_t maxDelay = 0, int64_t watermark = 0, int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED,
int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED) { int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) {
snprintf(expect.name, sizeof(expect.name), "0.%s", pStream); snprintf(expect.name, sizeof(expect.name), "0.%s", pStream);
snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb); snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb);
snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb);
@ -588,6 +588,7 @@ TEST_F(ParserInitialCTest, createStream) {
expect.triggerType = triggerType; expect.triggerType = triggerType;
expect.maxDelay = maxDelay; expect.maxDelay = maxDelay;
expect.watermark = watermark; expect.watermark = watermark;
expect.fillHistory = fillHistory;
expect.igExpired = igExpired; expect.igExpired = igExpired;
}; };
@ -619,6 +620,7 @@ TEST_F(ParserInitialCTest, createStream) {
ASSERT_EQ(req.triggerType, expect.triggerType); ASSERT_EQ(req.triggerType, expect.triggerType);
ASSERT_EQ(req.maxDelay, expect.maxDelay); ASSERT_EQ(req.maxDelay, expect.maxDelay);
ASSERT_EQ(req.watermark, expect.watermark); ASSERT_EQ(req.watermark, expect.watermark);
ASSERT_EQ(req.fillHistory, expect.fillHistory);
ASSERT_EQ(req.igExpired, expect.igExpired); ASSERT_EQ(req.igExpired, expect.igExpired);
ASSERT_EQ(req.numOfTags, expect.numOfTags); ASSERT_EQ(req.numOfTags, expect.numOfTags);
if (expect.numOfTags > 0) { if (expect.numOfTags > 0) {
@ -636,21 +638,21 @@ TEST_F(ParserInitialCTest, createStream) {
tFreeSCMCreateStreamReq(&req); tFreeSCMCreateStreamReq(&req);
}); });
setCreateStreamReqFunc("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1"); setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1");
run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)");
clearCreateStreamReq(); clearCreateStreamReq();
setCreateStreamReqFunc("s1", "test", setCreateStreamReq(
"create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 into st1 " "s1", "test",
"create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st1 "
"as select count(*) from t1 interval(10s)", "as select count(*) from t1 interval(10s)",
"st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, "st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1);
0); run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st1 AS "
run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 INTO st1 AS SELECT COUNT(*) " "SELECT COUNT(*) "
"FROM t1 INTERVAL(10S)"); "FROM t1 INTERVAL(10S)");
clearCreateStreamReq(); clearCreateStreamReq();
setCreateStreamReqFunc( setCreateStreamReq("s1", "test",
"s1", "test",
"create stream s1 into st3 tags(tname varchar(10), id int) subtable(concat('new-', tname)) as " "create stream s1 into st3 tags(tname varchar(10), id int) subtable(concat('new-', tname)) as "
"select _wstart wstart, count(*) cnt from st1 partition by tbname tname, tag1 id interval(10s)", "select _wstart wstart, count(*) cnt from st1 partition by tbname tname, tag1 id interval(10s)",
"st3"); "st3");

4
source/libs/parser/test/parTestUtil.cpp
View File

@ -295,6 +295,10 @@ class ParserTestBaseImpl {
char* pStr = NULL; char* pStr = NULL;
int32_t len = 0; int32_t len = 0;
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
// check toObject
SNode* pCopy = NULL;
DO_WITH_THROW(nodesStringToNode, pStr, &pCopy)
nodesDestroyNode(pCopy);
string str(pStr); string str(pStr);
taosMemoryFreeClear(pStr); taosMemoryFreeClear(pStr);
return str; return str;

37
source/libs/planner/src/planLogicCreater.c
View File

@ -339,29 +339,6 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
return code; return code;
} }
static int32_t createColumnByLastRow(SNodeList* pFuncs, SNodeList** pOutput) {
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pCols = NULL;
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
SFunctionNode* pLastRow = (SFunctionNode*)pFunc;
SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pLastRow->pParameterList, 0);
snprintf(pCol->colName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName);
snprintf(pCol->node.aliasName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName);
NODES_CLEAR_LIST(pLastRow->pParameterList);
code = nodesListMakeStrictAppend(&pCols, (SNode*)pCol);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pCols;
} else {
nodesDestroyList(pCols);
}
return code;
}
static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable,
SLogicNode** pLogicNode) { SLogicNode** pLogicNode) {
return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
@ -491,20 +468,6 @@ static SNode* createGroupingSetNode(SNode* pExpr) {
return (SNode*)pGroupingSet; return (SNode*)pGroupingSet;
} }
static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOutput) {
SNodeList* pGroupKeys = NULL;
SNode* pPartKey = NULL;
FOREACH(pPartKey, pPartKeys) {
int32_t code = nodesListMakeStrictAppend(&pGroupKeys, createGroupingSetNode(pPartKey));
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pGroupKeys);
return code;
}
}
*pOutput = pGroupKeys;
return TSDB_CODE_SUCCESS;
}
static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP
: GROUP_ACTION_NONE; : GROUP_ACTION_NONE;

48
source/libs/planner/src/planOptimizer.c
View File

@ -596,18 +596,6 @@ static int32_t pushDownCondOptPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogi
return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond); return pushDownCondOptAppendCond(&pJoin->pOnConditions, pCond);
} }
static int32_t pushDownCondOptPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) {
return pushDownCondOptAppendCond(&pScan->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToProject(SOptimizeContext* pCxt, SProjectLogicNode* pProject, SNode** pCond) {
return pushDownCondOptAppendCond(&pProject->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) {
return pushDownCondOptAppendCond(&pJoin->node.pConditions, pCond);
}
static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
return pushDownCondOptAppendCond(&pChild->pConditions, pCond); return pushDownCondOptAppendCond(&pChild->pConditions, pCond);
} }
@ -1201,40 +1189,6 @@ static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) {
return true; return true;
} }
static int32_t smaIndexOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets,
SLogicNode** pOutput) {
SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
if (NULL == pMerge) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMerge->node.precision = pChild->precision;
pMerge->numOfChannels = 2;
pMerge->pMergeKeys = pMergeKeys;
pMerge->node.pTargets = pTargets;
pMerge->pInputs = nodesCloneList(pChild->pTargets);
if (NULL == pMerge->pInputs) {
nodesDestroyNode((SNode*)pMerge);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pOutput = (SLogicNode*)pMerge;
return TSDB_CODE_SUCCESS;
}
static int32_t smaIndexOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge,
SLogicNode* pSmaScan) {
int32_t code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pInterval);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pMerge->pChildren, (SNode*)pSmaScan);
}
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, pInterval, pMerge);
pSmaScan->pParent = pMerge;
pInterval->pParent = pMerge;
}
return code;
}
static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols,
SLogicNode** pOutput) { SLogicNode** pOutput) {
SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); SScanLogicNode* pSmaScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
@ -1770,7 +1724,7 @@ static int32_t rewriteTailOptCreateLimit(SNode* pLimit, SNode* pOffset, SNode**
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i; pLimitNode->limit = NULL == pLimit ? -1 : ((SValueNode*)pLimit)->datum.i;
pLimitNode->offset = NULL == pOffset ? -1 : ((SValueNode*)pOffset)->datum.i; pLimitNode->offset = NULL == pOffset ? 0 : ((SValueNode*)pOffset)->datum.i;
*pOutput = (SNode*)pLimitNode; *pOutput = (SNode*)pLimitNode;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

1
source/libs/planner/src/planPhysiCreater.c
View File

@ -584,6 +584,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSubplan->showRewrite = pScanLogicNode->showRewrite;
pScan->showRewrite = pScanLogicNode->showRewrite; pScan->showRewrite = pScanLogicNode->showRewrite;
pScan->accountId = pCxt->pPlanCxt->acctId; pScan->accountId = pCxt->pPlanCxt->acctId;
pScan->sysInfo = pCxt->pPlanCxt->sysInfo; pScan->sysInfo = pCxt->pPlanCxt->sysInfo;

3
source/libs/planner/src/planSpliter.c
View File

@ -1419,7 +1419,8 @@ typedef struct SQnodeSplitInfo {
static bool qndSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, static bool qndSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode,
SQnodeSplitInfo* pInfo) { SQnodeSplitInfo* pInfo) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent && if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != pNode->pParent &&
((SScanLogicNode*)pNode)->scanSeq[0] < 1 && ((SScanLogicNode*)pNode)->scanSeq[1] < 1) { QUERY_NODE_LOGIC_PLAN_INTERP_FUNC != nodeType(pNode->pParent) && ((SScanLogicNode*)pNode)->scanSeq[0] <= 1 &&
((SScanLogicNode*)pNode)->scanSeq[1] <= 1) {
pInfo->pSplitNode = pNode; pInfo->pSplitNode = pNode;
pInfo->pSubplan = pSubplan; pInfo->pSubplan = pSubplan;
return true; return true;

9
source/libs/planner/src/planner.c
View File

@ -142,15 +142,6 @@ int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan) {
return nodesMsgToNode(pStr, len, (SNode**)pSubplan); return nodesMsgToNode(pStr, len, (SNode**)pSubplan);
} }
char* qQueryPlanToString(const SQueryPlan* pPlan) {
char* pStr = NULL;
int32_t len = 0;
if (TSDB_CODE_SUCCESS != nodesNodeToString((SNode*)pPlan, false, &pStr, &len)) {
return NULL;
}
return pStr;
}
SQueryPlan* qStringToQueryPlan(const char* pStr) { SQueryPlan* qStringToQueryPlan(const char* pStr) {
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
if (TSDB_CODE_SUCCESS != nodesStringToNode(pStr, (SNode**)&pPlan)) { if (TSDB_CODE_SUCCESS != nodesStringToNode(pStr, (SNode**)&pPlan)) {

5
source/libs/planner/test/planTestUtil.cpp
View File

@ -466,8 +466,13 @@ class PlannerTestBaseImpl {
char* pStr = NULL; char* pStr = NULL;
int32_t len = 0; int32_t len = 0;
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
// check toObject
SNode* pCopy = NULL;
DO_WITH_THROW(nodesStringToNode, pStr, &pCopy)
nodesDestroyNode(pCopy);
string str(pStr); string str(pStr);
taosMemoryFreeClear(pStr); taosMemoryFreeClear(pStr);
return str; return str;
} }

42
source/libs/stream/src/streamState.c
View File

@ -497,7 +497,7 @@ SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSess
if (pCur == NULL) return NULL; if (pCur == NULL) return NULL;
tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL); tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL);
int32_t c = 0; int32_t c = -2;
SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c); tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c);
if (c != 0) { if (c != 0) {
@ -727,3 +727,43 @@ _end:
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return res; return res;
} }
#if 0
char* streamStateSessionDump(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
return NULL;
}
pCur->number = pState->number;
if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) {
streamStateFreeCur(pCur);
return NULL;
}
tdbTbcMoveToFirst(pCur->pCur);
SSessionKey key = {0};
int32_t code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
return NULL;
}
int32_t size = 2048;
char* dumpBuf = taosMemoryCalloc(size, 1);
int64_t len = 0;
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
while (1) {
tdbTbcMoveToNext(pCur->pCur);
key = (SSessionKey){0};
code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
return dumpBuf;
}
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
}
return dumpBuf;
}
#endif

30
source/libs/wal/src/walMeta.c
View File

@ -72,7 +72,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
int64_t capacity = 0; int64_t capacity = 0;
int64_t readSize = 0; int64_t readSize = 0;
char* buf = NULL; char* buf = NULL;
char* found = NULL; int64_t found = -1;
bool firstTrial = pFileInfo->fileSize < fileSize; bool firstTrial = pFileInfo->fileSize < fileSize;
// search for the valid last WAL entry, e.g. block by block // search for the valid last WAL entry, e.g. block by block
@ -114,18 +114,22 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
char* candidate = NULL; char* candidate = NULL;
char* haystack = buf; char* haystack = buf;
int64_t pos = 0;
SWalCkHead* logContent = NULL;
while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) {
pos = candidate - buf;
// validate head // validate head
int64_t len = readSize - (candidate - buf); int64_t len = readSize - pos;
if (len < walCkHeadSz) { if (len < walCkHeadSz) {
break; break;
} }
SWalCkHead* logContent = (SWalCkHead*)candidate; logContent = (SWalCkHead*)(buf + pos);
if (walValidHeadCksum(logContent) != 0) { if (walValidHeadCksum(logContent) != 0) {
wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr); offset + pos, fnameStr);
haystack = candidate + 1; haystack = buf + pos + 1;
if (firstTrial) { if (firstTrial) {
break; break;
} else { } else {
@ -160,11 +164,13 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
break; break;
} }
} }
logContent = (SWalCkHead*)(buf + pos);
if (walValidBodyCksum(logContent) != 0) { if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + ((char*)(logContent)-buf), fnameStr); offset + pos, fnameStr);
haystack = candidate + 1; haystack = buf + pos + 1;
if (firstTrial) { if (firstTrial) {
break; break;
} else { } else {
@ -173,19 +179,19 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
} }
// found one // found one
found = candidate; found = pos;
haystack = candidate + 1; haystack = buf + pos + 1;
} }
if (found || offset == 0) break; if (found >= 0 || offset == 0) break;
// go backwards, e.g. by at most one WAL scan buf size // go backwards, e.g. by at most one WAL scan buf size
end = offset + walCkHeadSz - 1; end = TMIN(offset + walCkHeadSz - 1, fileSize);
firstTrial = false; firstTrial = false;
} }
// determine end of last entry // determine end of last entry
SWalCkHead* lastEntry = (SWalCkHead*)found; SWalCkHead* lastEntry = (found >= 0) ? (SWalCkHead*)(buf + found) : NULL;
int64_t retVer = -1; int64_t retVer = -1;
int64_t lastEntryBeginOffset = 0; int64_t lastEntryBeginOffset = 0;
int64_t lastEntryEndOffset = 0; int64_t lastEntryEndOffset = 0;

2
source/os/src/osDir.c
View File

@ -158,7 +158,7 @@ int32_t taosMulMkDir(const char *dirname) {
#ifdef WINDOWS #ifdef WINDOWS
code = _mkdir(temp, 0755); code = _mkdir(temp, 0755);
#elif defined(DARWIN) #elif defined(DARWIN)
code = mkdir(dirname, 0777); code = mkdir(temp, 0777);
#else #else
code = mkdir(temp, 0755); code = mkdir(temp, 0755);
#endif #endif

2
tests/parallel_test/container_build.sh
View File

@ -52,7 +52,7 @@ fi
docker run \ docker run \
-v $REP_MOUNT_PARAM \ -v $REP_MOUNT_PARAM \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT" --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT"
ret=$? ret=$?
exit $ret exit $ret

2
tests/pytest/util/dnodes.py
View File

@ -40,6 +40,7 @@ class TDSimClient:
"jniDebugFlag": "143", "jniDebugFlag": "143",
"qDebugFlag": "143", "qDebugFlag": "143",
"supportVnodes": "1024", "supportVnodes": "1024",
"enableQueryHb": "1",
"telemetryReporting": "0", "telemetryReporting": "0",
} }
@ -136,6 +137,7 @@ class TDDnode:
"wDebugFlag": "143", "wDebugFlag": "143",
"numOfLogLines": "100000000", "numOfLogLines": "100000000",
"statusInterval": "1", "statusInterval": "1",
"enableQueryHb": "1",
"supportVnodes": "1024", "supportVnodes": "1024",
"telemetryReporting": "0" "telemetryReporting": "0"
} }

5
tests/script/api/makefile
View File

@ -17,6 +17,5 @@ exe:
clean: clean:
rm $(ROOT)batchprepare rm $(ROOT)batchprepare
rm $(ROOT)stmtBatchTest rm $(ROOT)stopquery
rm $(ROOT)stmtTest rm $(ROOT)dbTableRoute
rm $(ROOT)stmt_function

11
tests/script/api/stopquery.c
View File

@ -36,7 +36,7 @@ int64_t st, et;
char hostName[128]; char hostName[128];
char dbName[128]; char dbName[128];
char tbName[128]; char tbName[128];
int32_t runTimes = 1000; int32_t runTimes = 10;
typedef struct { typedef struct {
int id; int id;
@ -657,7 +657,6 @@ int sqConCleanupAsyncQuery(bool fetch) {
void sqRunAllCase(void) { void sqRunAllCase(void) {
#if 1
sqStopSyncQuery(false); sqStopSyncQuery(false);
sqStopSyncQuery(true); sqStopSyncQuery(true);
sqStopAsyncQuery(false); sqStopAsyncQuery(false);
@ -678,33 +677,33 @@ void sqRunAllCase(void) {
sqConCloseAsyncQuery(false); sqConCloseAsyncQuery(false);
sqConCloseAsyncQuery(true); sqConCloseAsyncQuery(true);
sqKillSyncQuery(false); sqKillSyncQuery(false);
sqKillSyncQuery(true); sqKillSyncQuery(true);
sqKillAsyncQuery(false); sqKillAsyncQuery(false);
sqKillAsyncQuery(true); sqKillAsyncQuery(true);
#if 0
/*
sqConKillSyncQuery(false); sqConKillSyncQuery(false);
sqConKillSyncQuery(true); sqConKillSyncQuery(true);
sqConKillAsyncQuery(false); sqConKillAsyncQuery(false);
sqConKillAsyncQuery(true); sqConKillAsyncQuery(true);
#endif
/*
sqConCleanupSyncQuery(false); sqConCleanupSyncQuery(false);
sqConCleanupSyncQuery(true); sqConCleanupSyncQuery(true);
sqConCleanupAsyncQuery(false); sqConCleanupAsyncQuery(false);
sqConCleanupAsyncQuery(true); sqConCleanupAsyncQuery(true);
*/ */
#endif
int32_t l = 5; int32_t l = 5;
while (l) { while (l) {
printf("%d\n", l--); printf("%d\n", l--);
sleep(1); sleep(1);
} }
printf("test done\n");
} }
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
if (argc != 4) { if (argc != 4) {
printf("usage: %s server-ip dbname tablename\n", argv[0]); printf("usage: %s server-ip dbname tablename\n", argv[0]);

8
tests/script/jenkins/basic.txt
View File

@ -56,6 +56,8 @@
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
./test.sh -f tsim/dnode/vnode_clean.sim ./test.sh -f tsim/dnode/vnode_clean.sim
./test.sh -f tsim/dnode/use_dropped_dnode.sim ./test.sh -f tsim/dnode/use_dropped_dnode.sim
./test.sh -f tsim/dnode/split_vgroup_replica1.sim
./test.sh -f tsim/dnode/split_vgroup_replica3.sim
# ---- import ---- # ---- import ----
./test.sh -f tsim/import/basic.sim ./test.sh -f tsim/import/basic.sim
@ -227,7 +229,7 @@
./test.sh -f tsim/table/vgroup.sim ./test.sh -f tsim/table/vgroup.sim
# ---- stream # ---- stream
./test.sh -f tsim/stream/basic0.sim -v ./test.sh -f tsim/stream/basic0.sim -g
./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic1.sim
./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/basic2.sim
./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/drop_stream.sim
@ -241,7 +243,7 @@
./test.sh -f tsim/stream/triggerSession0.sim ./test.sh -f tsim/stream/triggerSession0.sim
./test.sh -f tsim/stream/partitionby.sim ./test.sh -f tsim/stream/partitionby.sim
./test.sh -f tsim/stream/partitionby1.sim ./test.sh -f tsim/stream/partitionby1.sim
# unsupport ./test.sh -f tsim/stream/schedSnode.sim ./test.sh -f tsim/stream/schedSnode.sim
./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/windowClose.sim
./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim
./test.sh -f tsim/stream/sliding.sim ./test.sh -f tsim/stream/sliding.sim
@ -446,5 +448,5 @@
./test.sh -f tsim/tag/tinyint.sim ./test.sh -f tsim/tag/tinyint.sim
./test.sh -f tsim/tag/drop_tag.sim ./test.sh -f tsim/tag/drop_tag.sim
./test.sh -f tmp/monitor.sim
#======================b1-end=============== #======================b1-end===============

8
tests/script/test.sh
View File

@ -15,7 +15,7 @@ VALGRIND=0
UNIQUE=0 UNIQUE=0
UNAME_BIN=`which uname` UNAME_BIN=`which uname`
OS_TYPE=`$UNAME_BIN` OS_TYPE=`$UNAME_BIN`
while getopts "f:avum" arg while getopts "f:agvum" arg
do do
case $arg in case $arg in
f) f)
@ -27,6 +27,9 @@ do
u) u)
UNIQUE=1 UNIQUE=1
;; ;;
g)
VALGRIND=2
;;
?) ?)
echo "unknow argument" echo "unknow argument"
;; ;;
@ -125,6 +128,9 @@ if [ -n "$FILE_NAME" ]; then
FLAG="-v" FLAG="-v"
echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
elif [ $VALGRIND -eq 2 ]; then
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME -v
$PROGRAM -c $CFG_DIR -f $FILE_NAME -v
else else
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
$PROGRAM -c $CFG_DIR -f $FILE_NAME $PROGRAM -c $CFG_DIR -f $FILE_NAME

2
tests/script/tmp/monitor.sim
View File

@ -24,3 +24,5 @@ print =============== create drop qnode 1
sql create qnode on dnode 1 sql create qnode on dnode 1
#sql create snode on dnode 1 #sql create snode on dnode 1
sleep 10000
system sh/exec.sh -n dnode1 -s stop

87
tests/script/tsim/dnode/split_vgroup_replica1.sim Normal file
View File

@ -0,0 +1,87 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 1
system sh/exec.sh -n dnode3 -s start
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step2
endi
if $data(2)[4] != ready then
goto step2
endi
if $data(3)[4] != ready then
goto step2
endi
print =============== step3: create database
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step4: split
print split vgroup 2
sql split vgroup 2
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

67
tests/script/tsim/dnode/split_vgroup_replica3.sim Normal file
View File

@ -0,0 +1,67 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
#system sh/exec.sh -n dnode5 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 4 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 3
print =============== step3: split
print split vgroup 2
sql split vgroup 2
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT

1
tests/script/tsim/insert/basic.sim
View File

@ -50,6 +50,7 @@ print $data[1000][1000]
$a1 = 0 $a1 = 0
$a2 = 0 $a2 = 0
$a3 = $a1 % $a2 $a3 = $a1 % $a2
$a4 = $a1 / $a2
print $a3 print $a3
$val = \\\1 $val = \\\1

7
tests/script/tsim/parser/limit1_stb.sim
View File

@ -39,7 +39,6 @@ endi
$val = $totalNum - 1 $val = $totalNum - 1
sql select * from $stb limit $totalNum offset 1 sql select * from $stb limit $totalNum offset 1
if $rows != $val then if $rows != $val then
print expect $val , actual: $rows
return -1 return -1
endi endi
if $data01 != 1 then if $data01 != 1 then
@ -493,9 +492,3 @@ sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8),
if $rows != 6 then if $rows != 6 then
return -1 return -1
endi endi
sql select * from $stb partition by tbname limit 1
if $rows != 10 then
return -1
endi

1
tests/script/tsim/parser/limit_stb.sim
View File

@ -722,7 +722,6 @@ sql select bottom(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu limit 5 offse
if $rows != 0 then if $rows != 0 then
return -1 return -1
endi endi
sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu limit 3 offset 5 sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu limit 3 offset 5
if $rows != 0 then if $rows != 0 then
return -1 return -1

58
tests/script/tsim/stream/partitionbyColumnInterval.sim
View File

@ -599,12 +599,62 @@ if $loop_count == 20 then
endi endi
if $rows != 4 then if $rows != 4 then
print =====rows=$rows print ======rows=$rows
print =====rows=$rows goto loop15
print =====rows=$rows
#goto loop15
endi endi
sql insert into t1 values(1648791223001,11,2,3,1.0);
sql insert into t2 values(1648791223001,21,2,3,1.0);
sql insert into t3 values(1648791223001,31,2,3,1.0);
sql insert into t4 values(1648791223001,41,2,3,1.0);
sleep 300
sql delete from st where ts = 1648791223001;
$loop_count = 0
loop16:
sleep 50
sql select * from test.streamt5 order by c1, c2, c3;
$loop_count = $loop_count + 1
if $loop_count == 20 then
print ======rows=$rows
return -1
endi
if $rows != 4 then
goto loop16
endi
sql insert into t1 values(1648791223001,12,2,3,1.0);
sql insert into t2 values(1648791223001,22,2,3,1.0);
sql insert into t3 values(1648791223001,32,2,3,1.0);
sql insert into t4 values(1648791223001,42,2,3,1.0);
sleep 300
sql delete from st where ts = 1648791223001;
$loop_count = 0
loop17:
sleep 50
sql select * from test.streamt5 order by c1, c2, c3;
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 4 then
print ======rows=$rows
print ======rows=$rows
print ======rows=$rows
return 1
#goto loop17
endi
$loop_all = $loop_all + 1 $loop_all = $loop_all + 1
print ============loop_all=$loop_all print ============loop_all=$loop_all

36
tests/script/tsim/stream/schedSnode.sim
View File

@ -6,10 +6,14 @@ system sh/exec.sh -n dnode1 -s start
sleep 50 sleep 50
sql connect sql connect
sleep 50
sql create database test vgroups 2;
sql create database target vgroups 1;
sql create snode on dnode 1 sql create snode on dnode 1
sql create database test vgroups 1;
sql create database target vgroups 1;
sql use test; sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1); sql create table ts1 using st tags(1,1,1);
@ -72,23 +76,23 @@ if $data01 != 8 then
goto loop1 goto loop1
endi endi
if $data02 != 4 then if $data02 != 6 then
print =====data02=$data02 print =====data02=$data02
goto loop1 goto loop1
endi endi
if $data03 != 4 then if $data03 != 52 then
print ======$data03 print ======data03=$data03
goto loop1 goto loop1
endi endi
if $data04 != 52 then if $data04 != 52 then
print ======$data04 print ======data04=$data04
goto loop1 goto loop1
endi endi
if $data05 != 13 then if $data05 != 13 then
print ======$data05 print ======data05=$data05
goto loop1 goto loop1
endi endi
@ -104,17 +108,17 @@ if $data12 != 6 then
endi endi
if $data13 != 92 then if $data13 != 92 then
print ======$data13 print ======data13=$data13
return -1 return -1
endi endi
if $data14 != 22 then if $data14 != 22 then
print ======$data14 print ======data14=$data14
return -1 return -1
endi endi
if $data15 != 3 then if $data15 != 3 then
print ======$data15 print ======data15=$data15
return -1 return -1
endi endi
@ -130,17 +134,17 @@ if $data22 != 4 then
endi endi
if $data23 != 32 then if $data23 != 32 then
print ======$data23 print ======data23=$data23
return -1 return -1
endi endi
if $data24 != 12 then if $data24 != 12 then
print ======$data24 print ======data24=$data24
return -1 return -1
endi endi
if $data25 != 3 then if $data25 != 3 then
print ======$data25 print ======data25=$data25
return -1 return -1
endi endi
@ -156,17 +160,17 @@ if $data32 != 30 then
endi endi
if $data33 != 180 then if $data33 != 180 then
print ======$data33 print ======data33=$data33
return -1 return -1
endi endi
if $data34 != 42 then if $data34 != 42 then
print ======$data34 print ======data34=$data34
return -1 return -1
endi endi
if $data35 != 3 then if $data35 != 3 then
print ======$data35 print ======data35=$data35
return -1 return -1
endi endi

7
tests/system-test/0-others/taosdShell.py
View File

@ -134,7 +134,11 @@ class TDTestCase:
tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);") tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);")
#TD-19944 -Q=3
tdsqlN=tdCom.newTdSql()
tdsqlN.query("select * from source_db.stb")
tdsqlN.query("select * from db0.stb")
def run(self): def run(self):
# tdSql.prepare() # tdSql.prepare()
@ -164,6 +168,7 @@ class TDTestCase:
startAction = " -s -c " + taosdCfgPath startAction = " -s -c " + taosdCfgPath
tdLog.printNoPrefix("================================ parameter: %s"%startAction) tdLog.printNoPrefix("================================ parameter: %s"%startAction)
self.taosdCommandExe(startAction,taosdCmdRun) self.taosdCommandExe(startAction,taosdCmdRun)
os.system(" rm -rf sdb.json ")
startAction = " --help" startAction = " --help"
@ -198,7 +203,7 @@ class TDTestCase:
startAction=" -E taosdCaseTmp/.env" startAction=" -E taosdCaseTmp/.env"
tdLog.printNoPrefix("================================ parameter: %s"%startAction) tdLog.printNoPrefix("================================ parameter: %s"%startAction)
os.system(" mkdir -p taosdCaseTmp/.env ") os.system(" mkdir -p taosdCaseTmp ")
os.system("echo \'TAOS_QUERY_POLICY=3\' > taosdCaseTmp/.env ") os.system("echo \'TAOS_QUERY_POLICY=3\' > taosdCaseTmp/.env ")
self.taosdCommandStop(startAction,taosdCmdRun) self.taosdCommandStop(startAction,taosdCmdRun)
os.system(" rm -rf taosdCaseTmp/.env ") os.system(" rm -rf taosdCaseTmp/.env ")

14
tests/system-test/1-insert/block_wise.py
View File

@ -426,7 +426,17 @@ class TDTestCase:
tdDnodes.stop(1) tdDnodes.stop(1)
tdDnodes.start(1) tdDnodes.start(1)
tdLog.printNoPrefix("==========step3:insert and flush in rollup database")
tdLog.printNoPrefix("==========step3: sleep 20s for catalogUpdateTableIndex")
tdSql.execute("create database db_s20")
tdSql.execute("use db_s20")
tdSql.execute(f"create stable stb1 (ts timestamp, c1 int) tags (t1 int) sma(c1);")
tdSql.execute("alter stable stb1 add column tinyint_col tinyint")
time.sleep(20)
tdSql.query("select count(*) from stb1")
tdSql.execute("drop database if exists db_s20 ")
tdLog.printNoPrefix("==========step4:insert and flush in rollup database")
tdSql.execute("create database db4 retentions 1s:4m,2s:8m,3s:12m") tdSql.execute("create database db4 retentions 1s:4m,2s:8m,3s:12m")
tdSql.execute("use db4") tdSql.execute("use db4")
self.__create_tb(rollup="first") self.__create_tb(rollup="first")
@ -435,7 +445,7 @@ class TDTestCase:
tdSql.execute(f'flush database db4') tdSql.execute(f'flush database db4')
tdLog.printNoPrefix("==========step4:after wal, all check again ") tdLog.printNoPrefix("==========step5:after wal, all check again ")
tdSql.prepare() tdSql.prepare()
self.__create_tb() self.__create_tb()
self.__insert_data() self.__insert_data()

Some files were not shown because too many files have changed in this diff Show More