xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feat/TD-30268

This commit is contained in:
dapan1121 2024-07-01 13:48:48 +08:00
parent 05917ea2a4 b07d2b4adc
commit 2a4ba7eafd
128 changed files with 10038 additions and 6953 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/mips/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

6
docs/en/12-taos-sql/10-function.md
View File

@ -398,7 +398,7 @@ Conversion functions change the data type of a value.
CAST(expr AS type_name)
```
**Description**: Convert the input data `expr` into the type specified by `type_name`. This function can be used only in SELECT statements.
**Description**: Convert the input data `expr` into the type specified by `type_name`.
**Return value type**: The type specified by parameter `type_name`
@ -435,8 +435,7 @@ TO_ISO8601(expr [, timezone])
**More explanations**:
- You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]. For example, TO_ISO8601(1, "+00:00").
- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp
- If the input is a column of TIMESTAMP type, the precision of the returned value is same as the precision set for the current data base in use
- The precision of the input timestamp will be recognized automatically according to the precision of the table used, milliseconds will be used if no table is specified.
#### TO_JSON
@ -650,6 +649,7 @@ use_current_timezone: {
- Time unit specified by `time_unit` can be:
1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks)
- The precision of the returned timestamp is same as the precision set for the current data base in use
- The precision of the input timestamp will be recognized automatically according to the precision of the table used, milliseconds will be used if no table is specified.
- If the input data is not formatted as a timestamp, the returned value is null.
- When using 1d/1w as the time unit to truncate timestamp, you can specify whether to truncate based on the current time zone by setting the use_current_timezone parameter.
Value 0 indicates truncation using the UTC time zone, value 1 indicates truncation using the current time zone.

6
docs/zh/12-taos-sql/10-function.md
View File

@ -398,7 +398,7 @@ UPPER(expr)
CAST(expr AS type_name)
```
**功能说明**：数据类型转换函数，返回 expr 转换为 type_name 指定的类型后的结果。只适用于 select 子句中。
**功能说明**：数据类型转换函数，返回 expr 转换为 type_name 指定的类型后的结果。
**返回结果类型**：CAST 中指定的类型（type_name)。
@ -435,8 +435,7 @@ TO_ISO8601(expr [, timezone])
**使用说明**：
- timezone 参数允许输入的时区格式为: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。
- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定;
- 如果输入是 TIMESTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。
- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒.
#### TO_JSON
@ -650,6 +649,7 @@ use_current_timezone: {
- 支持的时间单位 time_unit 如下：
1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。
- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。
- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒.
- 输入包含不符合时间日期格式的字符串则返回 NULL。
- 当使用 1d/1w 作为时间单位对时间戳进行截断时， 可通过设置 use_current_timezone 参数指定是否根据当前时区进行截断处理。
值 0 表示使用 UTC 时区进行截断，值 1 表示使用当前时区进行截断。

7
include/common/tglobal.h
View File

@ -24,10 +24,11 @@
extern "C" {
#endif
#define SLOW_LOG_TYPE_NULL 0x0
#define SLOW_LOG_TYPE_QUERY 0x1
#define SLOW_LOG_TYPE_INSERT 0x2
#define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF
#define SLOW_LOG_TYPE_ALL 0x7
typedef enum {
DND_CA_SM4 = 1,
@ -177,7 +178,10 @@ extern int32_t tsMaxRetryWaitTime;
extern bool tsUseAdapter;
extern int32_t tsMetaCacheMaxSize;
extern int32_t tsSlowLogThreshold;
extern int32_t tsSlowLogThresholdTest;
extern char tsSlowLogExceptDb[];
extern int32_t tsSlowLogScope;
extern int32_t tsSlowLogMaxLen;
extern int32_t tsTimeSeriesThreshold;
extern bool tsMultiResultFunctionStarReturnTags;
@ -273,6 +277,7 @@ void taosSetGlobalDebugFlag(int32_t flag);
void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal);
void taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
int8_t taosGranted(int8_t type);
int32_t taosSetSlowLogScope(char *pScope);
#ifdef __cplusplus
}

24
include/common/tmsg.h
View File

@ -344,6 +344,7 @@ typedef enum ENodeType {
QUERY_NODE_RESUME_STREAM_STMT,
QUERY_NODE_CREATE_VIEW_STMT,
QUERY_NODE_DROP_VIEW_STMT,
QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE,
// show statement nodes
// see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
@ -654,6 +655,16 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define SSCHMEA_BYTES(s) ((s)->bytes)
#define SSCHMEA_NAME(s) ((s)->name)
typedef struct {
bool tsEnableMonitor;
int32_t tsMonitorInterval;
int32_t tsSlowLogThreshold;
int32_t tsSlowLogMaxLen;
int32_t tsSlowLogScope;
int32_t tsSlowLogThresholdTest;
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN];
} SMonitorParas;
typedef struct {
int32_t nCols;
int32_t version;
@ -968,6 +979,7 @@ typedef struct {
char sVer[TSDB_VERSION_LEN];
char sDetailVer[128];
int64_t whiteListVer;
SMonitorParas monitorParas;
} SConnectRsp;
int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp);
@ -1635,6 +1647,7 @@ typedef struct {
int8_t enableWhiteList;
int8_t encryptionKeyStat;
uint32_t encryptionKeyChksum;
SMonitorParas monitorParas;
} SClusterCfg;
typedef struct {
@ -1726,9 +1739,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq);
typedef enum {
MONITOR_TYPE_COUNTER = 0,
MONITOR_TYPE_SLOW_LOG = 1,
} MONITOR_TYPE;
typedef struct {
int32_t contLen;
char* pCont;
int32_t contLen;
char* pCont;
MONITOR_TYPE type;
} SStatisReq;
int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
@ -3260,6 +3279,7 @@ typedef struct {
int64_t rspId;
int32_t svrTimestamp;
SArray* rsps; // SArray<SClientHbRsp>
SMonitorParas monitorParas;
} SClientHbBatchRsp;
static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { return taosIntHash_64(key, keyLen); }

446
include/common/ttokendef.h
View File

@ -141,229 +141,229 @@
#define TK_NK_EQ 123
#define TK_USING 124
#define TK_TAGS 125
#define TK_BOOL 126
#define TK_TINYINT 127
#define TK_SMALLINT 128
#define TK_INT 129
#define TK_INTEGER 130
#define TK_BIGINT 131
#define TK_FLOAT 132
#define TK_DOUBLE 133
#define TK_BINARY 134
#define TK_NCHAR 135
#define TK_UNSIGNED 136
#define TK_JSON 137
#define TK_VARCHAR 138
#define TK_MEDIUMBLOB 139
#define TK_BLOB 140
#define TK_VARBINARY 141
#define TK_GEOMETRY 142
#define TK_DECIMAL 143
#define TK_COMMENT 144
#define TK_MAX_DELAY 145
#define TK_WATERMARK 146
#define TK_ROLLUP 147
#define TK_TTL 148
#define TK_SMA 149
#define TK_DELETE_MARK 150
#define TK_FIRST 151
#define TK_LAST 152
#define TK_SHOW 153
#define TK_FULL 154
#define TK_PRIVILEGES 155
#define TK_DATABASES 156
#define TK_TABLES 157
#define TK_STABLES 158
#define TK_MNODES 159
#define TK_QNODES 160
#define TK_ARBGROUPS 161
#define TK_FUNCTIONS 162
#define TK_INDEXES 163
#define TK_ACCOUNTS 164
#define TK_APPS 165
#define TK_CONNECTIONS 166
#define TK_LICENCES 167
#define TK_GRANTS 168
#define TK_LOGS 169
#define TK_MACHINES 170
#define TK_ENCRYPTIONS 171
#define TK_QUERIES 172
#define TK_SCORES 173
#define TK_TOPICS 174
#define TK_VARIABLES 175
#define TK_BNODES 176
#define TK_SNODES 177
#define TK_TRANSACTIONS 178
#define TK_DISTRIBUTED 179
#define TK_CONSUMERS 180
#define TK_SUBSCRIPTIONS 181
#define TK_VNODES 182
#define TK_ALIVE 183
#define TK_VIEWS 184
#define TK_VIEW 185
#define TK_COMPACTS 186
#define TK_NORMAL 187
#define TK_CHILD 188
#define TK_LIKE 189
#define TK_TBNAME 190
#define TK_QTAGS 191
#define TK_AS 192
#define TK_SYSTEM 193
#define TK_TSMA 194
#define TK_INTERVAL 195
#define TK_RECURSIVE 196
#define TK_TSMAS 197
#define TK_FUNCTION 198
#define TK_INDEX 199
#define TK_COUNT 200
#define TK_LAST_ROW 201
#define TK_META 202
#define TK_ONLY 203
#define TK_TOPIC 204
#define TK_CONSUMER 205
#define TK_GROUP 206
#define TK_DESC 207
#define TK_DESCRIBE 208
#define TK_RESET 209
#define TK_QUERY 210
#define TK_CACHE 211
#define TK_EXPLAIN 212
#define TK_ANALYZE 213
#define TK_VERBOSE 214
#define TK_NK_BOOL 215
#define TK_RATIO 216
#define TK_NK_FLOAT 217
#define TK_OUTPUTTYPE 218
#define TK_AGGREGATE 219
#define TK_BUFSIZE 220
#define TK_LANGUAGE 221
#define TK_REPLACE 222
#define TK_STREAM 223
#define TK_INTO 224
#define TK_PAUSE 225
#define TK_RESUME 226
#define TK_PRIMARY 227
#define TK_KEY 228
#define TK_TRIGGER 229
#define TK_AT_ONCE 230
#define TK_WINDOW_CLOSE 231
#define TK_IGNORE 232
#define TK_EXPIRED 233
#define TK_FILL_HISTORY 234
#define TK_UPDATE 235
#define TK_SUBTABLE 236
#define TK_UNTREATED 237
#define TK_KILL 238
#define TK_CONNECTION 239
#define TK_TRANSACTION 240
#define TK_BALANCE 241
#define TK_VGROUP 242
#define TK_LEADER 243
#define TK_MERGE 244
#define TK_REDISTRIBUTE 245
#define TK_SPLIT 246
#define TK_DELETE 247
#define TK_INSERT 248
#define TK_NK_BIN 249
#define TK_NK_HEX 250
#define TK_NULL 251
#define TK_NK_QUESTION 252
#define TK_NK_ALIAS 253
#define TK_NK_ARROW 254
#define TK_ROWTS 255
#define TK_QSTART 256
#define TK_QEND 257
#define TK_QDURATION 258
#define TK_WSTART 259
#define TK_WEND 260
#define TK_WDURATION 261
#define TK_IROWTS 262
#define TK_ISFILLED 263
#define TK_CAST 264
#define TK_NOW 265
#define TK_TODAY 266
#define TK_TIMEZONE 267
#define TK_CLIENT_VERSION 268
#define TK_SERVER_VERSION 269
#define TK_SERVER_STATUS 270
#define TK_CURRENT_USER 271
#define TK_CASE 272
#define TK_WHEN 273
#define TK_THEN 274
#define TK_ELSE 275
#define TK_BETWEEN 276
#define TK_IS 277
#define TK_NK_LT 278
#define TK_NK_GT 279
#define TK_NK_LE 280
#define TK_NK_GE 281
#define TK_NK_NE 282
#define TK_MATCH 283
#define TK_NMATCH 284
#define TK_CONTAINS 285
#define TK_IN 286
#define TK_JOIN 287
#define TK_INNER 288
#define TK_LEFT 289
#define TK_RIGHT 290
#define TK_OUTER 291
#define TK_SEMI 292
#define TK_ANTI 293
#define TK_ASOF 294
#define TK_WINDOW 295
#define TK_WINDOW_OFFSET 296
#define TK_JLIMIT 297
#define TK_SELECT 298
#define TK_NK_HINT 299
#define TK_DISTINCT 300
#define TK_WHERE 301
#define TK_PARTITION 302
#define TK_BY 303
#define TK_SESSION 304
#define TK_STATE_WINDOW 305
#define TK_EVENT_WINDOW 306
#define TK_COUNT_WINDOW 307
#define TK_SLIDING 308
#define TK_FILL 309
#define TK_VALUE 310
#define TK_VALUE_F 311
#define TK_NONE 312
#define TK_PREV 313
#define TK_NULL_F 314
#define TK_LINEAR 315
#define TK_NEXT 316
#define TK_HAVING 317
#define TK_RANGE 318
#define TK_EVERY 319
#define TK_ORDER 320
#define TK_SLIMIT 321
#define TK_SOFFSET 322
#define TK_LIMIT 323
#define TK_OFFSET 324
#define TK_ASC 325
#define TK_NULLS 326
#define TK_ABORT 327
#define TK_AFTER 328
#define TK_ATTACH 329
#define TK_BEFORE 330
#define TK_BEGIN 331
#define TK_BITAND 332
#define TK_BITNOT 333
#define TK_BITOR 334
#define TK_BLOCKS 335
#define TK_CHANGE 336
#define TK_COMMA 337
#define TK_CONCAT 338
#define TK_CONFLICT 339
#define TK_COPY 340
#define TK_DEFERRED 341
#define TK_DELIMITERS 342
#define TK_DETACH 343
#define TK_DIVIDE 344
#define TK_DOT 345
#define TK_EACH 346
#define TK_FAIL 347
#define TK_FILE 348
#define TK_FILE 126
#define TK_BOOL 127
#define TK_TINYINT 128
#define TK_SMALLINT 129
#define TK_INT 130
#define TK_INTEGER 131
#define TK_BIGINT 132
#define TK_FLOAT 133
#define TK_DOUBLE 134
#define TK_BINARY 135
#define TK_NCHAR 136
#define TK_UNSIGNED 137
#define TK_JSON 138
#define TK_VARCHAR 139
#define TK_MEDIUMBLOB 140
#define TK_BLOB 141
#define TK_VARBINARY 142
#define TK_GEOMETRY 143
#define TK_DECIMAL 144
#define TK_COMMENT 145
#define TK_MAX_DELAY 146
#define TK_WATERMARK 147
#define TK_ROLLUP 148
#define TK_TTL 149
#define TK_SMA 150
#define TK_DELETE_MARK 151
#define TK_FIRST 152
#define TK_LAST 153
#define TK_SHOW 154
#define TK_FULL 155
#define TK_PRIVILEGES 156
#define TK_DATABASES 157
#define TK_TABLES 158
#define TK_STABLES 159
#define TK_MNODES 160
#define TK_QNODES 161
#define TK_ARBGROUPS 162
#define TK_FUNCTIONS 163
#define TK_INDEXES 164
#define TK_ACCOUNTS 165
#define TK_APPS 166
#define TK_CONNECTIONS 167
#define TK_LICENCES 168
#define TK_GRANTS 169
#define TK_LOGS 170
#define TK_MACHINES 171
#define TK_ENCRYPTIONS 172
#define TK_QUERIES 173
#define TK_SCORES 174
#define TK_TOPICS 175
#define TK_VARIABLES 176
#define TK_BNODES 177
#define TK_SNODES 178
#define TK_TRANSACTIONS 179
#define TK_DISTRIBUTED 180
#define TK_CONSUMERS 181
#define TK_SUBSCRIPTIONS 182
#define TK_VNODES 183
#define TK_ALIVE 184
#define TK_VIEWS 185
#define TK_VIEW 186
#define TK_COMPACTS 187
#define TK_NORMAL 188
#define TK_CHILD 189
#define TK_LIKE 190
#define TK_TBNAME 191
#define TK_QTAGS 192
#define TK_AS 193
#define TK_SYSTEM 194
#define TK_TSMA 195
#define TK_INTERVAL 196
#define TK_RECURSIVE 197
#define TK_TSMAS 198
#define TK_FUNCTION 199
#define TK_INDEX 200
#define TK_COUNT 201
#define TK_LAST_ROW 202
#define TK_META 203
#define TK_ONLY 204
#define TK_TOPIC 205
#define TK_CONSUMER 206
#define TK_GROUP 207
#define TK_DESC 208
#define TK_DESCRIBE 209
#define TK_RESET 210
#define TK_QUERY 211
#define TK_CACHE 212
#define TK_EXPLAIN 213
#define TK_ANALYZE 214
#define TK_VERBOSE 215
#define TK_NK_BOOL 216
#define TK_RATIO 217
#define TK_NK_FLOAT 218
#define TK_OUTPUTTYPE 219
#define TK_AGGREGATE 220
#define TK_BUFSIZE 221
#define TK_LANGUAGE 222
#define TK_REPLACE 223
#define TK_STREAM 224
#define TK_INTO 225
#define TK_PAUSE 226
#define TK_RESUME 227
#define TK_PRIMARY 228
#define TK_KEY 229
#define TK_TRIGGER 230
#define TK_AT_ONCE 231
#define TK_WINDOW_CLOSE 232
#define TK_IGNORE 233
#define TK_EXPIRED 234
#define TK_FILL_HISTORY 235
#define TK_UPDATE 236
#define TK_SUBTABLE 237
#define TK_UNTREATED 238
#define TK_KILL 239
#define TK_CONNECTION 240
#define TK_TRANSACTION 241
#define TK_BALANCE 242
#define TK_VGROUP 243
#define TK_LEADER 244
#define TK_MERGE 245
#define TK_REDISTRIBUTE 246
#define TK_SPLIT 247
#define TK_DELETE 248
#define TK_INSERT 249
#define TK_NK_BIN 250
#define TK_NK_HEX 251
#define TK_NULL 252
#define TK_NK_QUESTION 253
#define TK_NK_ALIAS 254
#define TK_NK_ARROW 255
#define TK_ROWTS 256
#define TK_QSTART 257
#define TK_QEND 258
#define TK_QDURATION 259
#define TK_WSTART 260
#define TK_WEND 261
#define TK_WDURATION 262
#define TK_IROWTS 263
#define TK_ISFILLED 264
#define TK_CAST 265
#define TK_NOW 266
#define TK_TODAY 267
#define TK_TIMEZONE 268
#define TK_CLIENT_VERSION 269
#define TK_SERVER_VERSION 270
#define TK_SERVER_STATUS 271
#define TK_CURRENT_USER 272
#define TK_CASE 273
#define TK_WHEN 274
#define TK_THEN 275
#define TK_ELSE 276
#define TK_BETWEEN 277
#define TK_IS 278
#define TK_NK_LT 279
#define TK_NK_GT 280
#define TK_NK_LE 281
#define TK_NK_GE 282
#define TK_NK_NE 283
#define TK_MATCH 284
#define TK_NMATCH 285
#define TK_CONTAINS 286
#define TK_IN 287
#define TK_JOIN 288
#define TK_INNER 289
#define TK_LEFT 290
#define TK_RIGHT 291
#define TK_OUTER 292
#define TK_SEMI 293
#define TK_ANTI 294
#define TK_ASOF 295
#define TK_WINDOW 296
#define TK_WINDOW_OFFSET 297
#define TK_JLIMIT 298
#define TK_SELECT 299
#define TK_NK_HINT 300
#define TK_DISTINCT 301
#define TK_WHERE 302
#define TK_PARTITION 303
#define TK_BY 304
#define TK_SESSION 305
#define TK_STATE_WINDOW 306
#define TK_EVENT_WINDOW 307
#define TK_COUNT_WINDOW 308
#define TK_SLIDING 309
#define TK_FILL 310
#define TK_VALUE 311
#define TK_VALUE_F 312
#define TK_NONE 313
#define TK_PREV 314
#define TK_NULL_F 315
#define TK_LINEAR 316
#define TK_NEXT 317
#define TK_HAVING 318
#define TK_RANGE 319
#define TK_EVERY 320
#define TK_ORDER 321
#define TK_SLIMIT 322
#define TK_SOFFSET 323
#define TK_LIMIT 324
#define TK_OFFSET 325
#define TK_ASC 326
#define TK_NULLS 327
#define TK_ABORT 328
#define TK_AFTER 329
#define TK_ATTACH 330
#define TK_BEFORE 331
#define TK_BEGIN 332
#define TK_BITAND 333
#define TK_BITNOT 334
#define TK_BITOR 335
#define TK_BLOCKS 336
#define TK_CHANGE 337
#define TK_COMMA 338
#define TK_CONCAT 339
#define TK_CONFLICT 340
#define TK_COPY 341
#define TK_DEFERRED 342
#define TK_DELIMITERS 343
#define TK_DETACH 344
#define TK_DIVIDE 345
#define TK_DOT 346
#define TK_EACH 347
#define TK_FAIL 348
#define TK_FOR 349
#define TK_GLOB 350
#define TK_ID 351

2
include/libs/catalog/catalog.h
View File

@ -206,7 +206,7 @@ int32_t catalogInit(SCatalogCfg* cfg);
* @param catalogHandle (output, NO need to free it)
* @return error code
*/
int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
int32_t catalogGetHandle(int64_t clusterId, SCatalog** catalogHandle);
int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* stateTs);

35
include/libs/monitor/clientMonitor.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "taos_monitor.h"
#include "thash.h"
#include "query.h"
#include "tqueue.h"
typedef enum SQL_RESULT_CODE {
SQL_RESULT_SUCCESS = 0,
@ -30,24 +31,36 @@ typedef enum SQL_RESULT_CODE {
SQL_RESULT_CANCEL = 2,
} SQL_RESULT_CODE;
const char* resultStr(SQL_RESULT_CODE code);
#define SLOW_LOG_SEND_SIZE 32*1024
typedef struct {
char clusterKey[512];
SEpSet epSet;
void* pTransporter;
int64_t clusterId;
taos_collector_registry_t* registry;
taos_collector_t* colector;
SHashObj* counters;
} ClientMonitor;
void* timer;
} MonitorClient;
void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter);
void clusterMonitorClose(const char* clusterKey);
taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
const char** label_keys);
int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values);
void cluster_monitor_stop();
typedef struct {
TdFilePtr pFile;
void* timer;
} SlowLogClient;
typedef struct {
int64_t clusterId;
char *value;
} MonitorSlowLogData;
void monitorClose();
void monitorInit();
void monitorClientSQLReqInit(int64_t clusterKey);
void monitorClientSlowQueryInit(int64_t clusterId);
void monitorCreateClient(int64_t clusterId);
void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count, const char** label_keys);
void monitorCounterInc(int64_t clusterId, const char* counterName, const char** label_values);
const char* monitorResultStr(SQL_RESULT_CODE code);
int32_t monitorPutData2MonitorQueue(int64_t clusterId, char* value);
#ifdef __cplusplus
}
#endif

2
include/libs/monitor/monitor.h
View File

@ -227,7 +227,7 @@ void monSetSmInfo(SMonSmInfo *pInfo);
void monSetBmInfo(SMonBmInfo *pInfo);
void monGenAndSendReport();
void monGenAndSendReportBasic();
void monSendContent(char *pCont);
void monSendContent(char *pCont, const char* uri);
void tFreeSMonMmInfo(SMonMmInfo *pInfo);
void tFreeSMonVmInfo(SMonVmInfo *pInfo);

12
include/libs/nodes/cmdnodes.h
View File

@ -214,6 +214,18 @@ typedef struct SCreateSubTableClause {
STableOptions* pOptions;
} SCreateSubTableClause;
typedef struct SCreateSubTableFromFileClause {
ENodeType type;
char useDbName[TSDB_DB_NAME_LEN];
char useTableName[TSDB_TABLE_NAME_LEN];
bool ignoreExists;
SNodeList* pSpecificTags;
char filePath[PATH_MAX];
TdFilePtr fp;
SArray* aCreateTbData;
SArray* aTagIndexs;
} SCreateSubTableFromFileClause;
typedef struct SCreateMultiTablesStmt {
ENodeType type;
SNodeList* pSubTables;

1
include/libs/parser/parser.h
View File

@ -101,6 +101,7 @@ typedef struct SParseContext {
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
bool qIsInsertValuesSql(const char* pStr, size_t length);
bool qParseDbName(const char* pStr, size_t length, char** pDbName);
// for async mode
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq);

12
include/libs/stream/streammsg.h
View File

@ -182,12 +182,12 @@ typedef struct SRetrieveChkptTriggerReq {
} SRetrieveChkptTriggerReq;
typedef struct SCheckpointTriggerRsp {
int64_t streamId;
int64_t checkpointId;
int32_t upstreamTaskId;
int32_t taskId;
int32_t transId;
int32_t rspCode;
int64_t streamId;
int64_t checkpointId;
int32_t upstreamTaskId;
int32_t taskId;
int32_t transId;
int32_t rspCode;
} SCheckpointTriggerRsp;
typedef struct SCheckpointReport {

115
include/libs/stream/tstream.h
View File

@ -18,13 +18,13 @@
#include "os.h"
#include "streamState.h"
#include "streammsg.h"
#include "tdatablock.h"
#include "tdbInt.h"
#include "tmsg.h"
#include "tmsgcb.h"
#include "tqueue.h"
#include "ttimer.h"
#include "streammsg.h"
#ifdef __cplusplus
extern "C" {
@ -265,14 +265,14 @@ typedef struct SStreamTaskId {
} SStreamTaskId;
typedef struct SCheckpointInfo {
int64_t startTs;
int64_t checkpointId; // latest checkpoint id
int64_t checkpointVer; // latest checkpoint offset in wal
int64_t checkpointTime; // latest checkpoint time
int64_t processedVer;
int64_t nextProcessVer; // current offset in WAL, not serialize it
int64_t startTs;
int64_t checkpointId; // latest checkpoint id
int64_t checkpointVer; // latest checkpoint offset in wal
int64_t checkpointTime; // latest checkpoint time
int64_t processedVer;
int64_t nextProcessVer; // current offset in WAL, not serialize it
SActiveCheckpointInfo* pActiveInfo;
int64_t msgVer;
int64_t msgVer;
} SCheckpointInfo;
typedef struct SStreamStatus {
@ -301,7 +301,7 @@ typedef struct SSTaskBasicInfo {
int32_t selfChildId;
int32_t totalLevel;
int8_t taskLevel;
int8_t fillHistory; // is fill history task or not
int8_t fillHistory; // is fill history task or not
int64_t delaySchedParam; // in msec
} SSTaskBasicInfo;
@ -323,7 +323,7 @@ typedef struct SDispatchMsgInfo {
void* pRetryTmr; // used to dispatch data after a given time duration
TdThreadMutex lock;
int8_t inMonitor;
SArray* pSendInfo; // SArray<SDispatchEntry>
SArray* pSendInfo; // SArray<SDispatchEntry>
} SDispatchMsgInfo;
typedef struct STaskQueue {
@ -345,11 +345,11 @@ typedef struct SSinkRecorder {
} SSinkRecorder;
typedef struct STaskExecStatisInfo {
int64_t created;
int64_t checkTs;
int64_t readyTs;
int64_t startCheckpointId;
int64_t startCheckpointVer;
int64_t created;
int64_t checkTs;
int64_t readyTs;
int64_t startCheckpointId;
int64_t startCheckpointVer;
int64_t step1Start;
double step1El;
@ -432,25 +432,25 @@ struct SStreamTask {
SCheckpointInfo chkInfo;
STaskExec exec;
SDataRange dataRange;
SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified.
SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified.
SHistoryTaskInfo hTaskInfo;
STaskId streamTaskId;
STaskExecStatisInfo execInfo;
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
int32_t refCnt;
int32_t transferStateAlignCnt;
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5;
char reserve[256];
int32_t refCnt;
int32_t transferStateAlignCnt;
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5;
char reserve[256];
};
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -459,7 +459,7 @@ typedef struct STaskStartInfo {
int64_t startTs;
int64_t readyTs;
int32_t tasksWillRestart;
int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure.
int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure.
SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing
SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed
int64_t elapsedTime;
@ -512,7 +512,7 @@ typedef struct SStreamMeta {
SArray* chkpSaved;
SArray* chkpInUse;
SRWLatch chkpDirLock;
void* qHandle; // todo remove it
void* qHandle; // todo remove it
void* bkdChkptMgt;
} SStreamMeta;
@ -567,14 +567,14 @@ typedef struct {
} SStreamScanHistoryReq;
typedef struct STaskCkptInfo {
int64_t latestId; // saved checkpoint id
int64_t latestVer; // saved checkpoint ver
int64_t latestTime; // latest checkpoint time
int64_t latestSize; // latest checkpoint size
int8_t remoteBackup; // latest checkpoint backup done
int64_t activeId; // current active checkpoint id
int32_t activeTransId; // checkpoint trans id
int8_t failed; // denote if the checkpoint is failed or not
int64_t latestId; // saved checkpoint id
int64_t latestVer; // saved checkpoint ver
int64_t latestTime; // latest checkpoint time
int64_t latestSize; // latest checkpoint size
int8_t remoteBackup; // latest checkpoint backup done
int64_t activeId; // current active checkpoint id
int32_t activeTransId; // checkpoint trans id
int8_t failed; // denote if the checkpoint is failed or not
} STaskCkptInfo;
typedef struct STaskStatusEntry {
@ -589,12 +589,12 @@ typedef struct STaskStatusEntry {
int64_t inputQUnchangeCounter;
double inputQUsed; // in MiB
double inputRate;
double procsThroughput; // duration between one element put into input queue and being processed.
double procsTotal; // duration between one element put into input queue and being processed.
double outputThroughput; // the size of dispatched result blocks in bytes
double outputTotal; // the size of dispatched result blocks in bytes
double sinkQuota; // existed quota size for sink task
double sinkDataSize; // sink to dst data size
double procsThroughput; // duration between one element put into input queue and being processed.
double procsTotal; // duration between one element put into input queue and being processed.
double outputThroughput; // the size of dispatched result blocks in bytes
double outputTotal; // the size of dispatched result blocks in bytes
double sinkQuota; // existed quota size for sink task
double sinkDataSize; // sink to dst data size
int64_t startTime;
int64_t startCheckpointId;
int64_t startCheckpointVer;
@ -622,12 +622,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
void streamTaskInputFail(SStreamTask* pTask);
void streamTaskInputFail(SStreamTask* pTask);
bool streamTaskShouldStop(const SStreamTask* pStatus);
bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
bool streamTaskShouldStop(const SStreamTask* pStatus);
bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask);
@ -636,8 +636,8 @@ void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask);
ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask);
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
// stream task sched
bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
@ -737,10 +737,10 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId,
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta);
@ -756,12 +756,13 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
// timer
tmr_h streamTimerGetInstance();
tmr_h streamTimerGetInstance();
// checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp);
int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t downstreamTaskId);
int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId,
int32_t downstreamTaskId);
int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId);
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
@ -786,8 +787,8 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask);
// stream task retrieve related API
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq);
int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq *req);
void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp);
int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* req);
void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
#ifdef __cplusplus
}

3
include/os/osFile.h
View File

@ -72,6 +72,9 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4
#define TD_TMP_FILE_PREFIX "tdengine-"
bool taosCheckAccessFile(const char *pathname, int mode);
int32_t taosLockFile(TdFilePtr pFile);

5
include/util/taoserror.h
View File

@ -388,6 +388,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal
#define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7)
#define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8)
#define TSDB_CODE_MND_TRANS_CONFLICT_COMPACT TAOS_DEF_ERROR_CODE(0, 0x03D9)
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF)
// mnode-mq
@ -457,6 +458,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_DNODE_INVALID_LOCALE TAOS_DEF_ERROR_CODE(0, 0x0426)
#define TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR TAOS_DEF_ERROR_CODE(0, 0x0427)
#define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428)
#define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
@ -827,6 +829,9 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x267A)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x267B)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x267C)
#define TSDB_CODE_PAR_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0x267D)
#define TSDB_CODE_PAR_TBNAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267E)
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

2
include/util/tdef.h
View File

@ -573,7 +573,7 @@ enum {
#define TSDB_CONFIG_OPTION_LEN 32
#define TSDB_CONFIG_VALUE_LEN 64
#define TSDB_CONFIG_SCOPE_LEN 8
#define TSDB_CONFIG_NUMBER 8
#define TSDB_CONFIG_NUMBER 16
#define QUERY_ID_SIZE 20
#define QUERY_OBJ_ID_SIZE 18

14
source/client/inc/clientInt.h
View File

@ -115,10 +115,11 @@ struct SAppInstInfo {
SArray* pQnodeList;
SAppClusterSummary summary;
SList* pConnList; // STscObj linked list
uint64_t clusterId;
int64_t clusterId;
void* pTransporter;
SAppHbMgr* pAppHbMgr;
char* instKey;
SMonitorParas monitorParas;
};
typedef struct SAppInfo {
@ -127,6 +128,7 @@ typedef struct SAppInfo {
int32_t pid;
int32_t numOfThreads;
SHashObj* pInstMap;
SHashObj* pInstMapByClusterId;
TdThreadMutex mutex;
} SAppInfo;
@ -350,7 +352,7 @@ void* createTscObj(const char* user, const char* auth, const char* db, int32_
void destroyTscObj(void* pObj);
STscObj* acquireTscObj(int64_t rid);
int32_t releaseTscObj(int64_t rid);
void destroyAppInst(SAppInstInfo* pAppInfo);
void destroyAppInst(void* pAppInfo);
uint64_t generateRequestId();
@ -403,7 +405,7 @@ void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr);
void destroyAllRequests(SHashObj* pRequests);
void stopAllRequests(SHashObj* pRequests);
SAppInstInfo* getAppInstInfo(const char* clusterKey);
//SAppInstInfo* getAppInstInfo(const char* clusterKey);
// conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
@ -441,10 +443,8 @@ void freeQueryParam(SSyncQueryParam* param);
int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
#endif
void clientSlowQueryMonitorInit(const char* clusterKey);
void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost);
void clientSQLReqMonitorInit(const char* clusterKey);
void slowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost);
enum {
MONITORSQLTYPESELECT = 0,
@ -454,8 +454,6 @@ enum {
void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type);
void clientMonitorClose(const char* clusterKey);
#ifdef __cplusplus
}
#endif

1
source/client/inc/clientStmt.h
View File

@ -219,6 +219,7 @@ const char *stmtErrstr(TAOS_STMT *stmt);
int stmtAffectedRows(TAOS_STMT *stmt);
int stmtAffectedRowsOnce(TAOS_STMT *stmt);
int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
int stmtSetDbName(TAOS_STMT* stmt, const char* dbName);
int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags);
int stmtGetTagFields(TAOS_STMT *stmt, int *nums, TAOS_FIELD_E **fields);

153
source/client/src/clientEnv.c
View File

@ -13,9 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <ttimer.h>
#include "cJSON.h"
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "clientMonitor.h"
#include "functionMgt.h"
#include "os.h"
#include "osSleep.h"
@ -26,6 +29,7 @@
#include "tglobal.h"
#include "thttp.h"
#include "tmsg.h"
#include "tqueue.h"
#include "tref.h"
#include "trpc.h"
#include "tsched.h"
@ -70,6 +74,114 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) {
return TSDB_CODE_SUCCESS;
}
static void concatStrings(SArray *list, char* buf, int size){
int len = 0;
for(int i = 0; i < taosArrayGetSize(list); i++){
char* db = taosArrayGet(list, i);
char* dot = strchr(db, '.');
if (dot != NULL) {
db = dot + 1;
}
if (i != 0){
strcat(buf, ",");
len += 1;
}
int ret = snprintf(buf + len, size - len, "%s", db);
if (ret < 0) {
tscError("snprintf failed, buf:%s, ret:%d", buf, ret);
break;
}
len += ret;
if (len >= size){
tscInfo("dbList is truncated, buf:%s, len:%d", buf, len);
break;
}
}
}
static void generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration){
cJSON* json = cJSON_CreateObject();
if (json == NULL) {
tscError("[monitor] cJSON_CreateObject failed");
return;
}
char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0){
tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId);
}
char startTs[32] = {0};
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start/1000) < 0){
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start/1000);
}
char requestId[32] = {0};
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0){
tscError("failed to generate requestId:%" PRIu64, pRequest->requestId);
}
cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId));
cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs));
cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId));
cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration/1000));
cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code));
cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)));
cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType));
cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows));
if(strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr));
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp;
}else{
cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr));
}
cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user));
cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName));
cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn));
char pid[32] = {0};
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0){
tscError("failed to generate pid:%d", appInfo.pid);
}
cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid));
if(pRequest->dbList != NULL){
char dbList[1024] = {0};
concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1);
cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList));
}else if(pRequest->pDb != NULL){
cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb));
}else{
cJSON_AddItemToObject(json, "db", cJSON_CreateString(""));
}
char* value = cJSON_PrintUnformatted(json);
if(monitorPutData2MonitorQueue(pTscObj->pAppInfo->clusterId, value) < 0){
taosMemoryFree(value);
}
cJSON_Delete(json);
}
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
if (pRequest->pDb != NULL) {
return strcmp(pRequest->pDb, exceptDb) != 0;
}
for (int i = 0; i < taosArrayGetSize(pRequest->dbList); i++) {
char *db = taosArrayGet(pRequest->dbList, i);
char *dot = strchr(db, '.');
if (dot != NULL) {
db = dot + 1;
}
if(strcmp(db, exceptDb) == 0){
return false;
}
}
return true;
}
static void deregisterRequest(SRequestObj *pRequest) {
if (pRequest == NULL) {
tscError("pRequest == NULL");
@ -113,21 +225,27 @@ static void deregisterRequest(SRequestObj *pRequest) {
nodesSimReleaseAllocator(pRequest->allocatorRefId);
}
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT);
} else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE);
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT);
} else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE);
}
}
if (duration >= (tsSlowLogThreshold * 1000000UL)) {
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) {
atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr);
SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
generateWriteSlowLog(pTscObj, pRequest, reqType, duration);
}
}
}
@ -233,14 +351,13 @@ void stopAllRequests(SHashObj *pRequests) {
}
}
void destroyAppInst(SAppInstInfo *pAppInfo) {
void destroyAppInst(void *info) {
SAppInstInfo* pAppInfo = *(SAppInstInfo**)info;
tscDebug("destroy app inst mgr %p", pAppInfo);
taosThreadMutexLock(&appInfo.mutex);
clientMonitorClose(pAppInfo->instKey);
hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
taosThreadMutexUnlock(&appInfo.mutex);
@ -477,13 +594,11 @@ void doDestroyRequest(void *p) {
destorySqlCallbackWrapper(pRequest->pWrapper);
taosMemoryFreeClear(pRequest->msgBuf);
taosMemoryFreeClear(pRequest->pDb);
doFreeReqResultInfo(&pRequest->body.resInfo);
tsem_destroy(&pRequest->body.rspSem);
taosArrayDestroy(pRequest->tableList);
taosArrayDestroy(pRequest->dbList);
taosArrayDestroy(pRequest->targetTableList);
destroyQueryExecRes(&pRequest->body.resInfo.execRes);
@ -492,6 +607,8 @@ void doDestroyRequest(void *p) {
deregisterRequest(pRequest);
}
taosMemoryFreeClear(pRequest->pDb);
taosArrayDestroy(pRequest->dbList);
if (pRequest->body.interParam) {
tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
}
@ -670,7 +787,7 @@ void tscStopCrashReport() {
}
if (atomic_val_compare_exchange_32(&clientStop, 0, 1)) {
tscDebug("hb thread already stopped");
tscDebug("crash report thread already stopped");
return;
}
@ -719,7 +836,8 @@ void taos_init_imp(void) {
appInfo.pid = taosGetPId();
appInfo.startTime = taosGetTimestampMs();
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst);
deltaToUtcInitOnce();
char logDirName[64] = {0};
@ -769,6 +887,7 @@ void taos_init_imp(void) {
taosThreadMutexInit(&appInfo.mutex, NULL);
tscCrashReportInit();
monitorInit();
tscDebug("client is initialized successfully");
}

8
source/client/src/clientHb.c
View File

@ -18,6 +18,7 @@
#include "clientLog.h"
#include "scheduler.h"
#include "trpc.h"
#include "tglobal.h"
typedef struct {
union {
@ -67,7 +68,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
}
static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *batchRsp) {
uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId;
int64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId;
for (int i = 0; i < TARRAY_SIZE(clientHbMgr.appHbMgrs); ++i) {
SAppHbMgr *hbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (!hbMgr || hbMgr->pAppInstInfo->clusterId != clusterId) {
@ -545,6 +546,9 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
}
SAppInstInfo *pInst = pAppHbMgr->pAppInstInfo;
pInst->monitorParas = pRsp.monitorParas;
tscDebug("[monitor] paras from hb, clusterId:%" PRIx64 " monitorParas threshold:%d scope:%d",
pInst->clusterId, pRsp.monitorParas.tsSlowLogThreshold, pRsp.monitorParas.tsSlowLogScope);
if (code != 0) {
pInst->onlineDnodes = pInst->totalDnodes ? 0 : -1;
@ -1129,7 +1133,7 @@ int32_t hbGatherAppInfo(void) {
SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (pAppHbMgr == NULL) continue;
uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId;
int64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId;
SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
if (NULL == pApp) {
memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary));

21
source/client/src/clientImpl.c
View File

@ -147,7 +147,7 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
}
p->pAppHbMgr = appHbMgrInit(p, key);
if (NULL == p->pAppHbMgr) {
destroyAppInst(p);
destroyAppInst(&p);
taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
return NULL;
@ -158,9 +158,6 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port);
pInst = &p;
clientSlowQueryMonitorInit(p->instKey);
clientSQLReqMonitorInit(p->instKey);
} else {
ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
// reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
@ -174,14 +171,14 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
}
SAppInstInfo* getAppInstInfo(const char* clusterKey) {
SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
return *ppAppInstInfo;
} else {
return NULL;
}
}
//SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo;
// } else {
// return NULL;
// }
//}
void freeQueryParam(SSyncQueryParam* param) {
if (param == NULL) return;

6
source/client/src/clientMain.c
View File

@ -17,6 +17,7 @@
#include "clientInt.h"
#include "clientLog.h"
#include "clientStmt.h"
#include "clientMonitor.h"
#include "functionMgt.h"
#include "os.h"
#include "query.h"
@ -55,6 +56,9 @@ void taos_cleanup(void) {
return;
}
monitorClose();
taosHashCleanup(appInfo.pInstMap);
taosHashCleanup(appInfo.pInstMapByClusterId);
tscStopCrashReport();
hbMgrCleanUp();
@ -191,6 +195,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pInfo);
tFreeSGetUserWhiteListRsp(&wlRsp);
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < wlRsp.numWhiteLists; ++i) {
@ -279,7 +284,6 @@ void taos_close_internal(void *taos) {
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
// clientMonitorClose(pTscObj->pAppInfo->instKey);
taosRemoveRef(clientConnRefPool, pTscObj->id);
}

679
source/client/src/clientMonitor.c Normal file
View File

@ -0,0 +1,679 @@
#include "clientMonitor.h"
#include "os.h"
#include "tmisce.h"
#include "ttime.h"
#include "ttimer.h"
#include "tglobal.h"
#include "tqueue.h"
#include "cJSON.h"
#include "clientInt.h"
SRWLatch monitorLock;
void* monitorTimer;
SHashObj* monitorCounterHash;
int32_t slowLogFlag = -1;
int32_t monitorFlag = -1;
tsem2_t monitorSem;
STaosQueue* monitorQueue;
SHashObj* monitorSlowLogHash;
static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size){
if (tsTempDir == NULL) {
return -1;
}
int ret = snprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir);
if (ret < 0){
uError("failed to get tmp path ret:%d", ret);
return ret;
}
return 0;
}
//static void destroyCounter(void* data){
// if (data == NULL) {
// return;
// }
// taos_counter_t* conuter = *(taos_counter_t**)data;
// if(conuter == NULL){
// return;
// }
// taos_counter_destroy(conuter);
//}
static void destroySlowLogClient(void* data){
if (data == NULL) {
return;
}
SlowLogClient* slowLogClient = *(SlowLogClient**)data;
if(slowLogClient == NULL){
return;
}
taosTmrStopA(&(*(SlowLogClient**)data)->timer);
TdFilePtr pFile = slowLogClient->pFile;
if(pFile == NULL){
taosMemoryFree(slowLogClient);
return;
}
taosUnLockFile(pFile);
taosCloseFile(&pFile);
taosMemoryFree(slowLogClient);
}
static void destroyMonitorClient(void* data){
if (data == NULL) {
return;
}
MonitorClient* pMonitor = *(MonitorClient**)data;
if(pMonitor == NULL){
return;
}
taosTmrStopA(&pMonitor->timer);
taosHashCleanup(pMonitor->counters);
taos_collector_registry_destroy(pMonitor->registry);
// taos_collector_destroy(pMonitor->colector);
taosMemoryFree(pMonitor);
}
static SAppInstInfo* getAppInstByClusterId(int64_t clusterId) {
void *p = taosHashGet(appInfo.pInstMapByClusterId, &clusterId, LONG_BYTES);
if(p == NULL){
uError("failed to get app inst, clusterId:%" PRIx64, clusterId);
return NULL;
}
return *(SAppInstInfo**)p;
}
static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) {
if (TSDB_CODE_SUCCESS != code) {
uError("found error in monitorReport send callback, code:%d, please check the network.", code);
}
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
return code;
}
static int32_t sendReport(void* pTransporter, SEpSet *epSet, char* pCont, MONITOR_TYPE type) {
SStatisReq sStatisReq;
sStatisReq.pCont = pCont;
sStatisReq.contLen = strlen(pCont);
sStatisReq.type = type;
int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq);
if (tlen < 0) return 0;
void* buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
uError("sendReport failed, out of memory, len:%d", tlen);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
tSerializeSStatisReq(buf, tlen, &sStatisReq);
SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pInfo == NULL) {
uError("sendReport failed, out of memory send info");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pInfo->fp = monitorReportAsyncCB;
pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_STATIS;
// pInfo->param = taosMemoryMalloc(sizeof(int32_t));
// *(int32_t*)pInfo->param = i;
pInfo->paramFreeFp = taosMemoryFree;
pInfo->requestId = tGenIdPI64();
pInfo->requestObjRefId = 0;
int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pTransporter, epSet, &transporterId, pInfo);
if (code != TSDB_CODE_SUCCESS) {
uError("sendReport failed, code:%d", code);
}
return code;
}
static void monitorReadSendSlowLog(TdFilePtr pFile, void* pTransporter, SEpSet *epSet){
char buf[SLOW_LOG_SEND_SIZE + 1] = {0}; // +1 for \0, for print log
char pCont[SLOW_LOG_SEND_SIZE + 1] = {0}; // +1 for \0, for print log
int32_t offset = 0;
if(taosLSeekFile(pFile, 0, SEEK_SET) < 0){
uError("failed to seek file:%p code: %d", pFile, errno);
return;
}
while(1){
int64_t readSize = taosReadFile(pFile, buf + offset, SLOW_LOG_SEND_SIZE - offset);
if (readSize <= 0) {
uError("failed to read len from file:%p since %s", pFile, terrstr());
return;
}
memset(pCont, 0, sizeof(pCont));
strcat(pCont, "[");
char* string = buf;
for(int i = 0; i < readSize + offset; i++){
if (buf[i] == '\0') {
if (string != buf) strcat(pCont, ",");
strcat(pCont, string);
uDebug("[monitor] monitorReadSendSlowLog slow log:%s", string);
string = buf + i + 1;
}
}
strcat(pCont, "]");
if (pTransporter && pCont != NULL) {
if(sendReport(pTransporter, epSet, pCont, MONITOR_TYPE_SLOW_LOG) != 0){
if(taosLSeekFile(pFile, -readSize, SEEK_CUR) < 0){
uError("failed to seek file:%p code: %d", pFile, errno);
}
uError("failed to send report:%s", pCont);
return;
}
uDebug("[monitor] monitorReadSendSlowLog send slow log to mnode:%s", pCont)
}
if (readSize + offset < SLOW_LOG_SEND_SIZE) {
break;
}
offset = SLOW_LOG_SEND_SIZE - (string - buf);
if(buf != string && offset != 0){
memmove(buf, string, offset);
uDebug("[monitor] monitorReadSendSlowLog left slow log:%s", buf)
}
}
if(taosFtruncateFile(pFile, 0) < 0){
uError("failed to truncate file:%p code: %d", pFile, errno);
}
uDebug("[monitor] monitorReadSendSlowLog send slow log file:%p", pFile);
}
static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet *epSet) {
char ts[50] = {0};
sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL);
if(NULL == pCont) {
uError("generateClusterReport failed, get null content.");
return;
}
if (strlen(pCont) != 0 && sendReport(pTransporter, epSet, pCont, MONITOR_TYPE_COUNTER) == 0) {
taos_collector_registry_clear_batch(registry);
}
taosMemoryFreeClear(pCont);
}
static void reportSendProcess(void* param, void* tmrId) {
taosRLockLatch(&monitorLock);
if (atomic_load_32(&monitorFlag) == 1) {
taosRUnLockLatch(&monitorLock);
return;
}
MonitorClient* pMonitor = (MonitorClient*)param;
SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId);
if(pInst == NULL){
taosRUnLockLatch(&monitorLock);
return;
}
SEpSet ep = getEpSet_s(&pInst->mgmtEp);
generateClusterReport(pMonitor->registry, pInst->pTransporter, &ep);
taosRUnLockLatch(&monitorLock);
taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId);
}
static void sendAllSlowLog(){
void* data = taosHashIterate(monitorSlowLogHash, NULL);
while (data != NULL) {
TdFilePtr pFile = (*(SlowLogClient**)data)->pFile;
if (pFile != NULL){
int64_t clusterId = *(int64_t*)taosHashGetKey(data, NULL);
SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
if(pInst == NULL){
taosHashCancelIterate(monitorSlowLogHash, data);
break;
}
SEpSet ep = getEpSet_s(&pInst->mgmtEp);
monitorReadSendSlowLog(pFile, pInst->pTransporter, &ep);
}
data = taosHashIterate(monitorSlowLogHash, data);
}
uDebug("[monitor] sendAllSlowLog when client close");
}
static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
SAppInstInfo* pInst = getAppInstByClusterId((int64_t)clusterId);
if(pInst == NULL || !pInst->monitorParas.tsEnableMonitor){
uInfo("[monitor] monitor is disabled, skip send slow log");
return;
}
char namePrefix[PATH_MAX] = {0};
if (snprintf(namePrefix, sizeof(namePrefix), "%s%"PRIx64, TD_TMP_FILE_PREFIX, pInst->clusterId) < 0) {
uError("failed to generate slow log file name prefix");
return;
}
taosRLockLatch(&monitorLock);
char tmpPath[PATH_MAX] = {0};
if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0) {
goto END;
}
TdDirPtr pDir = taosOpenDir(tmpPath);
if (pDir == NULL) {
goto END;
}
TdDirEntryPtr de = NULL;
while ((de = taosReadDir(pDir)) != NULL) {
if (taosDirEntryIsDir(de)) {
continue;
}
char *name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 ||
strcmp(name, "..") == 0 ||
strstr(name, namePrefix) == NULL) {
uInfo("skip file:%s, for cluster id:%"PRIx64, name, pInst->clusterId);
continue;
}
char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s%s", tmpPath, name);
TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ);
if (pFile == NULL) {
uError("failed to open file:%s since %s", filename, terrstr());
continue;
}
if (taosLockFile(pFile) < 0) {
uError("failed to lock file:%s since %s, maybe used by other process", filename, terrstr());
taosCloseFile(&pFile);
continue;
}
SEpSet ep = getEpSet_s(&pInst->mgmtEp);
monitorReadSendSlowLog(pFile, pInst->pTransporter, &ep);
taosUnLockFile(pFile);
taosCloseFile(&pFile);
taosRemoveFile(filename);
uDebug("[monitor] send and delete slow log file when reveive connect rsp:%s", filename);
}
taosCloseDir(&pDir);
END:
taosRUnLockLatch(&monitorLock);
}
static void sendAllCounter(){
MonitorClient** ppMonitor = (MonitorClient**)taosHashIterate(monitorCounterHash, NULL);
while (ppMonitor != NULL) {
MonitorClient* pMonitor = *ppMonitor;
if (pMonitor != NULL){
SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId);
if(pInst == NULL){
taosHashCancelIterate(monitorCounterHash, ppMonitor);
break;
}
SEpSet ep = getEpSet_s(&pInst->mgmtEp);
generateClusterReport(pMonitor->registry, pInst->pTransporter, &ep);
}
ppMonitor = taosHashIterate(monitorCounterHash, ppMonitor);
}
}
void monitorCreateClient(int64_t clusterId) {
MonitorClient* pMonitor = NULL;
taosWLockLatch(&monitorLock);
if (taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES) == NULL) {
uInfo("[monitor] monitorCreateClient for %" PRIx64, clusterId);
pMonitor = taosMemoryCalloc(1, sizeof(MonitorClient));
if (pMonitor == NULL) {
uError("failed to create monitor client");
goto fail;
}
pMonitor->clusterId = clusterId;
char clusterKey[32] = {0};
if(snprintf(clusterKey, sizeof(clusterKey), "%"PRId64, clusterId) < 0){
uError("failed to create cluster key");
goto fail;
}
pMonitor->registry = taos_collector_registry_new(clusterKey);
if(pMonitor->registry == NULL){
uError("failed to create registry");
goto fail;
}
pMonitor->colector = taos_collector_new(clusterKey);
if(pMonitor->colector == NULL){
uError("failed to create collector");
goto fail;
}
taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector);
pMonitor->counters = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pMonitor->counters == NULL) {
uError("failed to create monitor counters");
goto fail;
}
// taosHashSetFreeFp(pMonitor->counters, destroyCounter);
if(taosHashPut(monitorCounterHash, &clusterId, LONG_BYTES, &pMonitor, POINTER_BYTES) != 0){
uError("failed to put monitor client to hash");
goto fail;
}
SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
if(pInst == NULL){
uError("failed to get app instance by cluster id");
pMonitor = NULL;
goto fail;
}
pMonitor->timer = taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer);
if(pMonitor->timer == NULL){
uError("failed to start timer");
goto fail;
}
uInfo("[monitor] monitorCreateClient for %"PRIx64 "finished %p.", clusterId, pMonitor);
}
taosWUnLockLatch(&monitorLock);
if (-1 != atomic_val_compare_exchange_32(&monitorFlag, -1, 0)) {
uDebug("[monitor] monitorFlag already is 0");
}
return;
fail:
destroyMonitorClient(&pMonitor);
taosWUnLockLatch(&monitorLock);
}
void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count, const char** label_keys) {
taosWLockLatch(&monitorLock);
MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES);
if (ppMonitor == NULL || *ppMonitor == NULL) {
uError("failed to get monitor client");
goto end;
}
taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys);
if (newCounter == NULL)
return;
MonitorClient* pMonitor = *ppMonitor;
taos_collector_add_metric(pMonitor->colector, newCounter);
if(taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0){
uError("failed to put counter to monitor");
taos_counter_destroy(newCounter);
goto end;
}
uInfo("[monitor] monitorCreateClientCounter %"PRIx64"(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, newCounter);
end:
taosWUnLockLatch(&monitorLock);
}
void monitorCounterInc(int64_t clusterId, const char* counterName, const char** label_values) {
taosWLockLatch(&monitorLock);
MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES);
if (ppMonitor == NULL || *ppMonitor == NULL) {
uError("monitorCounterInc not found pMonitor %"PRId64, clusterId);
goto end;
}
MonitorClient* pMonitor = *ppMonitor;
taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName));
if (ppCounter == NULL || *ppCounter != NULL) {
uError("monitorCounterInc not found pCounter %"PRIx64":%s.", clusterId, counterName);
goto end;
}
taos_counter_inc(*ppCounter, label_values);
uInfo("[monitor] monitorCounterInc %"PRIx64"(%p):%s", pMonitor->clusterId, pMonitor, counterName);
end:
taosWUnLockLatch(&monitorLock);
}
const char* monitorResultStr(SQL_RESULT_CODE code) {
static const char* result_state[] = {"Success", "Failed", "Cancel"};
return result_state[code];
}
static void monitorFreeSlowLogData(MonitorSlowLogData* pData) {
if (pData == NULL) {
return;
}
taosMemoryFree(pData->value);
}
static void monitorThreadFuncUnexpectedStopped(void) { atomic_store_32(&slowLogFlag, -1); }
static void reportSlowLog(void* param, void* tmrId) {
taosRLockLatch(&monitorLock);
if (atomic_load_32(&monitorFlag) == 1) {
taosRUnLockLatch(&monitorLock);
return;
}
SAppInstInfo* pInst = getAppInstByClusterId((int64_t)param);
if(pInst == NULL){
uError("failed to get app inst, clusterId:%"PRIx64, (int64_t)param);
taosRUnLockLatch(&monitorLock);
return;
}
void* tmp = taosHashGet(monitorSlowLogHash, &param, LONG_BYTES);
if(tmp == NULL){
uError("failed to get file inst, clusterId:%"PRIx64, (int64_t)param);
taosRUnLockLatch(&monitorLock);
return;
}
SEpSet ep = getEpSet_s(&pInst->mgmtEp);
monitorReadSendSlowLog((*(SlowLogClient**)tmp)->pFile, pInst->pTransporter, &ep);
taosRUnLockLatch(&monitorLock);
taosTmrReset(reportSlowLog, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId);
}
static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpPath){
taosWLockLatch(&monitorLock);
TdFilePtr pFile = NULL;
void* tmp = taosHashGet(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES);
if (tmp == NULL){
char path[PATH_MAX] = {0};
char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRIx64, slowLogData->clusterId) < 0){
uError("failed to generate clusterId:%" PRIx64, slowLogData->clusterId);
goto FAILED;
}
taosGetTmpfilePath(tmpPath, clusterId, path);
uInfo("[monitor] create slow log file:%s", path);
pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to open file:%s since %s", path, terrstr());
goto FAILED;
}
SlowLogClient *pClient = taosMemoryCalloc(1, sizeof(SlowLogClient));
if (pClient == NULL){
uError("failed to allocate memory for slow log client");
taosCloseFile(&pFile);
goto FAILED;
}
pClient->pFile = pFile;
if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0){
uError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId);
taosCloseFile(&pFile);
taosMemoryFree(pClient);
goto FAILED;
}
if(taosLockFile(pFile) < 0){
uError("failed to lock file:%p since %s", pFile, terrstr());
goto FAILED;
}
SAppInstInfo* pInst = getAppInstByClusterId(slowLogData->clusterId);
if(pInst == NULL){
uError("failed to get app instance by clusterId:%" PRId64, slowLogData->clusterId);
goto FAILED;
}
pClient->timer = taosTmrStart(reportSlowLog, pInst->monitorParas.tsMonitorInterval * 1000, (void*)slowLogData->clusterId, monitorTimer);
}else{
pFile = (*(SlowLogClient**)tmp)->pFile;
}
if (taosWriteFile(pFile, slowLogData->value, strlen(slowLogData->value) + 1) < 0){
uError("failed to write len to file:%p since %s", pFile, terrstr());
}
uDebug("[monitor] write slow log to file:%p, clusterId:%"PRIx64, pFile, slowLogData->clusterId);
FAILED:
taosWUnLockLatch(&monitorLock);
}
static void* monitorThreadFunc(void *param){
setThreadName("client-monitor-slowlog");
#ifdef WINDOWS
if (taosCheckCurrentInDll()) {
atexit(monitorThreadFuncUnexpectedStopped);
}
#endif
if (-1 != atomic_val_compare_exchange_32(&slowLogFlag, -1, 0)) {
return NULL;
}
char tmpPath[PATH_MAX] = {0};
if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0){
return NULL;
}
if (taosMulModeMkDir(tmpPath, 0777, true) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
printf("failed to create dir:%s since %s", tmpPath, terrstr());
return NULL;
}
if (tsem2_init(&monitorSem, 0, 0) != 0) {
uError("sem init error since %s", terrstr());
return NULL;
}
monitorQueue = taosOpenQueue();
if(monitorQueue == NULL){
uError("open queue error since %s", terrstr());
return NULL;
}
uDebug("monitorThreadFunc start");
while (1) {
if (slowLogFlag > 0) break;
MonitorSlowLogData* slowLogData = NULL;
taosReadQitem(monitorQueue, (void**)&slowLogData);
if (slowLogData != NULL) {
uDebug("[monitor] read slow log data from queue, clusterId:%" PRIx64 " value:%s", slowLogData->clusterId, slowLogData->value);
if (slowLogData->value == NULL){
monitorSendAllSlowLogFromTempDir(slowLogData->clusterId);
}else{
monitorWriteSlowLog2File(slowLogData, tmpPath);
}
}
monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData);
tsem2_timewait(&monitorSem, 500);
}
taosCloseQueue(monitorQueue);
tsem2_destroy(&monitorSem);
slowLogFlag = -2;
return NULL;
}
static int32_t tscMonitortInit() {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
TdThread monitorThread;
if (taosThreadCreate(&monitorThread, &thAttr, monitorThreadFunc, NULL) != 0) {
uError("failed to create monitor thread since %s", strerror(errno));
return -1;
}
taosThreadAttrDestroy(&thAttr);
return 0;
}
static void tscMonitorStop() {
if (atomic_val_compare_exchange_32(&slowLogFlag, 0, 1)) {
uDebug("monitor thread already stopped");
return;
}
while (atomic_load_32(&slowLogFlag) > 0) {
taosMsleep(100);
}
}
void monitorInit() {
uInfo("[monitor] tscMonitor init");
monitorCounterHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorCounterHash == NULL) {
uError("failed to create monitorCounterHash");
}
taosHashSetFreeFp(monitorCounterHash, destroyMonitorClient);
monitorSlowLogHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorSlowLogHash == NULL) {
uError("failed to create monitorSlowLogHash");
}
taosHashSetFreeFp(monitorSlowLogHash, destroySlowLogClient);
monitorTimer = taosTmrInit(0, 0, 0, "MONITOR");
if (monitorTimer == NULL) {
uError("failed to create monitor timer");
}
taosInitRWLatch(&monitorLock);
tscMonitortInit();
}
void monitorClose() {
uInfo("[monitor] tscMonitor close");
taosWLockLatch(&monitorLock);
if (atomic_val_compare_exchange_32(&monitorFlag, 0, 1)) {
uDebug("[monitor] monitorFlag is not 0");
}
tscMonitorStop();
sendAllSlowLog();
sendAllCounter();
taosHashCleanup(monitorCounterHash);
taosHashCleanup(monitorSlowLogHash);
taosTmrCleanUp(monitorTimer);
taosWUnLockLatch(&monitorLock);
}
int32_t monitorPutData2MonitorQueue(int64_t clusterId, char* value){
MonitorSlowLogData* slowLogData = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0);
if (slowLogData == NULL) {
uError("[monitor] failed to allocate slow log data");
return -1;
}
slowLogData->clusterId = clusterId;
slowLogData->value = value;
uDebug("[monitor] write slow log to queue, clusterId:%"PRIx64 " value:%s", slowLogData->clusterId, slowLogData->value);
while (monitorQueue == NULL) {
taosMsleep(100);
}
if (taosWriteQitem(monitorQueue, slowLogData) == 0){
tsem2_post(&monitorSem);
}else{
monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData);
}
return 0;
}

29
source/client/src/slowQueryMonitor.c → source/client/src/clientMonitorSlow.c
View File

@ -21,7 +21,6 @@ const char* slowQueryName = "taos_slow_sql:count";
const char* slowQueryHelp = "slow query log when cost over than config duration";
const int slowQueryLabelCount = 4;
const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"};
static const char* defaultClusterID = "";
const int64_t usInSeconds = 1000 * 1000;
const int64_t msInMinutes = 60 * 1000;
@ -39,21 +38,21 @@ static const char* getSlowQueryLableCostDesc(int64_t cost) {
return "0-3s";
}
void clientSlowQueryMonitorInit(const char* clusterKey) {
if (!tsEnableMonitor) return;
SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels);
void monitorClientSlowQueryInit(int64_t clusterid) {
monitorCreateClient(clusterid);
monitorCreateClientCounter(clusterid, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels);
}
void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) {
const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)};
taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues);
void clientSlowQueryLog(int64_t clusterId, const char* user, SQL_RESULT_CODE result, int32_t cost) {
char clusterIdStr[32] = {0};
if (snprintf(clusterIdStr, sizeof(clusterIdStr), "%" PRId64, clusterId) < 0){
uError("failed to generate clusterId:%" PRId64, clusterId);
}
const char* slowQueryLabelValues[] = {clusterIdStr, user, monitorResultStr(result), getSlowQueryLableCostDesc(cost)};
monitorCounterInc(clusterId, slowQueryName, slowQueryLabelValues);
}
void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
if (!tsEnableMonitor) return;
void slowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
if (TSDB_CODE_SUCCESS != code) {
result = SQL_RESULT_FAILED;
@ -66,12 +65,12 @@ void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
STscObj* pTscObj = acquireTscObj(rid);
if (pTscObj != NULL) {
if(pTscObj->pAppInfo == NULL) {
tscLog("SlowQueryLog, not found pAppInfo");
tscLog("slowQueryLog, not found pAppInfo");
} else {
clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost);
clientSlowQueryLog(pTscObj->pAppInfo->clusterId, pTscObj->user, result, cost);
}
releaseTscObj(rid);
} else {
tscLog("SlowQueryLog, not found rid");
tscLog("slowQueryLog, not found rid");
}
}

29
source/client/src/clientSqlMonitor.c → source/client/src/clientMonitorSql.c
View File

@ -22,17 +22,12 @@ const char* selectMonitorHelp = "count for select sql";
const int selectMonitorLabelCount = 4;
const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"};
static const char* defaultClusterID = "";
void clientSQLReqMonitorInit(const char* clusterKey) {
if (!tsEnableMonitor) return;
SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels);
void monitorClientSQLReqInit(int64_t clusterId) {
monitorCreateClient(clusterId);
monitorCreateClientCounter(clusterId, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels);
}
void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) {
void clientSQLReqLog(int64_t clusterId, const char* user, SQL_RESULT_CODE result, int8_t type) {
const char* typeStr;
switch (type) {
case MONITORSQLTYPEDELETE:
@ -45,12 +40,15 @@ void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE r
typeStr = "select";
break;
}
const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)};
taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues);
char clusterIdStr[32] = {0};
if (snprintf(clusterIdStr, sizeof(clusterIdStr), "%" PRId64, clusterId) < 0){
uError("failed to generate clusterId:%" PRId64, clusterId);
}
const char* selectMonitorLabelValues[] = {clusterIdStr, typeStr, user, monitorResultStr(result)};
monitorCounterInc(clusterId, selectMonitorName, selectMonitorLabelValues);
}
void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) {
if (!tsEnableMonitor) return;
SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
if (TSDB_CODE_SUCCESS != code) {
result = SQL_RESULT_FAILED;
@ -65,15 +63,10 @@ void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) {
if (pTscObj->pAppInfo == NULL) {
tscLog("sqlReqLog, not found pAppInfo");
} else {
clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type);
clientSQLReqLog(pTscObj->pAppInfo->clusterId, pTscObj->user, result, type);
}
releaseTscObj(rid);
} else {
tscLog("sqlReqLog, not found rid");
}
}
void clientMonitorClose(const char* clusterKey) {
tscLog("clientMonitorClose, key:%s", clusterKey);
clusterMonitorClose(clusterKey);
}

15
source/client/src/clientMsgHandler.c
View File

@ -15,6 +15,7 @@
#include "catalog.h"
#include "clientInt.h"
#include "clientMonitor.h"
#include "clientLog.h"
#include "cmdnodes.h"
#include "os.h"
@ -140,6 +141,9 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
// update the appInstInfo
pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
pTscObj->pAppInfo->monitorParas = connectRsp.monitorParas;
tscDebug("[monitor] paras from connect rsp, clusterId:%" PRIx64 " monitorParas threshold:%d scope:%d",
connectRsp.clusterId, connectRsp.monitorParas.tsSlowLogThreshold, connectRsp.monitorParas.tsSlowLogScope);
lastClusterId = connectRsp.clusterId;
pTscObj->connType = connectRsp.connType;
@ -147,6 +151,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
pTscObj->authVer = connectRsp.authVer;
pTscObj->whiteListInfo.ver = connectRsp.whiteListVer;
if(taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL){
if(taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, POINTER_BYTES) != 0){
tscError("failed to put appInfo into appInfo.pInstMapByClusterId");
}
monitorPutData2MonitorQueue(pTscObj->pAppInfo->clusterId, NULL);
monitorClientSlowQueryInit(connectRsp.clusterId);
monitorClientSQLReqInit(connectRsp.clusterId);
}
taosThreadMutexLock(&clientHbMgr.lock);
SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx);
if (pAppHbMgr) {
@ -233,7 +246,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
struct SCatalog* pCatalog = NULL;
if (usedbRsp.vgVersion >= 0) { // cached in local
uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId;
int64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId;
int32_t code1 = catalogGetHandle(clusterId, &pCatalog);
if (code1 != TSDB_CODE_SUCCESS) {
tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId,

2
source/client/src/clientSml.c
View File

@ -1851,7 +1851,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
}
void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) {
if (tsSlowLogScope & SLOW_LOG_TYPE_INSERT) {
if (request->pTscObj->pAppInfo->monitorParas.tsSlowLogScope & SLOW_LOG_TYPE_INSERT) {
int32_t len = 0;
int32_t rlen = 0;
char *p = NULL;

224
source/client/src/clientStmt.c
View File

@ -17,11 +17,11 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void**
*pBuf = pTblBuf->pCurBuff;
pTblBuf->buffOffset = pTblBuf->buffUnit;
} else {
void *buff = taosMemoryMalloc(pTblBuf->buffSize);
void* buff = taosMemoryMalloc(pTblBuf->buffSize);
if (NULL == buff) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pTblBuf->pBufList, &buff);
pTblBuf->buffIdx++;
@ -33,18 +33,18 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void**
return TSDB_CODE_SUCCESS;
}
bool stmtDequeue(STscStmt* pStmt, SStmtQNode **param) {
bool stmtDequeue(STscStmt* pStmt, SStmtQNode** param) {
while (0 == atomic_load_64(&pStmt->queue.qRemainNum)) {
taosUsleep(1);
return false;
}
SStmtQNode *orig = pStmt->queue.head;
SStmtQNode* orig = pStmt->queue.head;
SStmtQNode *node = pStmt->queue.head->next;
SStmtQNode* node = pStmt->queue.head->next;
pStmt->queue.head = pStmt->queue.head->next;
//taosMemoryFreeClear(orig);
// taosMemoryFreeClear(orig);
*param = node;
@ -61,7 +61,6 @@ void stmtEnqueue(STscStmt* pStmt, SStmtQNode* param) {
atomic_add_fetch_64(&pStmt->queue.qRemainNum, 1);
}
static int32_t stmtCreateRequest(STscStmt* pStmt) {
int32_t code = 0;
@ -93,7 +92,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
switch (newStatus) {
case STMT_PREPARE:
pStmt->errCode = 0;
pStmt->errCode = 0;
break;
case STMT_SETTBNAME:
if (STMT_STATUS_EQ(INIT)) {
@ -265,7 +264,7 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa
*pVgHash = pStmt->sql.pVgHash;
pStmt->sql.pVgHash = NULL;
*pBlockHash = pStmt->exec.pBlockHash;
pStmt->exec.pBlockHash = NULL;
@ -325,7 +324,7 @@ int32_t stmtParseSql(STscStmt* pStmt) {
pStmt->stat.parseSqlNum++;
STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));
pStmt->sql.siInfo.pQuery = pStmt->sql.pQuery;
pStmt->bInfo.needParse = false;
if (pStmt->sql.pQuery->pRoot && 0 == pStmt->sql.type) {
@ -338,15 +337,16 @@ int32_t stmtParseSql(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS;
}
STableDataCxt** pSrc = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
STableDataCxt** pSrc =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pSrc || NULL == *pSrc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
STableDataCxt* pTableCtx = *pSrc;
if (pStmt->sql.stbInterlaceMode) {
int16_t lastIdx = -1;
int16_t lastIdx = -1;
for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) {
if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) {
pStmt->sql.stbInterlaceMode = false;
@ -381,7 +381,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
qDestroyBoundColInfo(pStmt->bInfo.boundTags);
taosMemoryFreeClear(pStmt->bInfo.boundTags);
}
pStmt->bInfo.stbFName[0] = 0;;
pStmt->bInfo.stbFName[0] = 0;
;
return TSDB_CODE_SUCCESS;
}
@ -457,12 +458,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) {
return TSDB_CODE_SUCCESS;
}
void stmtFreeTbBuf(void *buf) {
void stmtFreeTbBuf(void* buf) {
void* pBuf = *(void**)buf;
taosMemoryFree(pBuf);
}
void stmtFreeTbCols(void *buf) {
void stmtFreeTbCols(void* buf) {
SArray* pCols = *(SArray**)buf;
taosArrayDestroy(pCols);
}
@ -505,19 +506,17 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
memset(&pStmt->sql, 0, sizeof(pStmt->sql));
pStmt->sql.siInfo.tableColsReady = true;
STMT_DLOG_E("end to free SQL info");
return TSDB_CODE_SUCCESS;
}
int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) {
if (*vgId >= 0 && taosHashGet(pStmt->sql.pVgHash, (const char*)vgId, sizeof(*vgId))) {
return TSDB_CODE_SUCCESS;
}
SVgroupInfo vgInfo = {0};
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
.requestId = pStmt->exec.pRequest->requestId,
@ -528,17 +527,17 @@ int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) {
if (TSDB_CODE_SUCCESS != code) {
return code;
}
code = taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo));
code =
taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo));
if (TSDB_CODE_SUCCESS != code) {
return code;
}
*vgId = vgInfo.vgId;
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_SUCCESS;
}
int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid,
uint64_t suid, int32_t vgId) {
@ -554,7 +553,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx) {
pStmt->bInfo.needParse = false;
pStmt->bInfo.inExecCache = false;
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
pStmt->bInfo.needParse = true;
@ -589,7 +588,6 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS;
}
if (pStmt->sql.autoCreateTbl) {
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid));
if (pCache) {
@ -615,8 +613,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
}
uint64_t uid, suid;
int32_t vgId;
int8_t tableType;
int32_t vgId;
int8_t tableType;
STableMeta* pTableMeta = NULL;
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
@ -638,12 +636,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
uid = pTableMeta->uid;
suid = pTableMeta->suid;
tableType = pTableMeta->tableType;
tableType = pTableMeta->tableType;
pStmt->bInfo.tbVgId = pTableMeta->vgId;
vgId = pTableMeta->vgId;
taosMemoryFree(pTableMeta);
uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid;
if (uid == pStmt->bInfo.tbUid) {
@ -675,7 +673,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS;
}
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid));
if (pCache) {
pStmt->bInfo.needParse = false;
@ -720,7 +718,6 @@ int32_t stmtResetStmt(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS;
}
int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) {
SStmtQNode* pParam = (SStmtQNode*)param;
@ -729,20 +726,20 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) {
SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i);
*p = taosArrayInit(20, POINTER_BYTES);
}
atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true);
} else {
STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, &pStmt->sql.siInfo));
STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock,
&pStmt->sql.siInfo));
//taosMemoryFree(pParam->pTbData);
// taosMemoryFree(pParam->pTbData);
atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1);
}
return TSDB_CODE_SUCCESS;
}
void *stmtBindThreadFunc(void *param) {
void* stmtBindThreadFunc(void* param) {
setThreadName("stmtBind");
qInfo("stmt bind thread started");
@ -750,15 +747,15 @@ void *stmtBindThreadFunc(void *param) {
STscStmt* pStmt = (STscStmt*)param;
while (true) {
if (atomic_load_8((int8_t *)&pStmt->queue.stopQueue)) {
if (atomic_load_8((int8_t*)&pStmt->queue.stopQueue)) {
break;
}
SStmtQNode *asyncParam = NULL;
SStmtQNode* asyncParam = NULL;
if (!stmtDequeue(pStmt, &asyncParam)) {
continue;
}
stmtAsyncOutput(pStmt, asyncParam);
}
@ -767,7 +764,6 @@ void *stmtBindThreadFunc(void *param) {
return NULL;
}
int32_t stmtStartBindThread(STscStmt* pStmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
@ -798,7 +794,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) {
if (NULL == pTblBuf->pBufList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
void *buff = taosMemoryMalloc(pTblBuf->buffSize);
void* buff = taosMemoryMalloc(pTblBuf->buffSize);
if (NULL == buff) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -815,7 +811,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) {
TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) {
STscObj* pObj = (STscObj*)taos;
STscStmt* pStmt = NULL;
int32_t code = 0;
int32_t code = 0;
pStmt = taosMemoryCalloc(1, sizeof(STscStmt));
if (NULL == pStmt) {
@ -900,6 +896,12 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
pStmt->sql.sqlLen = length;
pStmt->sql.stbInterlaceMode = pStmt->stbInterlaceMode;
char* dbName = NULL;
if (qParseDbName(sql, length, &dbName)) {
stmtSetDbName(stmt, dbName);
taosMemoryFreeClear(dbName);
}
return TSDB_CODE_SUCCESS;
}
@ -909,23 +911,38 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) {
return TSDB_CODE_OUT_OF_MEMORY;
}
STableDataCxt* pDst = NULL;
STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true));
pStmt->sql.siInfo.pDataCtx = pDst;
SArray* pTblCols = NULL;
for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) {
pTblCols = taosArrayInit(20, POINTER_BYTES);
if (NULL == pTblCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols);
}
pStmt->sql.siInfo.boundTags = pStmt->bInfo.boundTags;
return TSDB_CODE_SUCCESS;
}
int stmtSetDbName(TAOS_STMT* stmt, const char* dbName) {
STscStmt *pStmt = (STscStmt *) stmt;
STMT_DLOG("start to set dbName: %s", dbName);
STMT_ERR_RET(stmtCreateRequest(pStmt));
// The SQL statement specifies a database name, overriding the previously specified database
taosMemoryFreeClear(pStmt->exec.pRequest->pDb);
pStmt->exec.pRequest->pDb = taosStrdup(dbName);
if (pStmt->exec.pRequest->pDb == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
@ -951,13 +968,13 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName);
STMT_ERR_RET(stmtGetFromCache(pStmt));
if (pStmt->bInfo.needParse) {
strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1);
pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0;
STMT_ERR_RET(stmtParseSql(pStmt));
}
} else {
@ -1032,7 +1049,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
if (pStmt->sql.stbInterlaceMode) {
pDataBlock = &pStmt->sql.siInfo.pDataCtx;
} else {
pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
@ -1050,7 +1068,7 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) {
if (pStmt->exec.smInfo.pColIdx >= STMT_COL_BUF_SIZE) {
pStmt->exec.smInfo.pColIdx = 0;
}
if ((pStmt->exec.smInfo.pColIdx + 1) == atomic_load_32(&pStmt->exec.smInfo.pColFreeIdx)) {
taosUsleep(1);
continue;
@ -1064,38 +1082,40 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) {
int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) {
if (NULL == pStmt->sql.siInfo.pVgroupHash) {
pStmt->sql.siInfo.pVgroupHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
pStmt->sql.siInfo.pVgroupHash =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
}
if (NULL == pStmt->sql.siInfo.pVgroupList) {
pStmt->sql.siInfo.pVgroupList = taosArrayInit(64, POINTER_BYTES);
}
if (NULL == pStmt->sql.siInfo.pRequest) {
STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, (SRequestObj**)&pStmt->sql.siInfo.pRequest,
pStmt->reqid));
STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false,
(SRequestObj**)&pStmt->sql.siInfo.pRequest, pStmt->reqid));
if (pStmt->reqid != 0) {
pStmt->reqid++;
}
pStmt->exec.pRequest->syncQuery = true;
pStmt->sql.siInfo.requestId = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->requestId;
pStmt->sql.siInfo.requestSelf = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->self;
}
if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] && 0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) {
if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] &&
0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) {
pStmt->sql.siInfo.tbFromHash = true;
}
if (0 == pStmt->sql.siInfo.firstName[0]) {
strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName);
}
param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash;
param->next = NULL;
atomic_add_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1);
stmtEnqueue(pStmt, param);
return TSDB_CODE_SUCCESS;
@ -1113,7 +1133,7 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray**
if (NULL == pTblCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols);
}
}
@ -1124,10 +1144,10 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray**
int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STscStmt* pStmt = (STscStmt*)stmt;
int32_t code = 0;
int32_t code = 0;
int64_t startUs = taosGetTimestampUs();
STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
@ -1210,7 +1230,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
}
int64_t startUs2 = taosGetTimestampUs();
pStmt->stat.bindDataUs1 += startUs2 - startUs;
pStmt->stat.bindDataUs1 += startUs2 - startUs;
SStmtQNode* param = NULL;
if (pStmt->sql.stbInterlaceMode) {
@ -1218,25 +1238,27 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STMT_ERR_RET(stmtGetTableColsFromCache(pStmt, &param->tblData.aCol));
taosArrayClear(param->tblData.aCol);
//param->tblData.aCol = taosArrayInit(20, POINTER_BYTES);
// param->tblData.aCol = taosArrayInit(20, POINTER_BYTES);
param->restoreTbCols = false;
strcpy(param->tblData.tbName, pStmt->bInfo.tbName);
}
int64_t startUs3 = taosGetTimestampUs();
pStmt->stat.bindDataUs2 += startUs3 - startUs2;
pStmt->stat.bindDataUs2 += startUs3 - startUs2;
SArray* pCols = pStmt->sql.stbInterlaceMode ? param->tblData.aCol : (*pDataBlock)->pData->aCol;
if (colIdx < 0) {
if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo);
code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo);
} else {
code = qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
code =
qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
}
if (code) {
tscError("qBindStmtColsValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code);
@ -1246,7 +1268,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
tscError("bind single column not allowed in stb insert mode");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) {
tscError("bind column index not in sequence");
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
@ -1267,13 +1289,13 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
}
int64_t startUs4 = taosGetTimestampUs();
pStmt->stat.bindDataUs3 += startUs4 - startUs3;
pStmt->stat.bindDataUs3 += startUs4 - startUs3;
if (pStmt->sql.stbInterlaceMode) {
STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param));
}
pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4;
pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4;
return TSDB_CODE_SUCCESS;
}
@ -1292,12 +1314,12 @@ int stmtAddBatch(TAOS_STMT* stmt) {
pStmt->stat.addBatchUs += startUs2 - startUs;
pStmt->sql.siInfo.tableColsReady = false;
SStmtQNode* param = NULL;
STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)&param));
param->restoreTbCols = true;
param->next = NULL;
stmtEnqueue(pStmt, param);
return TSDB_CODE_SUCCESS;
@ -1372,7 +1394,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta);
pStmt->stat.ctgGetTbMetaNum++;
taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
@ -1402,13 +1424,14 @@ int stmtStaticModeExec(TAOS_STMT* stmt) {
if (pStmt->sql.staticMode) {
return TSDB_CODE_TSC_STMT_API_ERROR;
}
STMT_DLOG_E("start to exec");
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList, pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum));
STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList,
pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum));
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL);
if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) {
@ -1430,7 +1453,7 @@ int stmtStaticModeExec(TAOS_STMT* stmt) {
_return:
stmtCleanExecInfo(pStmt, (code ? false : true), false);
tFreeSSubmitRsp(pRsp);
++pStmt->sql.runTimes;
@ -1459,7 +1482,7 @@ int stmtExec(TAOS_STMT* stmt) {
taosUsleep(1);
}
pStmt->stat.execWaitUs += taosGetTimestampUs() - startTs;
STMT_ERR_RET(qBuildStmtFinOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->sql.siInfo.pVgroupList));
taosHashCleanup(pStmt->sql.siInfo.pVgroupHash);
pStmt->sql.siInfo.pVgroupHash = NULL;
@ -1472,7 +1495,7 @@ int stmtExec(TAOS_STMT* stmt) {
STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash));
}
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL);
}
@ -1499,7 +1522,7 @@ _return:
}
stmtCleanExecInfo(pStmt, (code ? false : true), false);
tFreeSSubmitRsp(pRsp);
++pStmt->sql.runTimes;
@ -1522,13 +1545,16 @@ int stmtClose(TAOS_STMT* stmt) {
pStmt->bindThreadInUse = false;
}
STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 ", parseSqlNum=>%" PRId64
", pStmt->stat.bindDataNum=>%" PRId64 ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u"
", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64 ", execWaitUs:%" PRId64 ", execUseUs:%" PRId64,
pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo, pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum,
pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND], pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE],
pStmt->stat.setTbNameUs, pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4,
pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs);
STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64
", parseSqlNum=>%" PRId64 ", pStmt->stat.bindDataNum=>%" PRId64
", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u"
", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64
", execWaitUs:%" PRId64 ", execUseUs:%" PRId64,
pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo,
pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum, pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND],
pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE], pStmt->stat.setTbNameUs,
pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4,
pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs);
stmtCleanSQLInfo(pStmt);
taosMemoryFree(stmt);
@ -1567,9 +1593,9 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
}
int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0;
int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get tag fields");
@ -1600,14 +1626,14 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
_return:
pStmt->errCode = preCode;
return code;
}
int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0;
int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get col fields");
@ -1661,7 +1687,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
}

200
source/client/src/clientTmq.c
View File

@ -107,7 +107,7 @@ struct tmq_t {
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
tsem2_t rspSem;
tsem2_t rspSem;
};
typedef struct SAskEpInfo {
@ -681,7 +681,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName,
numOfVgroups);
numOfVgroups);
for (int32_t j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
@ -703,19 +703,19 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
}
tscDebug("consumer:0x%" PRIx64
" topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups);
" topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups);
tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset);
} else {
tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
}
}
}
taosRUnLockLatch(&tmq->lock);
tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1,
numOfTopics);
numOfTopics);
// request is sent
if (pParamSet->waitingRspNum != 1) {
@ -769,34 +769,35 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
}
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
if (pMsg) {
SMqHbRsp rsp = {0};
tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
if (pMsg == NULL) {
return code;
}
SMqHbRsp rsp = {0};
tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq != NULL) {
taosWLockLatch(&tmq->lock);
for (int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++) {
STopicPrivilege* privilege = taosArrayGet(rsp.topicPrivileges, i);
if (privilege->noPrivilege == 1) {
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
for (int32_t j = 0; j < topicNumCur; j++) {
SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
if (strcmp(pTopicCur->topicName, privilege->topic) == 0) {
tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic);
pTopicCur->noPrivilege = 1;
}
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq != NULL) {
taosWLockLatch(&tmq->lock);
for (int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++) {
STopicPrivilege* privilege = taosArrayGet(rsp.topicPrivileges, i);
if (privilege->noPrivilege == 1) {
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
for (int32_t j = 0; j < topicNumCur; j++) {
SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
if (strcmp(pTopicCur->topicName, privilege->topic) == 0) {
tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic);
pTopicCur->noPrivilege = 1;
}
}
}
taosWUnLockLatch(&tmq->lock);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDestroySMqHbRsp(&rsp);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
taosWUnLockLatch(&tmq->lock);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDestroySMqHbRsp(&rsp);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
return 0;
}
@ -830,7 +831,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset);
tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64,
tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows);
tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows);
}
}
taosRUnLockLatch(&tmq->lock);
@ -955,7 +956,7 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
}
}
}
void tmqClearUnhandleMsg(tmq_t* tmq) {
@ -984,10 +985,16 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
}
int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) {
return code;
}
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code;
taosMemoryFree(pMsg->pEpSet);
if(pMsg){
taosMemoryFree(pMsg->pEpSet);
}
tsem_post(&pParam->rspSem);
return 0;
}
@ -1021,17 +1028,16 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
taosSsleep(2); // sleep 2s for hb to send offset and rows to server
tmq_list_t* lst = tmq_list_new();
int32_t rsp = tmq_subscribe(tmq, lst);
int32_t rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst);
return rsp;
}
static void freeClientVg(void* param){
static void freeClientVg(void* param) {
SMqClientVg* pVg = param;
tOffsetDestroy(&pVg->offsetInfo.endOffset);
tOffsetDestroy(&pVg->offsetInfo.beginOffset);
tOffsetDestroy(&pVg->offsetInfo.committedOffset);
}
static void freeClientVgImpl(void* param) {
SMqClientTopic* pTopic = param;
@ -1282,8 +1288,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
int32_t retryCnt = 0;
while ((code = syncAskEp(tmq)) != 0) {
if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, strerror(code));
if(code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s",
tmq->consumerId, strerror(code));
if (code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
code = 0;
}
goto FAIL;
@ -1379,7 +1386,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
goto END;
}
if(pMsg->pData == NULL){
if (pMsg->pData == NULL) {
tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto END;
@ -1407,7 +1414,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){
if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) {
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1423,7 +1430,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0){
if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) {
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1434,7 +1441,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){
if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) {
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1445,7 +1452,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){
if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) {
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1529,11 +1536,11 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.walVerBegin = -1;
clientVg.offsetInfo.walVerEnd = -1;
clientVg.seekUpdated = false;
if(pInfo) {
if (pInfo) {
tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset);
tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset);
tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset);
}else{
} else {
clientVg.offsetInfo.endOffset = offsetNew;
clientVg.offsetInfo.committedOffset = offsetNew;
clientVg.offsetInfo.beginOffset = offsetNew;
@ -1548,7 +1555,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) {
tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId,
tmq->epoch, epoch, topicNumGet);
tmq->epoch, epoch, topicNumGet);
if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
}
@ -1642,7 +1649,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj));
if(pRspObj == NULL) {
if (pRspObj == NULL) {
return NULL;
}
pRspObj->resType = RES_TYPE__TMQ_META;
@ -1656,7 +1663,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj));
if(pRspObj == NULL) {
if (pRspObj == NULL) {
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META;
@ -1669,10 +1676,11 @@ SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper)
return pRspObj;
}
void changeByteEndian(char* pData){
void changeByteEndian(char* pData) {
char* p = pData;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version:
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
// length | version:
int32_t blockVersion = *(int32_t*)p;
ASSERT(blockVersion == BLOCK_VERSION_1);
*(int32_t*)p = BLOCK_VERSION_2;
@ -1693,23 +1701,24 @@ void changeByteEndian(char* pData){
}
}
static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, int64_t *rows, int32_t *precision){
if(*(int64_t*)pRetrieve == 0){
static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, int64_t* rows, int32_t* precision) {
if (*(int64_t*)pRetrieve == 0) {
*rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
if(precision != NULL){
if (precision != NULL) {
*precision = ((SRetrieveTableRsp*)pRetrieve)->precision;
}
}else if(*(int64_t*)pRetrieve == 1){
} else if (*(int64_t*)pRetrieve == 1) {
*rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
if(precision != NULL){
if (precision != NULL) {
*precision = ((SRetrieveTableRspForTmq*)pRetrieve)->precision;
}
}
}
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) {
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) {
(*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
@ -1727,8 +1736,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
}
// extract the rows in this data packet
for (int32_t i = 0; i < pDataRsp->blockNum; ++i) {
void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i);
void* rawData = NULL;
void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i);
void* rawData = NULL;
int64_t rows = 0;
// deal with compatibility
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, NULL);
@ -1736,9 +1745,9 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
pVg->numOfRows += rows;
(*numOfRows) += rows;
changeByteEndian(rawData);
if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if(schema){
if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if (schema) {
taosArrayPush(pDataRsp->blockSchema, &schema);
}
}
@ -1747,7 +1756,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
if(pRspObj == NULL){
if (pRspObj == NULL) {
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ;
@ -1758,7 +1767,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
if(pRspObj == NULL){
if (pRspObj == NULL) {
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ_METADATA;
@ -1790,6 +1799,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
code = TSDB_CODE_INVALID_MSG;
taosMemoryFreeClear(msg);
goto FAIL;
}
@ -1858,7 +1868,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
}
for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId);
@ -1956,7 +1966,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
@ -2096,7 +2106,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp;
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
@ -2112,9 +2122,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
return NULL;
}
updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset,
pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId,
pDataRsp->blockNum != 0);
updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever,
tmq->consumerId, pDataRsp->blockNum != 0);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
@ -2130,15 +2139,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
// build rsp
int64_t numOfRows = 0;
void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
tmq->totalRows += numOfRows;
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows,
tmq->totalRows, pollRspWrapper->reqId);
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
@ -2266,7 +2275,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
taosSsleep(2); // sleep 2s for hb to send offset and rows to server
tmq_list_t* lst = tmq_list_new();
int32_t code = tmq_subscribe(tmq, lst);
int32_t code = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst);
if (code != 0) {
return code;
@ -2285,9 +2294,9 @@ const char* tmq_err2str(int32_t err) {
} else if (err == -1) {
return "fail";
} else {
if(*(taosGetErrMsg()) == 0){
if (*(taosGetErrMsg()) == 0) {
return tstrerror(err);
} else{
} else {
snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg());
return (const char*)taosGetErrMsgReturn();
}
@ -2359,8 +2368,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
return TSDB_CODE_INVALID_PARA;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
STqOffsetVal* pOffset = &common->reqOffset;
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
STqOffsetVal* pOffset = &common->reqOffset;
if (common->reqOffset.type == TMQ_OFFSET__LOG) {
return common->reqOffset.version;
} else {
@ -2389,7 +2398,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 ||
@ -2563,6 +2572,7 @@ end:
}
int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) return code;
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) {
@ -2575,6 +2585,9 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
goto END;
}
if (pMsg == NULL) {
goto END;
}
SMqRspHead* head = pMsg->pData;
int32_t epoch = atomic_load_32(&tmq->epoch);
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
@ -2604,18 +2617,24 @@ END:
FAIL:
if (pParam->sync) {
SAskEpInfo* pInfo = pParam->pParam;
pInfo->code = code;
tsem_post(&pInfo->sem);
if(pInfo) {
pInfo->code = code;
tsem_post(&pInfo->sem);
}
}
if(pMsg){
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData);
}
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pParam);
return code;
}
int32_t syncAskEp(tmq_t* pTmq) {
SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo));
if(pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY;
tsem_init(&pInfo->sem, 0, 0);
askEp(pTmq, pInfo, true, false);
@ -2734,8 +2753,8 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons
}
SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
pRspObj->resIter++;
if (pRspObj->resIter < common->blockNum) {
if (common->withSchema) {
@ -2744,8 +2763,8 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
}
void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter);
void* rawData = NULL;
void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter);
void* rawData = NULL;
int64_t rows = 0;
int32_t precision = 0;
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, &precision);
@ -2767,6 +2786,9 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
}
static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) {
return code;
}
SMqVgWalInfoParam* pParam = param;
SMqVgCommon* pCommon = pParam->pCommon;
@ -2798,8 +2820,11 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
tsem_post(&pCommon->rsp);
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
if(pMsg){
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
return 0;
}
@ -3231,6 +3256,9 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
if(param == NULL) {
return code;
}
SMqSeekParam* pParam = param;
pParam->code = code;
tsem_post(&pParam->sem);

147
source/client/test/clientMonitorTests.cpp
View File

@ -35,34 +35,34 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
TEST(clientMonitorTest, monitorTest) {
const char* cluster1 = "cluster1";
const char* cluster2 = "cluster2";
SEpSet epSet;
clusterMonitorInit(cluster1, epSet, NULL);
const char* counterName1 = "slow_query";
const char* counterName2 = "select_count";
const char* help1 = "test for slowQuery";
const char* help2 = "test for selectSQL";
const char* lables[] = {"lable1"};
taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables);
ASSERT_TRUE(c1 != NULL);
taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables);
ASSERT_TRUE(c2 != NULL);
ASSERT_TRUE(c1 != c2);
taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
ASSERT_TRUE(c21 == NULL);
clusterMonitorInit(cluster2, epSet, NULL);
c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
ASSERT_TRUE(c21 != NULL);
int i = 0;
while (i < 12) {
taosMsleep(10);
++i;
}
clusterMonitorClose(cluster1);
clusterMonitorClose(cluster2);
}
//TEST(clientMonitorTest, monitorTest) {
// const char* cluster1 = "cluster1";
// const char* cluster2 = "cluster2";
// SEpSet epSet;
// clientMonitorInit(cluster1, epSet, NULL);
// const char* counterName1 = "slow_query";
// const char* counterName2 = "select_count";
// const char* help1 = "test for slowQuery";
// const char* help2 = "test for selectSQL";
// const char* lables[] = {"lable1"};
// taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables);
// ASSERT_TRUE(c1 != NULL);
// taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables);
// ASSERT_TRUE(c2 != NULL);
// ASSERT_TRUE(c1 != c2);
// taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
// ASSERT_TRUE(c21 == NULL);
// clientMonitorInit(cluster2, epSet, NULL);
// c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
// ASSERT_TRUE(c21 != NULL);
// int i = 0;
// while (i < 12) {
// taosMsleep(10);
// ++i;
// }
// clusterMonitorClose(cluster1);
// clusterMonitorClose(cluster2);
//}
TEST(clientMonitorTest, sendTest) {
TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
@ -70,13 +70,100 @@ TEST(clientMonitorTest, sendTest) {
printf("connect taosd sucessfully.\n");
int64_t rid = *(int64_t *)taos;
SlowQueryLog(rid, false, -1, 1000);
slowQueryLog(rid, false, -1, 1000);
int i = 0;
while (i < 20) {
SlowQueryLog(rid, false, 0, i * 1000);
slowQueryLog(rid, false, 0, i * 1000);
taosMsleep(10);
++i;
}
taos_close(taos);
}
TEST(clientMonitorTest, ReadOneFile) {
// Create a TdFilePtr object and set it up for testing
TdFilePtr pFile = taosOpenFile("./tdengine-1-wewe", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("failed to open file:./test.txt since %s", terrstr());
return;
}
const int batch = 10;
const int size = SLOW_LOG_SEND_SIZE/batch;
for(int i = 0; i < batch + 1; i++){
char value[size] = {0};
memset(value, '0' + i, size - 1);
if (taosWriteFile(pFile, value, strlen(value) + 1) < 0){
uError("failed to write len to file:%p since %s", pFile, terrstr());
}
}
// Create a void pointer and set it up for testing
void* pTransporter = NULL;
// Create an SEpSet object and set it up for testing
SEpSet* epSet = NULL;
// Call the function to be tested
// monitorReadSendSlowLog(pFile, (int64_t)pTransporter, epSet);
char value[size] = {0};
memset(value, '0', size - 1);
if (taosWriteFile(pFile, value, strlen(value) + 1) < 0){
uError("failed to write len to file:%p since %s", pFile, terrstr());
}
// monitorReadSendSlowLog(pFile, (int64_t)pTransporter, epSet);
// Clean up any resources created for testing
taosCloseFile(&pFile);
}
TEST(clientMonitorTest, ReadTwoFile) {
// Create a TdFilePtr object and set it up for testing
TdFilePtr pFile = taosOpenFile("/tmp/tdengine_slow_log/tdengine-1-wewe", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("failed to open file:./test.txt since %s", terrstr());
return;
}
const int batch = 10;
const int size = SLOW_LOG_SEND_SIZE/batch;
for(int i = 0; i < batch + 1; i++){
char value[size] = {0};
memset(value, '0' + i, size - 1);
if (taosWriteFile(pFile, value, strlen(value) + 1) < 0){
uError("failed to write len to file:%p since %s", pFile, terrstr());
}
}
taosFsyncFile(pFile);
taosCloseFile(&pFile);
pFile = taosOpenFile("/tmp/tdengine_slow_log/tdengine-2-wewe", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("failed to open file:./test.txt since %s", terrstr());
return;
}
for(int i = 0; i < batch + 1; i++){
char value[size] = {0};
memset(value, '0' + i, size - 1);
if (taosWriteFile(pFile, value, strlen(value) + 1) < 0){
uError("failed to write len to file:%p since %s", pFile, terrstr());
}
}
taosFsyncFile(pFile);
taosCloseFile(&pFile);
SAppInstInfo pAppInfo = {0};
pAppInfo.clusterId = 2;
pAppInfo.monitorParas.tsEnableMonitor = 1;
strcpy(tsTempDir,"/tmp");
// monitorSendAllSlowLogFromTempDir(&pAppInfo);
}

41
source/client/test/clientTests.cpp
View File

@ -1520,6 +1520,13 @@ TEST(clientCase, sub_tb_test) {
}
TEST(clientCase, sub_tb_mt_test) {
char *user = NULL;
char *auth = NULL;
char *ip = NULL;
int port = 0;
char key[512] = {0};
snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port);
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
TdThread qid[20] = {0};
@ -1532,4 +1539,38 @@ TEST(clientCase, sub_tb_mt_test) {
}
}
//static void concatStrings(SArray *list, char* buf, int size){
// int len = 0;
// for(int i = 0; i < taosArrayGetSize(list); i++){
// char* db = (char*)taosArrayGet(list, i);
// char* dot = strchr(db, '.');
// if (dot != NULL) {
// db = dot + 1;
// }
// if (i != 0){
// strcat(buf, ",");
// len += 1;
// }
// int ret = snprintf(buf + len, size - len, "%s", db);
// if (ret < 0) {
// printf("snprintf failed, buf:%s, ret:%d", buf, ret);
// break;
// }
// len += ret;
// if (len >= size){
// printf("dbList is truncated, buf:%s, len:%d", buf, len);
// break;
// }
// }
//}
//
//TEST(clientCase, concat_string_test) {
// SArray* list = taosArrayInit(10, TSDB_DB_FNAME_LEN);
// taosArrayPush(list, "1.db1");
// taosArrayPush(list, "2.db2");
//
// char buf[32] = {0};
// concatStrings(list, buf, sizeof(buf) - 1);
//}
#pragma GCC diagnostic pop

26
source/common/src/cos.c
View File

@ -83,10 +83,10 @@ int32_t s3CheckCfg() {
snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]);
snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]);
} else {
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]);
snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]);
}
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
@ -1075,32 +1075,32 @@ static SArray *getListByPrefix(const char *prefix) {
S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&listBucketCallback};
const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data = {0};
const char /**marker = 0,*/ *delimiter = 0;
int /*maxkeys = 0, */ allDetails = 0;
list_bucket_callback_data data = {0};
data.objectArray = taosArrayInit(32, sizeof(void *));
if (!data.objectArray) {
uError("%s: %s", __func__, "out of memoty");
return NULL;
}
if (marker) {
/*if (marker) {
snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
} else {
data.nextMarker[0] = 0;
}
} else {*/
data.nextMarker[0] = 0;
//}
data.keyCount = 0;
data.allDetails = allDetails;
do {
data.isTruncated = 0;
do {
S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler,
&data);
S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, 0 /*maxkeys*/, 0, timeoutMsG,
&listBucketHandler, &data);
} while (S3_status_is_retryable(data.status) && should_retry());
if (data.status != S3StatusOK) {
break;
}
} while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys)));
} while (data.isTruncated /* && (!maxkeys || (data.keyCount < maxkeys))*/);
if (data.status == S3StatusOK) {
if (data.keyCount > 0) {

9
source/common/src/cos_cp.c
View File

@ -20,7 +20,7 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
}
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0;
@ -380,7 +380,7 @@ void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPa
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
checkpoint->parts[part_index].completed = 1;
strncpy(checkpoint->parts[part_index].etag, etag, 128);
strncpy(checkpoint->parts[part_index].etag, etag, 127);
checkpoint->parts[part_index].crc64 = crc64;
}
@ -389,11 +389,12 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t
int i = 0;
checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
memset(checkpoint->file_path, 0, TSDB_FILENAME_LEN);
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN - 1);
checkpoint->file_size = size;
checkpoint->file_last_modified = mtime;
strncpy(checkpoint->upload_id, upload_id, 128);
strncpy(checkpoint->upload_id, upload_id, 127);
checkpoint->part_size = part_size;
for (; i * part_size < size; i++) {

107
source/common/src/rsync.c
View File

@ -5,8 +5,8 @@
#include <stdlib.h>
#include "tglobal.h"
#define ERRNO_ERR_FORMAT "errno:%d,msg:%s"
#define ERRNO_ERR_DATA errno,strerror(errno)
#define ERRNO_ERR_FORMAT "errno:%d,msg:%s"
#define ERRNO_ERR_DATA errno, strerror(errno)
// deleteRsync function produce empty directories, traverse base directory to remove them
static void removeEmptyDir() {
@ -24,14 +24,14 @@ static void removeEmptyDir() {
char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de));
TdDirPtr pDirTmp = taosOpenDir(filename);
TdDirPtr pDirTmp = taosOpenDir(filename);
TdDirEntryPtr deTmp = NULL;
bool empty = true;
while ((deTmp = taosReadDir(pDirTmp)) != NULL){
bool empty = true;
while ((deTmp = taosReadDir(pDirTmp)) != NULL) {
if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue;
empty = false;
}
if(empty) taosRemoveDir(filename);
if (empty) taosRemoveDir(filename);
taosCloseDir(&pDirTmp);
}
@ -40,10 +40,10 @@ static void removeEmptyDir() {
#ifdef WINDOWS
// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/
static void changeDirFromWindowsToLinux(char* from, char* to){
static void changeDirFromWindowsToLinux(char* from, char* to) {
to[0] = '/';
to[1] = from[0];
for(int32_t i = 2; i < strlen(from); i++) {
for (int32_t i = 2; i < strlen(from); i++) {
if (from[i] == '\\') {
to[i] = '/';
} else {
@ -56,7 +56,7 @@ static void changeDirFromWindowsToLinux(char* from, char* to){
static int32_t generateConfigFile(char* confDir) {
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1;
}
@ -65,8 +65,8 @@ static int32_t generateConfigFile(char* confDir) {
changeDirFromWindowsToLinux(tsCheckpointBackupDir, path);
#endif
char confContent[PATH_MAX*4] = {0};
snprintf(confContent, PATH_MAX*4,
char confContent[PATH_MAX * 4] = {0};
snprintf(confContent, PATH_MAX * 4,
#ifndef WINDOWS
"uid = root\n"
"gid = root\n"
@ -80,16 +80,17 @@ static int32_t generateConfigFile(char* confDir) {
"read only = false\n"
"list = false\n"
"[checkpoint]\n"
"path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir,
"path = %s",
tsCheckpointBackupDir, tsCheckpointBackupDir,
#ifdef WINDOWS
path
#else
tsCheckpointBackupDir
#endif
);
);
uDebug("[rsync] conf:%s", confContent);
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){
uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile);
return -1;
}
@ -98,10 +99,10 @@ static int32_t generateConfigFile(char* confDir) {
return 0;
}
static int32_t execCommand(char* command){
static int32_t execCommand(char* command) {
int32_t try = 3;
int32_t code = 0;
while(try-- > 0) {
while (try-- > 0) {
code = system(command);
if (code == 0) {
break;
@ -125,7 +126,7 @@ void stopRsync() {
uDebug("[rsync] stop rsync server successful");
}
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
}
void startRsync() {
@ -154,7 +155,6 @@ void startRsync() {
} else {
uDebug("[rsync] start server successful");
}
}
int32_t uploadByRsync(const char* id, const char* path) {
@ -165,12 +165,12 @@ int32_t uploadByRsync(const char* id, const char* path) {
char pathTransform[PATH_MAX] = {0};
changeDirFromWindowsToLinux(path, pathTransform);
if(pathTransform[strlen(pathTransform) - 1] != '/') {
if (pathTransform[strlen(pathTransform) - 1] != '/') {
#else
if (path[strlen(path) - 1] != '/') {
#endif
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s/ "
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 --exclude=\"*\" %s/ "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
@ -182,7 +182,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
tsSnodeAddress, id);
} else {
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s "
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 --exclude=\"*\" %s "
"rsync://%s/checkpoint/%s/",
tsLogDir,
#ifdef WINDOWS
@ -194,7 +194,51 @@ int32_t uploadByRsync(const char* id, const char* path) {
tsSnodeAddress, id);
}
// prepare the data directory
int32_t code = execCommand(command);
if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
} else {
int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
tsSnodeAddress, el);
}
#ifdef WINDOWS
memset(pathTransform, 0, PATH_MAX);
changeDirFromWindowsToLinux(path, pathTransform);
if (pathTransform[strlen(pathTransform) - 1] != '/') {
#else
if (path[strlen(path) - 1] != '/') {
#endif
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s/ "
"rsync://%s/checkpoint/%s/data/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
path
#endif
,
tsSnodeAddress, id);
} else {
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 --bwlimit=100000 %s "
"rsync://%s/checkpoint/%s/data/",
tsLogDir,
#ifdef WINDOWS
pathTransform
#else
path
#endif
,
tsSnodeAddress, id);
}
code = execCommand(command);
if (code != 0) {
uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
@ -210,7 +254,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
// abort from retry if quit
int32_t downloadRsync(const char* id, const char* path) {
int64_t st = taosGetTimestampMs();
int32_t MAX_RETRY = 60;
int32_t MAX_RETRY = 10;
int32_t times = 0;
int32_t code = 0;
@ -220,19 +264,20 @@ int32_t downloadRsync(const char* id, const char* path) {
#endif
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
tsLogDir, tsSnodeAddress, id,
snprintf(
command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/data/ %s",
tsLogDir, tsSnodeAddress, id,
#ifdef WINDOWS
pathTransform
pathTransform
#else
path
path
#endif
);
uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
while(times++ < MAX_RETRY) {
while (times++ < MAX_RETRY) {
code = execCommand(command);
if (code != TSDB_CODE_SUCCESS) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
@ -258,8 +303,8 @@ int32_t deleteRsync(const char* id) {
char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/", tsLogDir,
tmp, tsSnodeAddress, id);
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/data/",
tsLogDir, tmp, tsSnodeAddress, id);
code = execCommand(command);
taosRemoveDir(tmp);

2
source/common/src/systable.c
View File

@ -275,7 +275,7 @@ static const SSysDbTableSchema userUsersFullSchema[] = {
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "createdb", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypted_pass", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "encrypted_pass", .bytes = TSDB_PASSWORD_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};

2
source/common/src/tdataformat.c
View File

@ -2983,7 +2983,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
}
} else {
if (varDataTLen(data + offset) > bytes) {
uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset),
uError("var data length invalid, varDataTLen(data + offset):%d > bytes:%d", (int)varDataTLen(data + offset),
bytes);
code = TSDB_CODE_PAR_VALUE_TOO_LONG;
goto _exit;

149
source/common/src/tglobal.c
View File

@ -182,8 +182,12 @@ int32_t tsRedirectMaxPeriod = 1000;
int32_t tsMaxRetryWaitTime = 10000;
bool tsUseAdapter = false;
int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 3; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL;
int32_t tsSlowLogThreshold = 10; // seconds
int32_t tsSlowLogThresholdTest = INT32_MAX; // seconds
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN] = ""; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_QUERY;
char* tsSlowLogScopeString = "query";
int32_t tsSlowLogMaxLen = 4096;
int32_t tsTimeSeriesThreshold = 50;
bool tsMultiResultFunctionStarReturnTags = false;
@ -391,9 +395,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
return 0;
}
struct SConfig *taosGetCfg() {
return tsCfg;
}
struct SConfig *taosGetCfg() { return tsCfg; }
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) {
@ -546,9 +548,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1;
if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
@ -573,9 +572,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT,
CFG_DYN_CLIENT) != 0)
return -1;
@ -706,6 +702,15 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "slowLogThresholdTest", tsSlowLogThresholdTest, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -970,40 +975,52 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32;
}
static int32_t taosSetSlowLogScope(const char *pScope) {
int32_t taosSetSlowLogScope(char *pScope) {
if (NULL == pScope || 0 == strlen(pScope)) {
tsSlowLogScope = SLOW_LOG_TYPE_ALL;
return 0;
return SLOW_LOG_TYPE_QUERY;
}
if (0 == strcasecmp(pScope, "all")) {
tsSlowLogScope = SLOW_LOG_TYPE_ALL;
return 0;
int32_t slowScope = 0;
char* scope = NULL;
char *tmp = NULL;
while((scope = strsep(&pScope, "|")) != NULL){
taosMemoryFreeClear(tmp);
tmp = taosStrdup(scope);
strtrim(tmp);
if (0 == strcasecmp(tmp, "all")) {
slowScope |= SLOW_LOG_TYPE_ALL;
continue;
}
if (0 == strcasecmp(tmp, "query")) {
slowScope |= SLOW_LOG_TYPE_QUERY;
continue;
}
if (0 == strcasecmp(tmp, "insert")) {
slowScope |= SLOW_LOG_TYPE_INSERT;
continue;
}
if (0 == strcasecmp(tmp, "others")) {
slowScope |= SLOW_LOG_TYPE_OTHERS;
continue;
}
if (0 == strcasecmp(tmp, "none")) {
slowScope |= SLOW_LOG_TYPE_NULL;
continue;
}
taosMemoryFreeClear(tmp);
uError("Invalid slowLog scope value:%s", pScope);
terrno = TSDB_CODE_INVALID_CFG_VALUE;
return -1;
}
if (0 == strcasecmp(pScope, "query")) {
tsSlowLogScope = SLOW_LOG_TYPE_QUERY;
return 0;
}
if (0 == strcasecmp(pScope, "insert")) {
tsSlowLogScope = SLOW_LOG_TYPE_INSERT;
return 0;
}
if (0 == strcasecmp(pScope, "others")) {
tsSlowLogScope = SLOW_LOG_TYPE_OTHERS;
return 0;
}
if (0 == strcasecmp(pScope, "none")) {
tsSlowLogScope = 0;
return 0;
}
uError("Invalid slowLog scope value:%s", pScope);
terrno = TSDB_CODE_INVALID_CFG_VALUE;
return -1;
taosMemoryFreeClear(tmp);
return slowScope;
}
// for common configs
@ -1061,12 +1078,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64;
tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32;
tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
return -1;
}
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
@ -1142,6 +1153,16 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsSIMDEnable = (bool)cfgGetItem(pCfg, "simdEnable")->bval;
tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;
tstrncpy(tsSlowLogExceptDb, cfgGetItem(pCfg, "slowLogExceptDb")->str, TSDB_DB_NAME_LEN);
tsSlowLogThresholdTest = cfgGetItem(pCfg, "slowLogThresholdTest")->i32;
tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32;
tsSlowLogMaxLen = cfgGetItem(pCfg, "slowLogMaxLen")->i32;
int32_t scope = taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str);
if(scope < 0){
return -1;
}
tsSlowLogScope = scope;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);
@ -1429,8 +1450,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
char *name = pItem->name;
for (int32_t d = 0; d < optionSize; ++d) {
const char *optName = pOptions[d].optionName;
int32_t optLen = strlen(optName);
if (strncasecmp(name, optName, optLen) != 0) continue;
if (strcasecmp(name, optName) != 0) continue;
switch (pItem->dtype) {
case CFG_DTYPE_BOOL: {
int32_t flag = pItem->i32;
@ -1503,6 +1523,23 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
return 0;
}
if (strcasecmp("slowLogScope", name) == 0) {
int32_t scope = taosSetSlowLogScope(pItem->str);
if(scope < 0){
cfgUnLock(pCfg);
return -1;
}
tsSlowLogScope = scope;
cfgUnLock(pCfg);
return 0;
}
if (strcasecmp("slowLogExceptDb", name) == 0) {
tstrncpy(tsSlowLogExceptDb, pItem->str, TSDB_DB_NAME_LEN);
cfgUnLock(pCfg);
return 0;
}
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function
static OptionNameAndVar debugOptions[] = {
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
@ -1520,6 +1557,10 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"enableWhiteList", &tsEnableWhiteList},
{"telemetryReporting", &tsEnableTelem},
{"monitor", &tsEnableMonitor},
{"monitorInterval", &tsMonitorInterval},
{"slowLogThreshold", &tsSlowLogThreshold},
{"slowLogThresholdTest", &tsSlowLogThresholdTest},
{"slowLogMaxLen", &tsSlowLogMaxLen},
{"mndSdbWriteDelta", &tsMndSdbWriteDelta},
{"minDiskFreeSize", &tsMinDiskFreeSize},
@ -1662,10 +1703,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
uInfo("%s set to %" PRId64, name, tsLogSpace.reserved);
matched = true;
} else if (strcasecmp("monitor", name) == 0) {
tsEnableMonitor = pItem->bval;
uInfo("%s set to %d", name, tsEnableMonitor);
matched = true;
}
break;
}
@ -1709,13 +1746,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype, false);
uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst);
matched = true;
} else if (strcasecmp("slowLogScope", name) == 0) {
if (taosSetSlowLogScope(pItem->str)) {
cfgUnLock(pCfg);
return -1;
}
uInfo("%s set to %s", name, pItem->str);
matched = true;
}
break;
}
@ -1776,7 +1806,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
{"queryUseNodeAllocator", &tsQueryUseNodeAllocator},
{"smlDot2Underline", &tsSmlDot2Underline},
{"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental},
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags},

48
source/common/src/tmsg.c
View File

@ -69,6 +69,28 @@
pReq->sql = NULL; \
} while (0)
static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas* pMonitorParas) {
if (tEncodeI8(encoder, pMonitorParas->tsEnableMonitor) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsMonitorInterval) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsSlowLogScope) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold) < 0) return -1;
if (tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest) < 0) return -1;
if (tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb) < 0) return -1;
return 0;
}
static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas* pMonitorParas){
if (tDecodeI8(decoder, (int8_t *)&pMonitorParas->tsEnableMonitor) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsMonitorInterval) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold) < 0) return -1;
if (tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest) < 0) return -1;
if (tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb) < 0) return -1;
return 0;
}
static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp);
@ -519,6 +541,7 @@ int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBa
SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i);
if (tSerializeSClientHbRsp(&encoder, pRsp) < 0) return -1;
}
if (tSerializeSMonitorParas(&encoder, &pBatchRsp->monitorParas) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -546,6 +569,10 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
taosArrayPush(pBatchRsp->rsps, &rsp);
}
if (!tDecodeIsEnd(&decoder)) {
if (tDeserializeSMonitorParas(&decoder, &pBatchRsp->monitorParas) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -1308,6 +1335,9 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
}
if (tEncodeI64(&encoder, pReq->ipWhiteVer) < 0) return -1;
if (tSerializeSMonitorParas(&encoder, &pReq->clusterCfg.monitorParas) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1426,6 +1456,10 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeI64(&decoder, &pReq->ipWhiteVer) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDeserializeSMonitorParas(&decoder, &pReq->clusterCfg.monitorParas) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -1521,6 +1555,7 @@ int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1;
if (tEncodeI8(&encoder, pReq->type) < 0) return -1;
tEndEncode(&encoder);
@ -1541,7 +1576,9 @@ int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
if (pReq->pCont == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, (int8_t*)&pReq->type) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -5153,6 +5190,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->authVer) < 0) return -1;
if (tEncodeI64(&encoder, pRsp->whiteListVer) < 0) return -1;
if (tSerializeSMonitorParas(&encoder, &pRsp->monitorParas) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -5194,6 +5232,9 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
} else {
pRsp->whiteListVer = 0;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDeserializeSMonitorParas(&decoder, &pRsp->monitorParas) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -10481,7 +10522,10 @@ int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) {
pRet->ast = taosStrdup(pInfo->ast);
if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code) tFreeTableTSMAInfo(pRet);
if (code) {
tFreeTableTSMAInfo(pRet);
pRet = NULL;
}
*pRes = pRet;
return code;
}

21
source/common/src/tvariant.c
View File

@ -161,8 +161,7 @@ int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) {
int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
errno = 0;
char *endPtr = NULL;
switch (type)
{
switch (type) {
case TK_NK_INTEGER: {
*value = taosStr2Int64(z, &endPtr, 10);
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -172,7 +171,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} break;
case TK_NK_FLOAT: {
double val = round(taosStr2Double(z, &endPtr));
if(val < (double)INT64_MIN || val > (double)INT64_MAX){
if (val < (double)INT64_MIN || val > (double)INT64_MAX) {
return TSDB_CODE_FAILED;
}
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -181,8 +180,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
*value = val;
return TSDB_CODE_SUCCESS;
} break;
default:
break;
default:
break;
}
if (n == 0) {
@ -201,7 +200,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
// pure decimal part
const char *s = endPtr + 1;
const char *end = z + n;
bool pure = true;
bool pure = true;
while (s < end) {
if (*s < '0' || *s > '9') {
pure = false;
@ -210,17 +209,17 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
s++;
}
if (pure) {
if (endPtr+1 < end && endPtr[1] > '4') {
if (endPtr + 1 < end && endPtr[1] > '4') {
const char *p = z;
while (*p == ' ') {
p++;
}
if (*p == '-') {
if ( *value > INT64_MIN) {
if (*value > INT64_MIN) {
(*value)--;
}
} else {
if ( *value < INT64_MAX) {
if (*value < INT64_MAX) {
(*value)++;
}
}
@ -229,7 +228,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
}
}
// 2. parse as other
// 2. parse as other
bool is_neg = false;
uint64_t uv = 0;
int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, true);
@ -256,7 +255,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
errno = 0;
char *endPtr = NULL;
char *endPtr = NULL;
const char *p = z;
switch (type) {
case TK_NK_INTEGER: {

114
source/common/test/commonTests.cpp
View File

@ -726,4 +726,118 @@ TEST(AlreadyAddGroupIdTest, GroupIdAddedWithDifferentLength) {
EXPECT_FALSE(result);
}
#define SLOW_LOG_TYPE_NULL 0x0
#define SLOW_LOG_TYPE_QUERY 0x1
#define SLOW_LOG_TYPE_INSERT 0x2
#define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0x7
static int32_t taosSetSlowLogScope(char *pScope) {
if (NULL == pScope || 0 == strlen(pScope)) {
return SLOW_LOG_TYPE_QUERY;
}
int32_t slowScope = 0;
char* scope = NULL;
char *tmp = NULL;
while((scope = strsep(&pScope, "|")) != NULL){
taosMemoryFreeClear(tmp);
tmp = taosStrdup(scope);
strtrim(tmp);
if (0 == strcasecmp(tmp, "all")) {
slowScope |= SLOW_LOG_TYPE_ALL;
continue;
}
if (0 == strcasecmp(tmp, "query")) {
slowScope |= SLOW_LOG_TYPE_QUERY;
continue;
}
if (0 == strcasecmp(tmp, "insert")) {
slowScope |= SLOW_LOG_TYPE_INSERT;
continue;
}
if (0 == strcasecmp(tmp, "others")) {
slowScope |= SLOW_LOG_TYPE_OTHERS;
continue;
}
if (0 == strcasecmp(tmp, "none")) {
slowScope |= SLOW_LOG_TYPE_NULL;
continue;
}
taosMemoryFreeClear(tmp);
uError("Invalid slowLog scope value:%s", pScope);
terrno = TSDB_CODE_INVALID_CFG_VALUE;
return -1;
}
taosMemoryFreeClear(tmp);
return slowScope;
}
TEST(TaosSetSlowLogScopeTest, NullPointerInput) {
char *pScope = NULL;
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_QUERY);
}
TEST(TaosSetSlowLogScopeTest, EmptyStringInput) {
char pScope[1] = "";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_QUERY);
}
TEST(TaosSetSlowLogScopeTest, AllScopeInput) {
char pScope[] = "all";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_ALL);
}
TEST(TaosSetSlowLogScopeTest, QueryScopeInput) {
char pScope[] = " query";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_QUERY);
}
TEST(TaosSetSlowLogScopeTest, InsertScopeInput) {
char pScope[] = "insert";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_INSERT);
}
TEST(TaosSetSlowLogScopeTest, OthersScopeInput) {
char pScope[] = "others";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_OTHERS);
}
TEST(TaosSetSlowLogScopeTest, NoneScopeInput) {
char pScope[] = "none";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, SLOW_LOG_TYPE_NULL);
}
TEST(TaosSetSlowLogScopeTest, InvalidScopeInput) {
char pScope[] = "invalid";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, -1);
}
TEST(TaosSetSlowLogScopeTest, MixedScopesInput) {
char pScope[] = "query|insert|others|none";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, (SLOW_LOG_TYPE_QUERY | SLOW_LOG_TYPE_INSERT | SLOW_LOG_TYPE_OTHERS));
}
TEST(TaosSetSlowLogScopeTest, MixedScopesInputWithSpaces) {
char pScope[] = "query | insert | others ";
int32_t result = taosSetSlowLogScope(pScope);
EXPECT_EQ(result, (SLOW_LOG_TYPE_QUERY | SLOW_LOG_TYPE_INSERT | SLOW_LOG_TYPE_OTHERS));
}
#pragma GCC diagnostic pop

16
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -53,7 +53,8 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
.msgType = TDMT_MND_RETRIEVE_IP_WHITE,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 0};
.info.noResp = 0,
.info.handle = 0};
SEpSet epset = {0};
dmGetMnodeEpSet(pMgmt->pData, &epset);
@ -116,6 +117,13 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.monitorParas.tsEnableMonitor = tsEnableMonitor;
req.clusterCfg.monitorParas.tsMonitorInterval = tsMonitorInterval;
req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope;
req.clusterCfg.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
req.clusterCfg.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN);
@ -147,7 +155,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
.msgType = TDMT_MND_STATUS,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 0};
.info.noResp = 0,
.info.handle = 0};
SRpcMsg rpcRsp = {0};
dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq);
@ -179,7 +188,8 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
.msgType = TDMT_MND_NOTIFY,
.info.ahandle = (void *)0x9527,
.info.refId = 0,
.info.noResp = 1};
.info.noResp = 1,
.info.handle = 0};
SEpSet epSet = {0};
dmGetMnodeEpSet(pMgmt->pData, &epSet);

1
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -290,6 +290,7 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
} else {
pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
return 0;
}

6
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -143,6 +143,12 @@ typedef enum {
DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH,
DND_REASON_ENABLE_WHITELIST_NOT_MATCH,
DND_REASON_ENCRYPTION_KEY_NOT_MATCH,
DND_REASON_STATUS_MONITOR_NOT_MATCH,
DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH,
DND_REASON_STATUS_MONITOR_INTERVAL_NOT_MATCH,
DND_REASON_STATUS_MONITOR_SLOW_LOG_THRESHOLD_NOT_MATCH,
DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH,
DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH,
DND_REASON_OTHERS
} EDndReason;

1
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -84,6 +84,7 @@ void mndTransSetParallel(STrans *pTrans);
void mndTransSetChangeless(STrans *pTrans);
void mndTransSetOper(STrans *pTrans, EOperType oper);
int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans);
int32_t mndTransCheckConflictWithCompact(SMnode *pMnode, STrans *pTrans);
#ifndef BUILD_NO_CALL
static int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) {
return mndTransCheckConflict(pMnode, pTrans);

290
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -49,6 +49,12 @@ static const char *offlineReason[] = {
"ttlChangeOnWrite not match",
"enableWhiteList not match",
"encryptionKey not match",
"monitor not match",
"monitor switch not match",
"monitor interval not match",
"monitor slow log threshold not match",
"monitor slow log sql max len not match",
"monitor slow log scope not match",
"unknown",
};
@ -438,7 +444,27 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
}
}
#define CHECK_MONITOR_PARA(para,err) \
if (pCfg->monitorParas.para != para) { \
mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \
terrno = err; \
return err;\
}
static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const SClusterCfg *pCfg) {
CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH);
CHECK_MONITOR_PARA(tsMonitorInterval, DND_REASON_STATUS_MONITOR_INTERVAL_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogThreshold, DND_REASON_STATUS_MONITOR_SLOW_LOG_THRESHOLD_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogThresholdTest, DND_REASON_STATUS_MONITOR_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogMaxLen, DND_REASON_STATUS_MONITOR_SLOW_LOG_SQL_MAX_LEN_NOT_MATCH);
CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH);
if (0 != strcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) {
mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb);
terrno = TSDB_CODE_DNODE_INVALID_MONITOR_PARAS;
return DND_REASON_STATUS_MONITOR_NOT_MATCH;
}
if (pCfg->statusInterval != tsStatusInterval) {
mError("dnode:%d, statusInterval:%d inconsistent with cluster:%d", pDnode->id, pCfg->statusInterval,
tsStatusInterval);
@ -530,14 +556,13 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) {
return stateChanged;
}
extern char* tsMonFwUri;
extern char* tsMonSlowLogUri;
static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStatisReq statisReq = {0};
int32_t code = -1;
char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
sprintf(strClusterId, "%" PRId64, pMnode->clusterId);
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return code;
@ -547,188 +572,14 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
mInfo("process statis req,\n %s", statisReq.pCont);
}
SJson *pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson);
for (int32_t i = 0; i < ts_size; i++) {
SJson *item = tjsonGetArrayItem(pJson, i);
SJson *tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables);
for (int32_t i = 0; i < tableSize; i++) {
SJson *table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName);
SJson *metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups);
for (int32_t i = 0; i < size; i++) {
SJson *item = tjsonGetArrayItem(metricGroups, i);
SJson *arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag);
for (int32_t j = 0; j < tagSize; j++) {
SJson *item = tjsonGetArrayItem(arrayTag, j);
char tagName[MONITOR_TAG_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", tagName);
if (strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
tjsonDeleteItemFromObject(item, "value");
tjsonAddStringToObject(item, "value", strClusterId);
}
}
}
}
}
char *pCont = tjsonToString(pJson);
monSendContent(pCont);
if (pJson != NULL) {
tjsonDelete(pJson);
pJson = NULL;
}
if (pCont != NULL) {
taosMemoryFree(pCont);
pCont = NULL;
if (statisReq.type == MONITOR_TYPE_COUNTER){
monSendContent(statisReq.pCont, tsMonFwUri);
}else if(statisReq.type == MONITOR_TYPE_SLOW_LOG){
monSendContent(statisReq.pCont, tsMonSlowLogUri);
}
tFreeSStatisReq(&statisReq);
return 0;
/*
SJson* pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson);
for(int32_t i = 0; i < ts_size; i++){
SJson* item = tjsonGetArrayItem(pJson, i);
SJson* tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){
SJson* table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName);
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){
SJson* item = tjsonGetArrayItem(metricGroups, i);
SJson* arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag);
char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j);
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
tjsonGetStringValue(item, "name", *(labels + j));
*(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
tjsonGetStringValue(item, "value", *(sample_labels + j));
if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
}
}
SJson* metrics = tjsonGetObjectItem(item, "metrics");
int32_t metricLen = tjsonGetArraySize(metrics);
for(int32_t j = 0; j < metricLen; j++){
SJson *item = tjsonGetArrayItem(metrics, j);
char name[MONITOR_METRIC_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", name);
double value = 0;
tjsonGetDoubleValue(item, "value", &value);
double type = 0;
tjsonGetDoubleValue(item, "type", &type);
int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
char* metricName = taosMemoryMalloc(metricNameLen);
memset(metricName, 0, metricNameLen);
sprintf(metricName, "%s:%s", tableName, name);
taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
if(metric == NULL){
if(type == 0){
metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
}
if(type == 1){
metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
}
mTrace("fail to get metric from registry, new one metric:%p", metric);
if(taos_collector_registry_register_metric(metric) == 1){
if(type == 0){
taos_counter_destroy(metric);
}
if(type == 1){
taos_gauge_destroy(metric);
}
metric = taos_collector_registry_get_metric(metricName);
mTrace("fail to register metric, get metric from registry:%p", metric);
}
else{
mTrace("succeed to register metric:%p", metric);
}
}
else{
mTrace("get metric from registry:%p", metric);
}
if(type == 0){
taos_counter_add(metric, value, (const char**)sample_labels);
}
if(type == 1){
taos_gauge_set(metric, value, (const char**)sample_labels);
}
taosMemoryFreeClear(metricName);
}
for(int32_t j = 0; j < tagSize; j++){
taosMemoryFreeClear(*(labels + j));
taosMemoryFreeClear(*(sample_labels + j));
}
taosMemoryFreeClear(sample_labels);
taosMemoryFreeClear(labels);
}
}
}
code = 0;
_OVER:
if(pJson != NULL){
tjsonDelete(pJson);
pJson = NULL;
}
tFreeSStatisReq(&statisReq);
return code;
*/
}
static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
@ -1092,6 +943,32 @@ _OVER:
return code;
}
static void getSlowLogScopeString(int32_t scope, char* result){
if(scope == SLOW_LOG_TYPE_NULL) {
strcat(result, "NONE");
return;
}
while(scope > 0){
if(scope & SLOW_LOG_TYPE_QUERY) {
strcat(result, "QUERY");
scope &= ~SLOW_LOG_TYPE_QUERY;
} else if(scope & SLOW_LOG_TYPE_INSERT) {
strcat(result, "INSERT");
scope &= ~SLOW_LOG_TYPE_INSERT;
} else if(scope & SLOW_LOG_TYPE_OTHERS) {
strcat(result, "OTHERS");
scope &= ~SLOW_LOG_TYPE_OTHERS;
} else{
printf("invalid slow log scope:%d", scope);
return;
}
if(scope > 0) {
strcat(result, "|");
}
}
}
static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
SShowVariablesRsp rsp = {0};
int32_t code = -1;
@ -1100,7 +977,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
goto _OVER;
}
rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo));
rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo));
if (NULL == rsp.variables) {
mError("failed to alloc SVariablesInfo array while process show variables req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -1129,6 +1006,33 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
strcpy(info.scope, "both");
taosArrayPush(rsp.variables, &info);
strcpy(info.name, "monitor");
snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsEnableMonitor);
strcpy(info.scope, "server");
taosArrayPush(rsp.variables, &info);
strcpy(info.name, "monitorInterval");
snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsMonitorInterval);
strcpy(info.scope, "server");
taosArrayPush(rsp.variables, &info);
strcpy(info.name, "slowLogThreshold");
snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogThreshold);
strcpy(info.scope, "server");
taosArrayPush(rsp.variables, &info);
strcpy(info.name, "slowLogMaxLen");
snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogMaxLen);
strcpy(info.scope, "server");
taosArrayPush(rsp.variables, &info);
char scopeStr[64] = {0};
getSlowLogScopeString(tsSlowLogScope, scopeStr);
strcpy(info.name, "slowLogScope");
snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", scopeStr);
strcpy(info.scope, "server");
taosArrayPush(rsp.variables, &info);
int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
@ -1679,6 +1583,28 @@ static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%s", tsCharset);
totalRows++;
cfgOpts[totalRows] = "monitor";
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%d", tsEnableMonitor);
totalRows++;
cfgOpts[totalRows] = "monitorInterval";
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%d", tsMonitorInterval);
totalRows++;
cfgOpts[totalRows] = "slowLogThreshold";
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogThreshold);
totalRows++;
cfgOpts[totalRows] = "slowLogMaxLen";
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogMaxLen);
totalRows++;
char scopeStr[64] = {0};
getSlowLogScopeString(tsSlowLogScope, scopeStr);
cfgOpts[totalRows] = "slowLogScope";
snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%s", scopeStr);
totalRows++;
char buf[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
char bufVal[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};

15
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -282,7 +282,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
}
}
_CONNECT:
pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp,
pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime);
if (pConn == NULL) {
@ -301,6 +300,13 @@ _CONNECT:
connectRsp.svrTimestamp = taosGetTimestampSec();
connectRsp.passVer = pUser->passVersion;
connectRsp.authVer = pUser->authVersion;
connectRsp.monitorParas.tsEnableMonitor = tsEnableMonitor;
connectRsp.monitorParas.tsMonitorInterval = tsMonitorInterval;
connectRsp.monitorParas.tsSlowLogScope = tsSlowLogScope;
connectRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
connectRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
connectRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
connectRsp.whiteListVer = pUser->ipWhiteListVer;
strcpy(connectRsp.sVer, version);
@ -660,6 +666,13 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
SClientHbBatchRsp batchRsp = {0};
batchRsp.svrTimestamp = taosGetTimestampSec();
batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp));
batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor;
batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval;
batchRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold;
batchRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest;
tstrncpy(batchRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN);
batchRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen;
batchRsp.monitorParas.tsSlowLogScope = tsSlowLogScope;
int32_t sz = taosArrayGetSize(batchReq.reqs);
for (int i = 0; i < sz; i++) {

103
source/dnode/mnode/impl/src/mndStream.c
View File

@ -56,7 +56,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
@ -301,17 +301,17 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
return 0;
}
static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrapper) {
static int32_t createSchemaByFields(const SArray *pFields, SSchemaWrapper *pWrapper) {
pWrapper->nCols = taosArrayGetSize(pFields);
pWrapper->pSchema = taosMemoryCalloc(pWrapper->nCols, sizeof(SSchema));
if (NULL == pWrapper->pSchema) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode;
SNode *pNode;
int32_t index = 0;
for(int32_t i = 0; i < pWrapper->nCols; i++) {
SField* pField = (SField*)taosArrayGet(pFields, i);
for (int32_t i = 0; i < pWrapper->nCols; i++) {
SField *pField = (SField *)taosArrayGet(pFields, i);
if (TSDB_DATA_TYPE_NULL == pField->type) {
pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR;
pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE;
@ -328,14 +328,14 @@ static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrap
return TSDB_CODE_SUCCESS;
}
static bool hasDestPrimaryKey(SSchemaWrapper* pWrapper) {
static bool hasDestPrimaryKey(SSchemaWrapper *pWrapper) {
if (pWrapper->nCols < 2) {
return false;
}
for (int32_t i = 1; i < pWrapper->nCols; i++) {
if(pWrapper->pSchema[i].flags & COL_IS_KEY) {
if (pWrapper->pSchema[i].flags & COL_IS_KEY) {
return true;
}
}
}
return false;
}
@ -445,7 +445,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->outputSchema.pSchema = pFullSchema;
}
bool hasKey = hasDestPrimaryKey(&pObj->outputSchema);
bool hasKey = hasDestPrimaryKey(&pObj->outputSchema);
SPlanContext cxt = {
.pAstRoot = pAst,
.topicQuery = false,
@ -492,7 +492,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
SEncoder encoder;
tEncoderInit(&encoder, NULL, 0);
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
pTask->ver = SSTREAM_TASK_VER;
}
tEncodeStreamTask(&encoder, pTask);
@ -700,7 +700,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
SStreamObj streamObj = {0};
char *sql = NULL;
int32_t sqlLen = 0;
const char* pMsg = "create stream tasks on dnodes";
const char *pMsg = "create stream tasks on dnodes";
terrno = TSDB_CODE_SUCCESS;
@ -959,7 +959,8 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return -1;
}
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
code =
setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
if (code != 0) {
taosMemoryFree(buf);
}
@ -1106,7 +1107,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
}
SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId));
SArray *pInvalidList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
@ -1116,8 +1117,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
}
if (pEntry->status == TASK_STATUS__STOP) {
for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId* pId = taosArrayGet(pInvalidList, j);
for (int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId *pId = taosArrayGet(pInvalidList, j);
if (pEntry->id.streamId == pId->streamId) {
taosArrayPush(pInvalidList, &pEntry->id);
break;
@ -1126,8 +1127,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
}
if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued",
pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId,
(int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false;
}
@ -1153,14 +1154,14 @@ typedef struct {
int64_t duration;
} SCheckpointInterval;
static int32_t streamWaitComparFn(const void* p1, const void* p2) {
const SCheckpointInterval* pInt1 = p1;
const SCheckpointInterval* pInt2 = p2;
static int32_t streamWaitComparFn(const void *p1, const void *p2) {
const SCheckpointInterval *pInt1 = p1;
const SCheckpointInterval *pInt2 = p2;
if (pInt1->duration == pInt2->duration) {
return 0;
}
return pInt1->duration > pInt2->duration? -1:1;
return pInt1->duration > pInt2->duration ? -1 : 1;
}
static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
@ -1176,7 +1177,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
return -1;
}
SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval));
SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval));
int64_t now = taosGetTimestampMs();
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) {
@ -1190,8 +1191,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
taosArrayPush(pList, &in);
int32_t currentSize = taosArrayGetSize(pList);
mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64
"s) beyond threshold:%d",
mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 "s) beyond threshold:%d",
pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize);
sdbRelease(pSdb, pStream);
@ -1278,11 +1278,11 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
if (pStream->smaId != 0) {
mDebug("stream:%s, uid:0x%"PRIx64" try to drop sma related stream", dropReq.name, pStream->uid);
mDebug("stream:%s, uid:0x%" PRIx64 " try to drop sma related stream", dropReq.name, pStream->uid);
void *pIter = NULL;
SSmaObj *pSma = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
while (pIter) {
if (pSma && pSma->uid == pStream->smaId) {
sdbRelease(pMnode->pSdb, pSma);
@ -1321,7 +1321,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) {
mError("stream:%s uid:0x%"PRIx64" failed to drop since %s", dropReq.name, pStream->uid, terrstr());
mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr());
sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq);
return -1;
@ -1674,8 +1674,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
// input total
const char* formatTotalMb = "%7.2f MiB";
const char* formatTotalGb = "%7.2f GiB";
const char *formatTotalMb = "%7.2f MiB";
const char *formatTotalGb = "%7.2f GiB";
if (pe->procsTotal < 1024) {
snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal);
} else {
@ -1689,8 +1689,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
// process throughput
const char* formatKb = "%7.2f KiB/s";
const char* formatMb = "%7.2f MiB/s";
const char *formatKb = "%7.2f KiB/s";
const char *formatMb = "%7.2f MiB/s";
if (pe->procsThroughput < 1024) {
snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput);
} else {
@ -1760,15 +1760,15 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// start_time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startTime, false);
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startTime, false);
// start id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointId, false);
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointId, false);
// start ver
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointVer, false);
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointVer, false);
// checkpoint time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1780,11 +1780,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// checkpoint_id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestId, false);
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestId, false);
// checkpoint version
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false);
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestVer, false);
// checkpoint size
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1915,13 +1915,12 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1;
}
{ // check for tasks, if tasks are not ready, not allowed to pause
{ // check for tasks, if tasks are not ready, not allowed to pause
bool found = false;
bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock);
for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
@ -2237,7 +2236,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0;
}
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
@ -2359,7 +2358,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
return 0;
}
int32_t size = sizeof(SMStreamNodeCheckMsg);
int32_t size = sizeof(SMStreamNodeCheckMsg);
SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size};
@ -2386,8 +2385,8 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode)
// add the new vgroups if not added yet
bool exist = false;
for(int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) {
SNodeEntry* pEntry = taosArrayGet(pExecNode->pNodeList, j);
for (int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) {
SNodeEntry *pEntry = taosArrayGet(pExecNode->pNodeList, j);
if (pEntry->nodeId == pTask->info.nodeId) {
exist = true;
break;
@ -2514,10 +2513,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
return 0;
}
static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) {
static void doAddTaskInfo(SArray *pList, SCheckpointReport *pReport) {
bool existed = false;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo* p = taosArrayGet(pList ,i);
for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo *p = taosArrayGet(pList, i);
if (p->taskId == pReport->taskId) {
existed = true;
break;
@ -2563,8 +2562,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf",
req.streamId);
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans.
@ -2671,7 +2669,7 @@ void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
pExecInfo->initTaskList = true;
}
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
@ -2687,13 +2685,14 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
}
}
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info");
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME,
"update checkpoint-info");
if (pTrans == NULL) {
return terrno;
}
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);

36
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -39,10 +39,10 @@ static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
}
}
static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pInfo) {
static void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
int32_t num = taosArrayGetSize(pList);
for(int32_t i = 0; i < num; ++i) {
SFailedCheckpointInfo* p = taosArrayGet(pList, i);
for (int32_t i = 0; i < num; ++i) {
SFailedCheckpointInfo *p = taosArrayGet(pList, i);
if (p->transId == pInfo->transId) {
return;
}
@ -52,12 +52,13 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI
}
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint");
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
" reset from failed checkpoint");
if (pTrans == NULL) {
return terrno;
}
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);
@ -140,8 +141,8 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
return TSDB_CODE_SUCCESS;
}
static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
SOrphanTask* pTask = taosArrayGet(pList, 0);
static int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
SOrphanTask *pTask = taosArrayGet(pList, 0);
// check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false);
@ -150,7 +151,7 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
}
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) {
mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1;
@ -180,21 +181,21 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
return 0;
}
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void* pIter = NULL;
while(1) {
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
if(pStream->status != STREAM_STATUS__PAUSE){
if (pStream->status != STREAM_STATUS__PAUSE) {
SMPauseStreamReq reqPause = {0};
strcpy(reqPause.name, pStream->name);
reqPause.igNotExists = 1;
int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause);
void * pHead = rpcMallocCont(contLen);
void *pHead = rpcMallocCont(contLen);
tSerializeSMPauseStreamReq(pHead, contLen, &reqPause);
SRpcMsg rpcMsg = {
@ -205,7 +206,8 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
mInfo("receive pause stream:%s, %s, %"PRId64 ", because grant expired", pStream->name, reqPause.name, pStream->uid);
mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired", pStream->name, reqPause.name,
pStream->uid);
}
sdbRelease(pSdb, pStream);
@ -311,7 +313,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (allReady || snodeChanged) {
// if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
for(int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) {
SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedChkpt, i);
mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
pInfo->checkpointId, pInfo->transId);
@ -341,11 +343,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead* pHead = rsp.pCont;
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.vgId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
pReq->info.handle = NULL; // disable auto rsp
}
return TSDB_CODE_SUCCESS;

36
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -17,21 +17,21 @@
#include "mndTrans.h"
typedef struct SKeyInfo {
void* pKey;
void *pKey;
int32_t keyLen;
} SKeyInfo;
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) {
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId) {
SStreamTransInfo info = {
.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId};
taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo));
return 0;
}
int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) {
int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) {
size_t keyLen = 0;
void* pIter = NULL;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
void *pIter = NULL;
SArray *pList = taosArrayInit(4, sizeof(SKeyInfo));
int32_t num = 0;
while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
@ -43,7 +43,8 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
void *pKey = taosHashGetKey(pEntry, &keyLen);
// key is the name of src/dst db name
SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime);
mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
pEntry->startTime);
taosArrayPush(pList, &info);
} else {
if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) {
@ -55,7 +56,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
int32_t size = taosArrayGetSize(pList);
for (int32_t i = 0; i < size; ++i) {
SKeyInfo* pKey = taosArrayGet(pList, i);
SKeyInfo *pKey = taosArrayGet(pList, i);
taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
}
@ -76,7 +77,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
// For a given stream:
// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans.
// 2. create/drop/reset/update trans are conflict with any other trans.
bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) {
bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) {
if (lock) {
taosThreadMutexLock(&execInfo.lock);
}
@ -127,7 +128,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
return false;
}
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) {
@ -136,7 +137,7 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
}
mndStreamClearFinishedTrans(pMnode, NULL);
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock);
@ -152,7 +153,8 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
return 0;
}
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) {
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@ -208,7 +210,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
terrno = TSDB_CODE_SUCCESS;
STREAM_ENCODE_OVER:
STREAM_ENCODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr());
@ -248,12 +250,16 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode,
STransAction action = {.epSet = *pEpset,
.contLen = contLen,
.pCont = pCont,
.msgType = msgType,
.retryCode = retryCode,
.acceptableCode = acceptCode};
return mndTransAppendRedoAction(pTrans, &action);
}
static bool identicalName(const char* pDb, const char* pParam, int32_t len) {
static bool identicalName(const char *pDb, const char *pParam, int32_t len) {
return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0);
}
@ -300,5 +306,3 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) {
mDebug("complete clear checkpoints in Dbs");
}

265
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -41,7 +41,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg);
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg);
static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* hash);
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash);
static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) {
SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
@ -95,13 +95,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
return pSub;
}
static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg, SSubplan* pPlan) {
static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg,
SSubplan *pPlan) {
SMqRebVgReq req = {0};
req.oldConsumerId = pRebVg->oldConsumerId;
req.newConsumerId = pRebVg->newConsumerId;
req.vgId = pRebVg->pVgEp->vgId;
if(pPlan){
if (pPlan) {
pPlan->execNode.epSet = pRebVg->pVgEp->epSet;
pPlan->execNode.nodeId = pRebVg->pVgEp->vgId;
int32_t msgLen;
@ -109,7 +109,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
}else{
} else {
req.qmsg = taosStrdup("");
}
req.subType = pSub->subType;
@ -126,7 +126,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
}
tlen += sizeof(SMsgHead);
void *buf = taosMemoryMalloc(tlen);
void *buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(req.qmsg);
@ -155,9 +155,9 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
}
static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg, SSubplan* pPlan) {
const SMqRebOutputVg *pRebVg, SSubplan *pPlan) {
if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
if(pRebVg->oldConsumerId == -1) return 0; //drop stream, no consumer, while split vnode,all consumerId is -1
if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1
terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
return -1;
}
@ -223,19 +223,18 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
return pRebInfo;
}
static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char* key){
static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(vgs);
SMqRebOutputVg outputVg = {consumerId, -1, pVgEp};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64,
key, pVgEp->vgId, consumerId);
mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, key, pVgEp->vgId, consumerId);
}
static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t actualRemoved = 0;
for (int32_t i = 0; i < numOfRemoved; i++) {
uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
if (pConsumerEp == NULL) {
continue;
@ -253,14 +252,15 @@ static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, c
}
if (numOfRemoved != actualRemoved) {
mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved, actualRemoved);
mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved,
actualRemoved);
} else {
mInfo("[rebalance] sub:%s removed %d consumers", pOutput->pSub->key, numOfRemoved);
}
}
static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) {
int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers);
int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers);
for (int32_t i = 0; i < numOfNewConsumers; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
@ -276,26 +276,26 @@ static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *
}
static void processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs);
int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs);
for (int32_t i = 0; i < numOfVgroups; i++) {
pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key);
}
}
//static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){
// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){
// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i);
// SMqRebOutputVg outputVg = {
// .oldConsumerId = pConsumerEp->consumerId,
// .newConsumerId = pConsumerEp->consumerId,
// .pVgEp = pVgEp,
// };
// taosArrayPush(pOutput->rebVgs, &outputVg);
// }
//}
// static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){
// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){
// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i);
// SMqRebOutputVg outputVg = {
// .oldConsumerId = pConsumerEp->consumerId,
// .newConsumerId = pConsumerEp->consumerId,
// .pVgEp = pVgEp,
// };
// taosArrayPush(pOutput->rebVgs, &outputVg);
// }
// }
static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt,
int32_t remainderVgCnt) {
int32_t remainderVgCnt) {
int32_t cnt = 0;
void *pIter = NULL;
@ -311,7 +311,7 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash,
taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) {
if (cnt < remainderVgCnt) {
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1
pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key);
}
cnt++;
@ -321,17 +321,17 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash,
}
}
}
// putNoTransferToOutput(pOutput, pConsumerEp);
// putNoTransferToOutput(pOutput, pConsumerEp);
}
}
static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = 0;
SVgObj* pVgroup = NULL;
void* pIter = NULL;
SArray* newVgs = taosArrayInit(0, POINTER_BYTES);
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SArray *newVgs = taosArrayInit(0, POINTER_BYTES);
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) {
break;
}
@ -342,7 +342,7 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
}
totalVgNum++;
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId;
taosArrayPush(newVgs, &pVgEp);
@ -358,17 +358,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
int32_t j = 0;
while (j < taosArrayGetSize(pConsumerEp->vgs)) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
bool find = false;
for(int32_t k = 0; k < taosArrayGetSize(newVgs); k++){
bool find = false;
for (int32_t k = 0; k < taosArrayGetSize(newVgs); k++) {
SMqVgEp *pnewVgEp = taosArrayGetP(newVgs, k);
if(pVgEp->vgId == pnewVgEp->vgId){
if (pVgEp->vgId == pnewVgEp->vgId) {
tDeleteSMqVgEp(pnewVgEp);
taosArrayRemove(newVgs, k);
find = true;
break;
}
}
if(!find){
if (!find) {
mInfo("[rebalance] processRemoveAddVgs old vgId:%d", pVgEp->vgId);
tDeleteSMqVgEp(pVgEp);
taosArrayRemove(pConsumerEp->vgs, j);
@ -378,17 +378,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
}
}
if(taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0){
if (taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0) {
taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs);
mInfo("[rebalance] processRemoveAddVgs add new vg num:%d", (int)taosArrayGetSize(newVgs));
taosArrayDestroy(newVgs);
}else{
} else {
taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp);
}
return totalVgNum;
}
static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput){
static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows
if (pSub == NULL) {
return;
@ -406,16 +406,17 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) {
OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j);
bool jump = false;
for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){
bool jump = false;
for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i);
if(pVgEp->vgId == d1->vgId){
if (pVgEp->vgId == d1->vgId) {
jump = true;
mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId);
mInfo("pSub->offsetRows jump, because consumer id:0x%" PRIx64 " and vgId:%d not change",
pConsumerEp->consumerId, pVgEp->vgId);
break;
}
}
if(jump) continue;
if (jump) continue;
bool find = false;
for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) {
OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i);
@ -424,11 +425,11 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
d2->offset = d1->offset;
d2->ever = d1->ever;
find = true;
mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows);
mInfo("pSub->offsetRows add vgId:%d, after:%" PRId64 ", before:%" PRId64, d2->vgId, d2->rows, d1->rows);
break;
}
}
if(!find){
if (!find) {
taosArrayPush(pOutput->pSub->offsetRows, d1);
}
}
@ -437,7 +438,7 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
mndReleaseSubscribe(pMnode, pSub);
}
static void printRebalanceLog(SMqRebOutputObj *pOutput){
static void printRebalanceLog(SMqRebOutputObj *pOutput) {
mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key);
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
@ -451,7 +452,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){
if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key, pConsumerEp->consumerId, sz);
mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key,
pConsumerEp->consumerId, sz);
for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
mInfo("sub:%s mq rebalance final cfg: vg %d to consumer:0x%" PRIx64, pOutput->pSub->key, pVgEp->vgId,
@ -460,8 +462,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){
}
}
static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey,
int32_t *minVgCnt, int32_t *remainderVgCnt){
static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, int32_t *minVgCnt,
int32_t *remainderVgCnt) {
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers);
int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved;
@ -473,11 +475,12 @@ static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, con
} else {
mInfo("[rebalance] sub:%s no consumer subscribe this topic", pSubKey);
}
mInfo("[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d",
pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt);
mInfo(
"[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d",
pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt);
}
static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt){
static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) {
SMqRebOutputVg *pRebVg = NULL;
void *pAssignIter = NULL;
void *pIter = NULL;
@ -498,8 +501,8 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
pRebVg = (SMqRebOutputVg *)pAssignIter;
pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average",
pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
}
}
@ -519,13 +522,13 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
pRebVg = (SMqRebOutputVg *)pAssignIter;
pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1",
pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
}
}
taosHashCancelIterate(pOutput->pSub->consumerHash, pIter);
if(pAssignIter != NULL){
if (pAssignIter != NULL) {
mError("[rebalance]sub:%s assign iter is not NULL, never should reach here", pOutput->pSub->key);
}
while (1) {
@ -534,19 +537,19 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
break;
}
SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pAssignIter;
SMqRebOutputVg *pRebOutput = (SMqRebOutputVg *)pAssignIter;
taosArrayPush(pOutput->rebVgs, pRebOutput);
if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed
if (taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned
}
}
}
static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput);
int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput);
const char *pSubKey = pOutput->pSub->key;
int32_t minVgCnt = 0;
int32_t remainderVgCnt = 0;
int32_t minVgCnt = 0;
int32_t remainderVgCnt = 0;
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@ -561,14 +564,14 @@ static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebO
taosHashCleanup(pHash);
}
static int32_t presistConsumerByType(STrans *pTrans, SArray* consumers, int8_t type, char *cgroup, char *topic){
int32_t code = 0;
SMqConsumerObj *pConsumerNew = NULL;
int32_t consumerNum = taosArrayGetSize(consumers);
static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t type, char *cgroup, char *topic) {
int32_t code = 0;
SMqConsumerObj *pConsumerNew = NULL;
int32_t consumerNum = taosArrayGetSize(consumers);
for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(consumers, i);
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, type, topic, NULL);
if (pConsumerNew == NULL){
if (pConsumerNew == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
@ -587,7 +590,7 @@ END:
return code;
}
static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic){
static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic) {
int32_t code = presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL);
if (code != 0) {
return code;
@ -603,10 +606,10 @@ static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput
static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
struct SSubplan *pPlan = NULL;
int32_t code = 0;
int32_t code = 0;
STrans *pTrans = NULL;
if(strcmp(pOutput->pSub->qmsg, "") != 0){
if (strcmp(pOutput->pSub->qmsg, "") != 0) {
code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan);
if (code != 0) {
terrno = code;
@ -660,7 +663,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
code = mndTransPrepare(pMnode, pTrans);
END:
nodesDestroyNode((SNode*)pPlan);
nodesDestroyNode((SNode *)pPlan);
mndTransDrop(pTrans);
return code;
}
@ -672,24 +675,24 @@ static void freeRebalanceItem(void *param) {
}
// type = 0 remove type = 1 add
static void buildRebInfo(SHashObj* rebSubHash, SArray* topicList, int8_t type, char *group, int64_t consumerId){
static void buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) {
int32_t topicNum = taosArrayGetSize(topicList);
for (int32_t i = 0; i < topicNum; i++) {
char key[TSDB_SUBSCRIBE_KEY_LEN];
char *removedTopic = taosArrayGetP(topicList, i);
mndMakeSubscribeKey(key, group, removedTopic);
SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key);
if(type == 0)
if (type == 0)
taosArrayPush(pRebSub->removedConsumers, &consumerId);
else if(type == 1)
else if (type == 1)
taosArrayPush(pRebSub->newConsumers, &consumerId);
}
}
static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj* rebSubHash){
static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj *rebSubHash) {
int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < newTopicNum; i++) {
char * topic = taosArrayGetP(pConsumer->currentTopics, i);
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
if (pSub == NULL) {
continue;
@ -702,7 +705,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash
for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (!pVgroup) {
char key[TSDB_SUBSCRIBE_KEY_LEN];
mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
@ -716,7 +719,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash
}
}
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) {
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) {
SMnode *pMnode = pMsg->info.node;
SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = NULL;
@ -732,17 +735,19 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) {
int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
int32_t status = atomic_load_32(&pConsumer->status);
mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d",
pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus);
mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64
", hbstatus:%d",
pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime,
pConsumer->createTime, hbStatus);
if (status == MQ_CONSUMER_STATUS_READY) {
if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {// unsubscribe or close
if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info);
} else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) {
taosRLockLatch(&pConsumer->lock);
buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId);
taosRUnLockLatch(&pConsumer->lock);
}else{
} else {
checkForVgroupSplit(pMnode, pConsumer, rebSubHash);
}
} else if (status == MQ_CONSUMER_STATUS_LOST) {
@ -762,8 +767,7 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) {
bool mndRebTryStart() {
int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
if (old > 0) mInfo("[rebalance] counter old val:%d", old)
return old == 0;
if (old > 0) mInfo("[rebalance] counter old val:%d", old) return old == 0;
}
void mndRebCntInc() {
@ -776,7 +780,7 @@ void mndRebCntDec() {
if (val > 0) mInfo("[rebalance] cnt sub, value:%d", val)
}
static void clearRebOutput(SMqRebOutputObj *rebOutput){
static void clearRebOutput(SMqRebOutputObj *rebOutput) {
taosArrayDestroy(rebOutput->newConsumers);
taosArrayDestroy(rebOutput->modifyConsumers);
taosArrayDestroy(rebOutput->removedConsumers);
@ -800,15 +804,15 @@ static int32_t initRebOutput(SMqRebOutputObj *rebOutput) {
}
// This function only works when there are dirty consumers
static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){
void *pIter = NULL;
static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) {
void *pIter = NULL;
while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) {
break;
}
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId);
if (pConsumer != NULL) {
mndReleaseConsumer(pMnode, pConsumer);
@ -822,8 +826,8 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){
}
}
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput){
const char *key = rebInput->pRebInfo->key;
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) {
const char *key = rebInput->pRebInfo->key;
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key);
if (pSub == NULL) {
@ -869,14 +873,13 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
}
static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
int code = 0;
void *pIter = NULL;
SMnode *pMnode = pMsg->info.node;
int code = 0;
void *pIter = NULL;
SMnode *pMnode = pMsg->info.node;
mDebug("[rebalance] start to process mq timer")
if (!mndRebTryStart()) {
mInfo("[rebalance] mq rebalance already in progress, do nothing")
return code;
if (!mndRebTryStart()) {
mInfo("[rebalance] mq rebalance already in progress, do nothing") return code;
}
SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
@ -900,13 +903,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SMqRebInputObj rebInput = {0};
SMqRebOutputObj rebOutput = {0};
code = initRebOutput(&rebOutput);
if (code != 0){
if (code != 0) {
goto END;
}
rebInput.pRebInfo = (SMqRebInfo*)pIter;
rebInput.pRebInfo = (SMqRebInfo *)pIter;
if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0){
if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0) {
continue;
}
@ -931,12 +934,12 @@ END:
return code;
}
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){
void* pIter = NULL;
SVgObj* pVgObj = NULL;
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) {
void *pIter = NULL;
SVgObj *pVgObj = NULL;
int32_t ret = 0;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj);
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgObj);
if (pIter == NULL) {
break;
}
@ -946,7 +949,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
continue;
}
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if(pReq == NULL){
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
ret = -1;
goto END;
@ -957,7 +960,8 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);;
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
;
action.pCont = pReq;
action.contLen = sizeof(SMqVDeleteReq);
action.msgType = TDMT_VND_TMQ_DELETE_SUB;
@ -969,13 +973,13 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
goto END;
}
}
END:
END:
sdbRelease(pMnode->pSdb, pVgObj);
sdbCancelFetch(pMnode->pSdb, pIter);
return ret;
}
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic){
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) {
void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL;
int ret = 0;
@ -1012,10 +1016,10 @@ END:
}
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
SMDropCgroupReq dropReq = {0};
STrans *pTrans = NULL;
int32_t code = TSDB_CODE_ACTION_IN_PROGRESS;
SMnode *pMnode = pMsg->info.node;
SMDropCgroupReq dropReq = {0};
STrans *pTrans = NULL;
int32_t code = TSDB_CODE_ACTION_IN_PROGRESS;
if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@ -1262,7 +1266,6 @@ int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) {
pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub);
if (pIter == NULL) break;
char topic[TSDB_TOPIC_FNAME_LEN] = {0};
char cgroup[TSDB_CGROUP_LEN] = {0};
mndSplitSubscribeKey(pSub->key, topic, cgroup, true);
@ -1340,7 +1343,8 @@ END:
return code;
}
static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t consumerId, const char* topic, const char* cgroup, SArray* vgs, SArray *offsetRows){
static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic,
const char *cgroup, SArray *vgs, SArray *offsetRows) {
int32_t sz = taosArrayGetSize(vgs);
for (int32_t j = 0; j < sz; j++) {
SMqVgEp *pVgEp = taosArrayGetP(vgs, j);
@ -1359,37 +1363,37 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons
colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false);
// consumer id
char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, consumerId);
char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId);
varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1);
mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
consumerId, varDataVal(cgroup), pVgEp->vgId);
mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), consumerId,
varDataVal(cgroup), pVgEp->vgId);
// offset
OffsetRows *data = NULL;
for(int i = 0; i < taosArrayGetSize(offsetRows); i++){
for (int i = 0; i < taosArrayGetSize(offsetRows); i++) {
OffsetRows *tmp = taosArrayGet(offsetRows, i);
if(tmp->vgId != pVgEp->vgId){
if (tmp->vgId != pVgEp->vgId) {
mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId);
continue;
}
data = tmp;
}
if(data){
if (data) {
// vg id
char buf[TSDB_OFFSET_LEN*2 + VARSTR_HEADER_SIZE] = {0};
char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0};
tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset);
sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%"PRId64, data->ever);
sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever);
varDataSetLen(buf, strlen(varDataVal(buf)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false);
}else{
} else {
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, *numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1435,7 +1439,8 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock
if (pIter == NULL) break;
pConsumerEp = (SMqConsumerEp *)pIter;
buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, pConsumerEp->offsetRows);
buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs,
pConsumerEp->offsetRows);
}
buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows);

35
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -900,6 +900,41 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) {
return 0;
}
int32_t mndTransCheckConflictWithCompact(SMnode *pMnode, STrans *pTrans) {
void *pIter = NULL;
bool conflict = false;
SCompactObj *pCompact = NULL;
while (1) {
bool thisConflict = false;
pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT, pIter, (void **)&pCompact);
if (pIter == NULL) break;
if (pTrans->conflict == TRN_CONFLICT_GLOBAL || pTrans->conflict == TRN_CONFLICT_DB ||
pTrans->conflict == TRN_CONFLICT_DB_INSIDE) {
if (strcasecmp(pTrans->dbname, pCompact->dbname) == 0) thisConflict = true;
}
if (thisConflict) {
mError("trans:%d, db:%s stb:%s type:%d, can't execute since conflict with compact:%d db:%s", pTrans->id,
pTrans->dbname, pTrans->stbname, pTrans->conflict, pCompact->compactId, pCompact->dbname);
conflict = true;
} else {
mInfo("trans:%d, db:%s stb:%s type:%d, not conflict with compact:%d db:%s", pTrans->id, pTrans->dbname,
pTrans->stbname, pTrans->conflict, pCompact->compactId, pCompact->dbname);
}
sdbRelease(pMnode->pSdb, pCompact);
}
if (conflict) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT_COMPACT;
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
return terrno;
}
return 0;
}
static bool mndTransActionsOfSameType(SArray *pActions) {
int32_t size = taosArrayGetSize(pActions);
ETrnAct lastActType = TRANS_ACTION_NULL;

4
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -1952,6 +1952,10 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup");
if (pTrans == NULL) goto _OVER;
mndTransSetDbName(pTrans, pVgroup->dbName, NULL);
if (mndTransCheckConflictWithCompact(pMnode, pTrans) != 0) goto _OVER;
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId);

1
source/dnode/mnode/impl/test/CMakeLists.txt
View File

@ -16,3 +16,4 @@ add_subdirectory(stb)
add_subdirectory(topic)
add_subdirectory(trans)
#add_subdirectory(user)
#add_subdirectory(mnode)

20
source/dnode/mnode/impl/test/mnode/CMakeLists.txt
View File

@ -1,11 +1,11 @@
# aux_source_directory(. MNODE_MNODE_TEST_SRC)
# add_executable(mmnodeTest ${MNODE_MNODE_TEST_SRC})
# target_link_libraries(
# mmnodeTest
# PUBLIC sut
# )
aux_source_directory(. MNODE_MNODE_TEST_SRC)
add_executable(mmnodeTest ${MNODE_MNODE_TEST_SRC})
target_link_libraries(
mmnodeTest
PUBLIC sut
)
# add_test(
# NAME mmnodeTest
# COMMAND mmnodeTest
# )
add_test(
NAME mmnodeTest
COMMAND mmnodeTest
)

110
source/dnode/mnode/impl/test/mnode/mnode.cpp
View File

@ -280,4 +280,114 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) {
ASSERT_NE(retry, retryMax);
}
}
#define SLOW_LOG_TYPE_NULL 0x0
#define SLOW_LOG_TYPE_QUERY 0x1
#define SLOW_LOG_TYPE_INSERT 0x2
#define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0x7
void getSlowLogScopeString(int32_t scope, char* result){
if(scope == SLOW_LOG_TYPE_NULL) {
strcat(result, "NONE");
return;
}
while(scope > 0){
if(scope & SLOW_LOG_TYPE_QUERY) {
strcat(result, "QUERY");
scope &= ~SLOW_LOG_TYPE_QUERY;
} else if(scope & SLOW_LOG_TYPE_INSERT) {
strcat(result, "INSERT");
scope &= ~SLOW_LOG_TYPE_INSERT;
} else if(scope & SLOW_LOG_TYPE_OTHERS) {
strcat(result, "OTHERS");
scope &= ~SLOW_LOG_TYPE_OTHERS;
} else{
printf("invalid slow log scope:%d", scope);
return;
}
if(scope > 0) {
strcat(result, "|");
}
}
}
// Define test cases
TEST_F(MndTestMnode, ScopeIsNull) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_NULL, result);
// Assert
EXPECT_STREQ(result, "NONE");
}
TEST_F(MndTestMnode, ScopeIsQuery) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_QUERY, result);
// Assert
EXPECT_STREQ(result, "QUERY");
}
TEST_F(MndTestMnode, ScopeIsInsert) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_INSERT, result);
// Assert
EXPECT_STREQ(result, "INSERT");
}
TEST_F(MndTestMnode, ScopeIsOthers) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_OTHERS, result);
// Assert
EXPECT_STREQ(result, "OTHERS");
}
TEST_F(MndTestMnode, ScopeIsMixed) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_OTHERS|SLOW_LOG_TYPE_INSERT, result);
// Assert
EXPECT_STREQ(result, "INSERT|OTHERS");
}
TEST_F(MndTestMnode, ScopeIsMixed1) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(SLOW_LOG_TYPE_ALL, result);
// Assert
EXPECT_STREQ(result, "QUERY|INSERT|OTHERS");
}
TEST_F(MndTestMnode, ScopeIsInvalid) {
// Arrange
char result[256] = {0};
// Act
getSlowLogScopeString(0xF000, result);
// Assert
EXPECT_STREQ(result, ""); // Expect an empty string since the scope is invalid
// You may also want to check if the error message is correctly logged
}

12
source/dnode/snode/src/snode.c
View File

@ -41,7 +41,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name;
char *p = streamTaskGetStatus(pTask)->name;
if (pTask->info.fillHistory) {
sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
@ -71,7 +71,8 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
startRsync();
pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback);
if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
@ -138,9 +139,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (pMsg->msgType) {
case TDMT_STREAM_TASK_DEPLOY: {
void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead);
return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb,pMsg->info.conn.applyIndex, pReq, len, true, true);
return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb, pMsg->info.conn.applyIndex, pReq, len, true,
true);
}
case TDMT_STREAM_TASK_DROP:
@ -159,4 +161,4 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ASSERT(0);
}
return 0;
}
}

10
source/dnode/vnode/src/meta/metaCache.c
View File

@ -686,8 +686,8 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int
}
// add to cache.
taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL,
TAOS_LRU_PRIORITY_LOW, NULL);
(void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL,
TAOS_LRU_PRIORITY_LOW, NULL);
_end:
taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
@ -909,7 +909,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
return TSDB_CODE_SUCCESS;
}
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) {
bool metaTbInFilterCache(SMeta* pMeta, const void* key, int8_t type) {
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) {
return true;
}
@ -921,7 +921,7 @@ bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) {
return false;
}
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) {
int32_t metaPutTbToFilterCache(SMeta* pMeta, const void* key, int8_t type) {
if (type == 0) {
return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0);
}
@ -933,7 +933,7 @@ int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) {
return 0;
}
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) {
int32_t metaSizeOfTbFilterCache(SMeta* pMeta, int8_t type) {
if (type == 0) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
}

88
source/dnode/vnode/src/meta/metaTable.c
View File

@ -647,13 +647,18 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SCtbIdxKey *table = (SCtbIdxKey *)pKey;
STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal);
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes;
}
}
rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey);
tdbFree(pKey);
@ -779,13 +784,18 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
SCtbIdxKey *table = (SCtbIdxKey *)pKey;
STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal);
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes;
}
}
rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey);
tdbFree(pKey);
@ -1272,13 +1282,18 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
int32_t nTagData = 0;
STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)e.ctbEntry.pTags, &tagVal);
if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
if (tTagGet((const STag *)e.ctbEntry.pTags, &tagVal)) {
if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
}
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
if (!IS_VAR_DATA_TYPE(pTagColumn->type)) {
nTagData = tDataTypes[pTagColumn->type].bytes;
}
}
if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid,
@ -2036,13 +2051,18 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb
int32_t nTagData = 0;
STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal);
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes;
}
}
if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) {
tdbFree(pKey);
@ -2497,19 +2517,25 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
goto end;
} else {
for (int i = 0; i < pTagSchema->nCols; i++) {
pTagData = NULL;
nTagData = 0;
pTagColumn = &pTagSchema->pSchema[i];
if (!IS_IDX_ON(pTagColumn)) continue;
STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal);
if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) {
if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
}
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
if (!IS_VAR_DATA_TYPE(pTagColumn->type)) {
nTagData = tDataTypes[pTagColumn->type].bytes;
}
}
if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
ret = -1;

5
source/dnode/vnode/src/tq/tq.c
View File

@ -697,6 +697,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} else {
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
req.newConsumerId);
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
atomic_store_32(&pHandle->epoch, 0);
tqUnregisterPushHandle(pTq, pHandle);
@ -963,8 +964,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr);
tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
tqDebug("s-task:%s fill-history task set status to be dropping", id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
streamMetaReleaseTask(pMeta, pTask);

59
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -19,13 +19,13 @@
typedef struct SMStreamCheckpointReadyRspMsg {
SMsgHead head;
int64_t streamId;
int32_t upstreamTaskId;
int32_t upstreamNodeId;
int32_t downstreamTaskId;
int32_t downstreamNodeId;
int64_t checkpointId;
int32_t transId;
int64_t streamId;
int32_t upstreamTaskId;
int32_t upstreamNodeId;
int32_t downstreamTaskId;
int32_t downstreamNodeId;
int64_t checkpointId;
int32_t transId;
} SMStreamCheckpointReadyRspMsg;
static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
@ -160,7 +160,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL || *ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, req.taskId);
tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId,
req.taskId);
rsp.code = TSDB_CODE_SUCCESS;
streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList);
@ -170,7 +171,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SStreamTask* pTask = *ppTask;
const char* idstr = pTask->id.idStr;
if ((pMeta->updateInfo.transId != req.transId) && (pMeta->updateInfo.transId != -1)) {
if (pMeta->updateInfo.transId == -1) { // info needs to be kept till the new trans to update the nodeEp arrived.
streamMetaInitUpdateTaskList(pMeta, req.transId);
}
if (pMeta->updateInfo.transId != req.transId) {
if (req.transId < pMeta->updateInfo.transId) {
tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId,
pMeta->updateInfo.transId, req.transId);
@ -194,7 +199,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
if (pReqTask != NULL) {
tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, req.transId);
tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId,
req.transId);
rsp.code = TSDB_CODE_SUCCESS;
streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList);
@ -838,10 +844,11 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks
SStreamTaskState *pState = streamTaskGetStatus(pTask);
SStreamTaskState* pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) {
int32_t tranId = 0;
int64_t activeChkId = 0;
@ -866,7 +873,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
}
int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont;
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId);
if (pTask == NULL) {
@ -883,7 +890,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
@ -895,14 +903,15 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
int64_t checkpointId = 0;
streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId);
ASSERT (checkpointId == pReq->checkpointId);
ASSERT(checkpointId == pReq->checkpointId);
if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) {
// re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0;
streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
@ -914,7 +923,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger",
pTask->id.idStr, recv, total);
}
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
}
} else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT);
@ -922,7 +932,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger",
pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
}
streamMetaReleaseTask(pMeta, pTask);
@ -930,7 +941,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
}
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SCheckpointTriggerRsp* pRsp = pMsg->pCont;
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId);
if (pTask == NULL) {
@ -940,8 +951,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
}
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%"PRId64", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask);
@ -998,7 +1009,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
int32_t level = pTask->info.taskLevel;
if (level == TASK_LEVEL__SINK) {
ASSERT (status != TASK_STATUS__UNINIT);
ASSERT(status != TASK_STATUS__UNINIT);
streamMetaReleaseTask(pMeta, pTask);
return 0;
}
@ -1025,7 +1036,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
streamTrySchedExec(pTask);
}
} else {
ASSERT (status != TASK_STATUS__UNINIT);
ASSERT(status != TASK_STATUS__UNINIT);
}
streamMetaReleaseTask(pMeta, pTask);
@ -1063,7 +1074,7 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);}
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;

1
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -771,6 +771,7 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap
STSchema *pTSchema = NULL;
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema);
if (code != TSDB_CODE_SUCCESS) {
taosThreadMutexUnlock(&pTsdb->lruMutex);
terrno = code;
return -1;
}

96
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -49,7 +49,8 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
STsdbReader* pReader);
static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader);
static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList,
@ -68,7 +69,8 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRo
STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow);
static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,
STsdbReader* pReader);
static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
STsdbReader* pReader);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);
static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
@ -126,7 +128,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_
}
// for test purpose, todo remove it
static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) {
static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) {
int32_t n = 0;
n += tGetI8(p + n, &index->type);
n += tGetU32v(p + n, &index->offset);
@ -137,7 +139,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
uint8_t *data = pRow->data;
uint8_t* data = pRow->data;
for (int32_t i = 0; i < pRow->numOfPKs; i++) {
data += tGetPrimaryKeyIndex(data, &indices[i]);
}
@ -146,7 +148,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
for (int32_t i = 0; i < pRow->numOfPKs; i++) {
pKey->pks[i].type = indices[i].type;
uint8_t *tdata = data + indices[i].offset;
uint8_t* tdata = data + indices[i].offset;
if (pRow->flag >> 4) {
tdata += tGetI16v(tdata, NULL);
}
@ -363,9 +365,7 @@ _err:
return code;
}
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp) {
return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type);
}
bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); }
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) {
pIter->order = order;
@ -927,6 +927,10 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData
} else {
int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey;
endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order);
if (endPos == -1) {
return endPos;
}
endPos = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, endPos, asc);
}
@ -1077,10 +1081,10 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf
pLast->type = pSupp->pk.type;
if (IS_VAR_DATA_TYPE(pFirst->type)) {
pFirst->pData = (uint8_t*) varDataVal(pBlockInfo->firstPk.pData);
pFirst->pData = (uint8_t*)varDataVal(pBlockInfo->firstPk.pData);
pFirst->nData = varDataLen(pBlockInfo->firstPk.pData);
pLast->pData = (uint8_t*) varDataVal(pBlockInfo->lastPk.pData);
pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData);
pLast->nData = varDataLen(pBlockInfo->lastPk.pData);
} else {
pFirst->val = pBlockInfo->firstPk.val;
@ -1417,7 +1421,8 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
}
// todo: this attribute could be acquired during extractin the global ordered block list.
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType, int32_t numOfPk) {
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType,
int32_t numOfPk) {
// it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) {
if (pBlock->lastKey == pRec->firstKey.key.ts) {
@ -1429,7 +1434,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
v1.val = pBlock->lastPk.val;
}
return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);
} else { // no pk
} else { // no pk
return true;
}
} else {
@ -1445,7 +1450,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
v1.val = pBlock->firstPk.val;
}
return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);
} else { // no pk
} else { // no pk
return true;
}
} else {
@ -1585,7 +1590,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
*copied = false;
bool asc = (pReader->info.order == TSDB_ORDER_ASC);
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1 : -1;
int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
SRowKey nextRowKey;
tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey);
@ -2077,7 +2082,8 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
return code;
}
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo,
STsdbReader* pReader) {
SRowKey rowKey = {0};
while (1) {
@ -2119,7 +2125,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
startKey.key = pBlockScanInfo->lastProcKey;
startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer;
if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) {
startKey.key.ts = asc? pWindow->skey:pWindow->ekey;
startKey.key.ts = asc ? pWindow->skey : pWindow->ekey;
forward = false;
}
@ -2168,8 +2174,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable
return false;
}
if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf
SRowKey nextRowKey; // lazy eval
if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf
SRowKey nextRowKey; // lazy eval
tColRowGetKey(pBlockData, rowIndex, &nextRowKey);
if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) {
return false;
@ -2268,12 +2274,12 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA;
SRowKey* p = asc? &pScanInfo->sttRange.skey:&pScanInfo->sttRange.ekey;
SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey;
tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p);
hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
} else { // not clean stt blocks
INIT_KEYRANGE(&pScanInfo->sttRange); //reset the time window
} else { // not clean stt blocks
INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window
pScanInfo->sttBlockReturned = false;
hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
}
@ -2362,8 +2368,9 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn
TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) {
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid,
fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData, pReader->idStr);
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow,
pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData,
pReader->idStr);
}
int32_t code =
@ -2513,7 +2520,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
return code;
}
STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
STableBlockScanInfo* pBlockScanInfo =
getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
if (pBlockScanInfo == NULL) {
goto _end;
}
@ -3560,7 +3568,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
int32_t order = pReader->info.order;
TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter);
TSDBKEY key;
TSDBKEY key;
TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false;
@ -3572,7 +3580,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow;
} else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0);
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0);
if (!dropped) {
return pRow;
}
@ -3597,7 +3606,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow;
} else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0);
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0);
if (!dropped) {
return pRow;
}
@ -3606,7 +3616,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
}
}
int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArray* pDelList, STsdbReader* pReader) {
int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) {
SRowMerger* pMerger = &pReader->status.merger;
while (1) {
@ -3698,7 +3708,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
return code;
}
int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader) {
int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SRowMerger* pMerger = &pReader->status.merger;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
@ -3707,8 +3718,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
pDumpInfo->rowIndex += step;
if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) {
pDumpInfo->rowIndex =
doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step);
pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step);
}
// all rows are consumed, let's try next file block
@ -3912,8 +3922,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRow
static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow,
int64_t endKey, bool* freeTSRow) {
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
SArray* pDelList = pBlockScanInfo->delSkyline;
uint64_t uid = pBlockScanInfo->uid;
@ -4484,8 +4494,8 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) {
while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) {
STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;
clearBlockScanInfo(pInfo);
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step;
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step;
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step;
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step;
}
pStatus->uidList.currentIndex = 0;
@ -4815,7 +4825,6 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
return code;
}
if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
// prepare for the next row scan
int32_t step = -1;
@ -4956,7 +4965,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
if (pResBlock->pBlockAgg == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for(int i = 0; i < num; ++i) {
for (int i = 0; i < num; ++i) {
pResBlock->pBlockAgg[i].colId = -1;
}
}
@ -4976,6 +4985,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
} else if (pAgg->colId < pSup->colId[j]) {
i += 1;
} else if (pSup->colId[j] < pAgg->colId) {
pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1;
*allHave = false;
j += 1;
}
@ -5034,9 +5044,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
SReaderStatus* pStatus = &pTReader->status;
if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
return pTReader->resBlockInfo.pResBlock;
}
@ -5046,8 +5055,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader);
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
// tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader);
return ret;
}
@ -5466,8 +5475,7 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) {
pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr;
}
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/
}
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ }
void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; }

1
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1103,6 +1103,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
if (vnodeValidateTableHash(pVnode, tbName) < 0) {
cRsp.code = TSDB_CODE_VND_HASH_MISMATCH;
taosArrayPush(rsp.pArray, &cRsp);
vError("vgId:%d create-table:%s failed due to hash value mismatch", TD_VID(pVnode), tbName);
continue;
}

2
source/libs/catalog/inc/catalogInt.h
View File

@ -363,7 +363,7 @@ typedef struct SCtgUserAuth {
} SCtgUserAuth;
typedef struct SCatalog {
uint64_t clusterId;
int64_t clusterId;
bool stopUpdate;
SDynViewVersion dynViewVer;
SHashObj* userCache; // key:user, value:SCtgUserAuth

2
source/libs/catalog/src/catalog.c
View File

@ -834,7 +834,7 @@ int32_t catalogInit(SCatalogCfg* cfg) {
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
int32_t catalogGetHandle(int64_t clusterId, SCatalog** catalogHandle) {
if (NULL == catalogHandle) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}

1
source/libs/catalog/src/ctgDbg.c
View File

@ -426,6 +426,7 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
break;
case CTG_DBG_VIEW_NUM:
num += ctgdGetViewNum(dbCache);
break;
default:
ctgError("invalid type:%d", type);
break;

4
source/libs/catalog/src/ctgUtil.c
View File

@ -410,7 +410,7 @@ void ctgFreeHandle(SCatalog* pCtg) {
return;
}
uint64_t clusterId = pCtg->clusterId;
int64_t clusterId = pCtg->clusterId;
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
@ -498,7 +498,7 @@ void ctgClearHandle(SCatalog* pCtg) {
return;
}
uint64_t clusterId = pCtg->clusterId;
int64_t clusterId = pCtg->clusterId;
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);

2
source/libs/executor/src/hashjoinoperator.c
View File

@ -116,7 +116,7 @@ int32_t hJoinLaunchPrimExpr(SSDataBlock* pBlock, SHJoinTableCtx* pTable, int32_t
SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId);
if (0 != pCtx->timezoneUnit) {
for (int32_t i = startIdx; i <= endIdx; ++i) {
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] / pCtx->truncateUnit * pCtx->truncateUnit - pCtx->timezoneUnit;
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] - pCtx->timezoneUnit) % pCtx->truncateUnit;
}
} else {
for (int32_t i = startIdx; i <= endIdx; ++i) {

11
source/libs/executor/src/mergejoin.c
View File

@ -99,7 +99,7 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp);
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
@ -225,7 +225,7 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp);
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
@ -291,7 +291,6 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break;
}
@ -682,7 +681,7 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp);
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
@ -1402,7 +1401,6 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break;
}
@ -1712,7 +1710,6 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break;
}
@ -2004,7 +2001,7 @@ int32_t mAsofBackwardDumpGrpCache(SMJoinWindowCtx* pCtx) {
}
buildGrp.endIdx = buildGrp.readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp);
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp));
pCtx->cache.outRowIdx += rowsLeft;
break;
}

2
source/libs/executor/src/mergejoinoperator.c
View File

@ -948,7 +948,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId);
if (0 != pCtx->timezoneUnit) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] / pCtx->truncateUnit * pCtx->truncateUnit - pCtx->timezoneUnit;
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit;
}
} else {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {

2
source/libs/executor/src/mergeoperator.c
View File

@ -182,7 +182,7 @@ SSDataBlock* doSortMerge(SOperatorInfo* pOperator) {
resetLimitInfoForNextGroup(&pInfo->limitInfo);
}
applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo);
(void)applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo);
if (p->info.rows > 0) {
break;

31
source/libs/executor/src/scanoperator.c
View File

@ -2265,13 +2265,16 @@ static void processPrimaryKey(SSDataBlock* pBlock, bool hasPrimaryKey, STqOffset
doBlockDataPrimaryKeyFilter(pBlock, offset);
SColumnInfoData* pColPk = taosArrayGet(pBlock->pDataBlock, 1);
if (pBlock->info.rows < 1) {
return ;
}
void* tmp = colDataGetData(pColPk, pBlock->info.rows - 1);
val.type = pColPk->info.type;
if(IS_VAR_DATA_TYPE(pColPk->info.type)) {
if (IS_VAR_DATA_TYPE(pColPk->info.type)) {
val.pData = taosMemoryMalloc(varDataLen(tmp));
val.nData = varDataLen(tmp);
memcpy(val.pData, varDataVal(tmp), varDataLen(tmp));
}else{
} else {
memcpy(&val.val, tmp, pColPk->info.bytes);
}
}
@ -2292,13 +2295,19 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
}
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
while (1) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
bool hasPrimaryKey = pAPI->tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader);
processPrimaryKey(pResult, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset);
qDebug("tmqsnap doQueueScan get data uid:%" PRId64 "", pResult->info.id.uid);
return pResult;
if (pResult && pResult->info.rows > 0) {
bool hasPrimaryKey = pAPI->tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader);
processPrimaryKey(pResult, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset);
qDebug("tmqsnap doQueueScan get data uid:%" PRId64 "", pResult->info.id.uid);
if (pResult->info.rows > 0) {
return pResult;
}
} else {
break;
}
}
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
@ -3902,17 +3911,19 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) {
if (pInput->rowIdx == -1) {
continue;
}
if (pInput->type == SUB_TABLE_MEM_BLOCK) {
pInput->rowIdx = 0;
pInput->pageIdx = -1;
}
pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock;
SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId);
pInput->aTs = (int64_t*)col->pData;
}
__merge_compare_fn_t mergeCompareFn = (!pSubTblsInfo->pPkOrderInfo) ? subTblRowCompareTsFn : subTblRowCompareTsPkFn;
tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn);
return TSDB_CODE_SUCCESS;
return tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn);
}
static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) {

57
source/libs/executor/src/streamfilloperator.c
View File

@ -172,10 +172,17 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
SWinKey key = {.ts = ts, .groupId = groupId};
void* curVal = NULL;
int32_t curVLen = 0;
bool hasCurKey = true;
int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts;
pFillSup->cur.pRowVal = curVal;
if (code == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = key.ts;
pFillSup->cur.pRowVal = curVal;
} else {
qDebug("streamStateFillGet key failed, Data may be deleted. ts:%" PRId64 ", groupId:%" PRId64, ts, groupId);
pFillSup->cur.key = ts;
pFillSup->cur.pRowVal = NULL;
hasCurKey = false;
}
SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId};
@ -187,8 +194,10 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
pFillSup->prev.key = preKey.ts;
pFillSup->prev.pRowVal = preVal;
code = pAPI->stateStore.streamStateCurNext(pState, pCur);
ASSERT(code == TSDB_CODE_SUCCESS);
if (hasCurKey) {
code = pAPI->stateStore.streamStateCurNext(pState, pCur);
ASSERT(code == TSDB_CODE_SUCCESS);
}
code = pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code != TSDB_CODE_SUCCESS) {
@ -741,8 +750,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
SWinKey key = {.ts = startTs, .groupId = groupId};
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
if (!pInfo->pFillInfo->needFill) {
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
} else {
STimeRange tw = {
@ -751,11 +760,27 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
.groupId = groupId,
};
taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
while (key.ts <= endTs) {
key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit,
pInfo->pFillSup->interval.precision);
tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
}
}
}
static void getWindowInfoByKey(SStorageAPI* pAPI, void* pState, TSKEY ts, int64_t groupId, SResultRowData* pWinData) {
SWinKey key = {.ts = ts, .groupId = groupId};
void* val = NULL;
int32_t len = 0;
int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&val, &len);
if (code != TSDB_CODE_SUCCESS) {
qDebug("get window info by key failed, Data may be deleted, try next window. ts:%" PRId64 ", groupId:%" PRId64, ts,
groupId);
SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &key, (const void**)&val, &len);
pAPI->stateStore.streamStateFreeCur(pCur);
qDebug("get window info by key ts:%" PRId64 ", groupId:%" PRId64 ", res%d", ts, groupId, code);
}
if (code == TSDB_CODE_SUCCESS) {
resetFillWindow(pWinData);
pWinData->key = key.ts;
pWinData->pRowVal = val;
}
}
@ -765,20 +790,22 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
int32_t size = taosArrayGetSize(pFillInfo->delRanges);
tSimpleHashClear(pInfo->pFillSup->pResMap);
for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) {
while (pFillInfo->delIndex < size) {
STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) {
return;
}
getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup);
TSKEY realEnd = range->ekey + 1;
if (pInfo->pFillInfo->type == TSDB_FILL_NEXT && pInfo->pFillSup->next.key != realEnd) {
getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, &pInfo->pFillSup->next);
}
setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo);
pFillInfo->delIndex++;
if (pInfo->pFillInfo->needFill) {
doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
pInfo->pRes->info.id.groupId = range->groupId;
}
SWinKey key = {.ts = range->skey, .groupId = range->groupId};
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
}
}

34
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -2632,7 +2632,7 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp
}
// 4.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
// 5.checksum
if (isParent) {
@ -3086,15 +3086,17 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
// for stream
void* buff = NULL;
int32_t len = 0;
int32_t res =
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) {
doStreamSessionDecodeOpState(buff, len, pOperator, true);
taosMemoryFree(buff);
if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
// for stream
void* buff = NULL;
int32_t len = 0;
int32_t res =
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) {
doStreamSessionDecodeOpState(buff, len, pOperator, true);
taosMemoryFree(buff);
}
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
@ -3279,6 +3281,16 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);
taosArrayPush(pInfo->pChildren, &pChildOp);
}
void* buff = NULL;
int32_t len = 0;
int32_t res =
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) {
doStreamSessionDecodeOpState(buff, len, pOperator, true);
taosMemoryFree(buff);
}
}
if (!IS_FINAL_SESSION_OP(pOperator) || numOfChild == 0) {
@ -3621,7 +3633,7 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
}
// 4.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
// 5.checksum
if (isParent) {

9
source/libs/function/src/builtins.c
View File

@ -2166,15 +2166,6 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (!validateTimestampDigits(pValue)) {
pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
}
// param1
if (numOfParams == 2) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);

36
source/libs/index/src/indexFilter.c
View File

@ -624,6 +624,23 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ
}
return 0;
}
static int8_t sifShouldUseIndexBasedOnType(SIFParam *left, SIFParam *right) {
// not compress
if (left->colValType == TSDB_DATA_TYPE_FLOAT) return 0;
if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY ||
left->colValType == TSDB_DATA_TYPE_JSON || right->colValType == TSDB_DATA_TYPE_JSON) {
return 0;
}
if (IS_VAR_DATA_TYPE(left->colValType)) {
if (!IS_VAR_DATA_TYPE(right->colValType)) return 0;
} else if (IS_NUMERIC_TYPE(left->colValType)) {
if (left->colValType != right->colValType) return 0;
}
return 1;
}
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
int ret = 0;
SIndexMetaArg *arg = &output->arg;
@ -641,8 +658,10 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
ret = indexJsonSearch(arg->ivtIdx, mtm, output->result);
indexMultiTermQueryDestroy(mtm);
} else {
if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY) {
return TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR;
int8_t useIndex = sifShouldUseIndexBasedOnType(left, right);
if (!useIndex) {
output->status = SFLT_NOT_INDEX;
return -1;
}
bool reverse = false, equal = false;
@ -660,15 +679,12 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
SDataTypeBuf typedata;
memset(&typedata, 0, sizeof(typedata));
if (IS_VAR_DATA_TYPE(left->colValType)) {
if (!IS_VAR_DATA_TYPE(right->colValType)) {
NUM_TO_STRING(right->colValType, right->condValue, sizeof(buf) - 2, buf + VARSTR_HEADER_SIZE);
varDataSetLen(buf, strlen(buf + VARSTR_HEADER_SIZE));
param.val = buf;
}
} else {
if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
if (sifSetFltParam(left, right, &typedata, &param) != 0) {
output->status = SFLT_NOT_INDEX;
return -1;
}
ret = left->api.metaFilterTableIds(arg->metaEx, &param, output->result);
if (ret == 0) {
taosArraySort(output->result, uidCompare);

210
source/libs/monitor/src/clientMonitor.c
View File

@ -1,210 +0,0 @@
#include "clientMonitor.h"
#include "os.h"
#include "tmisce.h"
#include "ttime.h"
#include "ttimer.h"
#include "tglobal.h"
SRWLatch monitorLock;
void* tmrClientMonitor;
tmr_h tmrStartHandle;
SHashObj* clusterMonitorInfoTable;
static int interval = 30 * 1000;
static int sendBathchSize = 1;
int32_t sendReport(ClientMonitor* pMonitor, char* pCont);
void generateClusterReport(ClientMonitor* pMonitor, bool send) {
char ts[50];
sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL);
if(NULL == pCont) {
uError("generateClusterReport failed, get null content.");
return;
}
if (send && strlen(pCont) != 0) {
if (sendReport(pMonitor, pCont) == 0) {
taos_collector_registry_clear_batch(pMonitor->registry);
}
}
taosMemoryFreeClear(pCont);
}
void reportSendProcess(void* param, void* tmrId) {
taosTmrReset(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor, &tmrStartHandle);
taosRLockLatch(&monitorLock);
static int index = 0;
index++;
ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL);
while (ppMonitor != NULL && *ppMonitor != NULL) {
ClientMonitor* pMonitor = *ppMonitor;
generateClusterReport(*ppMonitor, index == sendBathchSize);
ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor);
}
if (index == sendBathchSize) index = 0;
taosRUnLockLatch(&monitorLock);
}
void monitorClientInitOnce() {
static int8_t init = 0;
if (atomic_exchange_8(&init, 1) == 0) {
uInfo("tscMonitorInit once.");
clusterMonitorInfoTable =
(SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR");
tmrStartHandle = taosTmrStart(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor);
if(tsMonitorInterval < 1){
interval = 30 * 1000;
} else {
interval = tsMonitorInterval * 1000;
}
if (tsMonitorInterval < 10) {
sendBathchSize = (10 / sendBathchSize) + 1;
}
taosInitRWLatch(&monitorLock);
}
}
void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) {
if (clusterKey == NULL || strlen(clusterKey) == 0) {
uError("createMonitorClient failed, clusterKey is NULL");
return;
}
taosWLockLatch(&monitorLock);
if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) {
uInfo("createMonitorClient for %s.", clusterKey);
ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor));
snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey);
pMonitor->registry = taos_collector_registry_new(clusterKey);
pMonitor->colector = taos_collector_new(clusterKey);
epsetAssign(&pMonitor->epSet, &epSet);
pMonitor->pTransporter = pTransporter;
taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector);
pMonitor->counters =
(SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
taosHashPut(clusterMonitorInfoTable, clusterKey, strlen(clusterKey), &pMonitor, sizeof(ClientMonitor*));
uInfo("createMonitorClient for %s finished %p.", clusterKey, pMonitor);
}
taosWUnLockLatch(&monitorLock);
}
static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) {
static int32_t emptyRspNum = 0;
if (TSDB_CODE_SUCCESS != code) {
uError("found error in monitorReport send callback, code:%d, please check the network.", code);
}
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
return code;
}
int32_t sendReport(ClientMonitor* pMonitor, char* pCont) {
SStatisReq sStatisReq;
sStatisReq.pCont = pCont;
sStatisReq.contLen = strlen(pCont);
int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq);
if (tlen < 0) return 0;
void* buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
tSerializeSStatisReq(buf, tlen, &sStatisReq);
SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pInfo->fp = monitorReportAsyncCB;
pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_STATIS;
// pInfo->param = taosMemoryMalloc(sizeof(int32_t));
// *(int32_t*)pInfo->param = i;
pInfo->paramFreeFp = taosMemoryFree;
pInfo->requestId = tGenIdPI64();
pInfo->requestObjRefId = 0;
int64_t transporterId = 0;
return asyncSendMsgToServer(pMonitor->pTransporter, &pMonitor->epSet, &transporterId, pInfo);
}
void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter) {
monitorClientInitOnce();
createMonitorClient(clusterKey, epSet, pTransporter);
}
taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
const char** label_keys) {
ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
if (ppMonitor != NULL && *ppMonitor != NULL) {
ClientMonitor* pMonitor = *ppMonitor;
taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, name, strlen(name));
if (ppCounter != NULL && *ppCounter != NULL) {
taosHashRemove(pMonitor->counters, name, strlen(name));
uInfo("createClusterCounter remove old counter: %s.", name);
}
taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys);
if (newCounter != NULL) {
taos_collector_add_metric(pMonitor->colector, newCounter);
taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, sizeof(taos_counter_t*));
uInfo("createClusterCounter %s(%p):%s : %p.", pMonitor->clusterKey, pMonitor, name, newCounter);
return newCounter;
} else {
return NULL;
}
} else {
return NULL;
}
return NULL;
}
int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values) {
taosRLockLatch(&monitorLock);
ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
if (ppMonitor != NULL && *ppMonitor != NULL) {
ClientMonitor* pMonitor = *ppMonitor;
taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName));
if (ppCounter != NULL && *ppCounter != NULL) {
taos_counter_inc(*ppCounter, label_values);
} else {
uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName);
}
} else {
uError("taosClusterCounterInc not found pMonitor %s.", clusterKey);
}
taosRUnLockLatch(&monitorLock);
return 0;
}
void clusterMonitorClose(const char* clusterKey) {
taosWLockLatch(&monitorLock);
ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
if (ppMonitor != NULL && *ppMonitor != NULL) {
ClientMonitor* pMonitor = *ppMonitor;
uInfo("clusterMonitorClose valule:%p clusterKey:%s.", pMonitor, pMonitor->clusterKey);
taosHashCleanup(pMonitor->counters);
taos_collector_registry_destroy(pMonitor->registry);
taosMemoryFree(pMonitor);
taosHashRemove(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
}
taosWUnLockLatch(&monitorLock);
}
const char* resultStr(SQL_RESULT_CODE code) {
static const char* result_state[] = {"Success", "Failed", "Cancel"};
return result_state[code];
}

5
source/libs/monitor/src/monMain.c
View File

@ -24,6 +24,7 @@
SMonitor tsMonitor = {0};
char* tsMonUri = "/report";
char* tsMonFwUri = "/general-metric";
char* tsMonSlowLogUri = "/slow-sql-detail-batch";
char* tsMonFwBasicUri = "/taosd-cluster-basic";
void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
@ -631,7 +632,7 @@ void monGenAndSendReportBasic() {
monCleanupMonitorInfo(pMonitor);
}
void monSendContent(char *pCont) {
void monSendContent(char *pCont, const char* uri) {
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
if(tsMonitorLogProtocol){
if (pCont != NULL){
@ -640,7 +641,7 @@ void monSendContent(char *pCont) {
}
if (pCont != NULL) {
EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
if (taosSendHttpReport(tsMonitor.cfg.server, uri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
uError("failed to send monitor msg");
}
}

22
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -60,7 +60,7 @@ char* getFullJoinTypeString(EJoinType type, EJoinSubType stype) {
{"LEFT", "LEFT", "LEFT OUTER", "LEFT SEMI", "LEFT ANTI", "LEFT ANY", "LEFT ASOF", "LEFT WINDOW"},
{"RIGHT", "RIGHT", "RIGHT OUTER", "RIGHT SEMI", "RIGHT ANTI", "RIGHT ANY", "RIGHT ASOF", "RIGHT WINDOW"},
{"FULL", "FULL", "FULL OUTER", "FULL", "FULL", "FULL ANY", "FULL", "FULL"}
};
};
return joinFullType[type][stype];
}
@ -89,7 +89,7 @@ int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc) {
}
nodesDestroyNode(*ppSrc);
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
}
@ -422,6 +422,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
return makeNode(type, sizeof(SCreateSubTableClause));
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
return makeNode(type, sizeof(SCreateSubTableFromFileClause));
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return makeNode(type, sizeof(SCreateMultiTablesStmt));
case QUERY_NODE_DROP_TABLE_CLAUSE:
@ -1006,7 +1008,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pWin->pStartOffset);
nodesDestroyNode(pWin->pEndOffset);
break;
}
}
case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList);
@ -1089,6 +1091,20 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode((SNode*)pStmt->pOptions);
break;
}
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE: {
SCreateSubTableFromFileClause* pStmt = (SCreateSubTableFromFileClause*)pNode;
if (pStmt->aCreateTbData) {
taosArrayDestroy(pStmt->aCreateTbData);
}
if (pStmt->aTagIndexs) {
taosArrayDestroy(pStmt->aTagIndexs);
}
if (pStmt->fp) {
taosCloseFile(&pStmt->fp);
}
nodesDestroyList(pStmt->pSpecificTags);
break;
}
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables);
break;

2
source/libs/parser/inc/parAst.h
View File

@ -201,6 +201,8 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode*
SNodeList* pTags, SNode* pOptions);
SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable,
SNodeList* pSpecificTags, SNodeList* pValsOfTags, SNode* pOptions);
SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pUseRealTable,
SNodeList* pSpecificTags, const SToken* pFilePath);
SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables);
SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables);

3
source/libs/parser/inc/parUtil.h
View File

@ -135,6 +135,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type);
int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag);
int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);
@ -150,7 +151,7 @@ int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pM
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache);
int32_t reserveViewUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache);
SParseMetaCache* pMetaCache);
int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache);
int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);

58
source/libs/parser/inc/sql.y
View File

@ -385,11 +385,15 @@ alter_table_clause(A) ::=
%destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); }
multi_create_clause(A) ::= create_from_file_clause(B). { A = createNodeList(pCxt, B); }
create_subtable_clause(A) ::=
not_exists_opt(B) full_table_name(C) USING full_table_name(D)
specific_cols_opt(E) TAGS NK_LP tags_literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
create_from_file_clause(A) ::= not_exists_opt(B) USING full_table_name(C)
NK_LP tag_list_opt(D) NK_RP FILE NK_STRING(E). { A = createCreateSubTableFromFileClause(pCxt, B, C, D, &E); }
%type multi_drop_clause { SNodeList* }
%destructor multi_drop_clause { nodesDestroyList($$); }
multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); }
@ -532,7 +536,7 @@ cmd ::= SHOW GRANTS LOGS.
cmd ::= SHOW CLUSTER MACHINES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); }
cmd ::= SHOW CREATE DATABASE db_name(A). { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
cmd ::= SHOW CREATE TABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT,
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT,
A); }
cmd ::= SHOW ENCRYPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ENCRYPTIONS_STMT); }
cmd ::= SHOW QUERIES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); }
@ -820,13 +824,13 @@ tags_literal(A) ::= NK_INTEGER(B).
tags_literal(A) ::= NK_INTEGER(B) NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_INTEGER(B) NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_INTEGER(C). {
@ -837,13 +841,13 @@ tags_literal(A) ::= NK_PLUS(B) NK_INTEGER(C).
tags_literal(A) ::= NK_PLUS(B) NK_INTEGER NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_INTEGER NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_INTEGER(C). {
@ -854,13 +858,13 @@ tags_literal(A) ::= NK_MINUS(B) NK_INTEGER(C).
tags_literal(A) ::= NK_MINUS(B) NK_INTEGER NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_INTEGER NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_FLOAT(B). { A = createRawValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B, NULL); }
@ -879,13 +883,13 @@ tags_literal(A) ::= NK_BIN(B).
tags_literal(A) ::= NK_BIN(B) NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_BIN(B) NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_BIN(C). {
@ -896,13 +900,13 @@ tags_literal(A) ::= NK_PLUS(B) NK_BIN(C).
tags_literal(A) ::= NK_PLUS(B) NK_BIN NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_BIN NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_BIN(C). {
@ -913,26 +917,26 @@ tags_literal(A) ::= NK_MINUS(B) NK_BIN(C).
tags_literal(A) ::= NK_MINUS(B) NK_BIN NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_BIN NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_HEX(B). { A = createRawValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B, NULL); }
tags_literal(A) ::= NK_HEX(B) NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_HEX(B) NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_HEX(C). {
@ -943,13 +947,13 @@ tags_literal(A) ::= NK_PLUS(B) NK_HEX(C).
tags_literal(A) ::= NK_PLUS(B) NK_HEX NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_PLUS(B) NK_HEX NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_HEX(C). {
@ -960,13 +964,13 @@ tags_literal(A) ::= NK_MINUS(B) NK_HEX(C).
tags_literal(A) ::= NK_MINUS(B) NK_HEX NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_MINUS(B) NK_HEX NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
@ -974,13 +978,13 @@ tags_literal(A) ::= NK_STRING(B).
tags_literal(A) ::= NK_STRING(B) NK_PLUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_STRING(B) NK_MINUS duration_literal(C). {
SToken l = B;
SToken r = getTokenFromRawExprNode(pCxt, C);
l.n = (r.z + r.n) - l.z;
l.n = (r.z + r.n) - l.z;
A = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, C);
}
tags_literal(A) ::= NK_BOOL(B). { A = createRawValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B, NULL); }
@ -1348,9 +1352,9 @@ parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP.
/************************************************ joined_table ********************************************************/
joined_table(A) ::=
table_reference(B) join_type(C) join_subtype(D) JOIN table_reference(E) join_on_clause_opt(F)
window_offset_clause_opt(G) jlimit_clause_opt(H). {
A = createJoinTableNode(pCxt, C, D, B, E, F);
table_reference(B) join_type(C) join_subtype(D) JOIN table_reference(E) join_on_clause_opt(F)
window_offset_clause_opt(G) jlimit_clause_opt(H). {
A = createJoinTableNode(pCxt, C, D, B, E, F);
A = addWindowOffsetClause(pCxt, A, G);
A = addJLimitClause(pCxt, A, H);
}
@ -1376,16 +1380,16 @@ join_on_clause_opt(A) ::= .
join_on_clause_opt(A) ::= ON search_condition(B). { A = B; }
window_offset_clause_opt(A) ::= . { A = NULL; }
window_offset_clause_opt(A) ::= WINDOW_OFFSET NK_LP window_offset_literal(B)
window_offset_clause_opt(A) ::= WINDOW_OFFSET NK_LP window_offset_literal(B)
NK_COMMA window_offset_literal(C) NK_RP. { A = createWindowOffsetNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); }
window_offset_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createTimeOffsetValueNode(pCxt, &B)); }
window_offset_literal(A) ::= NK_MINUS(B) NK_VARIABLE(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createTimeOffsetValueNode(pCxt, &t));
A = createRawExprNode(pCxt, &t, createTimeOffsetValueNode(pCxt, &t));
}
jlimit_clause_opt(A) ::= . { A = NULL; }
jlimit_clause_opt(A) ::= JLIMIT NK_INTEGER(B). { A = createLimitNode(pCxt, &B, NULL); }

20
source/libs/parser/src/parAstCreater.c
View File

@ -1828,6 +1828,26 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN
return (SNode*)pStmt;
}
SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pUseRealTable,
SNodeList* pSpecificTags, const SToken* pFilePath) {
CHECK_PARSER_STATUS(pCxt);
SCreateSubTableFromFileClause* pStmt =
(SCreateSubTableFromFileClause*)nodesMakeNode(QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE);
CHECK_OUT_OF_MEM(pStmt);
strcpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName);
strcpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName);
pStmt->ignoreExists = ignoreExists;
pStmt->pSpecificTags = pSpecificTags;
if (TK_NK_STRING == pFilePath->type) {
trimString(pFilePath->z, pFilePath->n, pStmt->filePath, PATH_MAX);
} else {
strncpy(pStmt->filePath, pFilePath->z, pFilePath->n);
}
nodesDestroyNode(pUseRealTable);
return (SNode*)pStmt;
}
SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) {
CHECK_PARSER_STATUS(pCxt);
SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLES_STMT);

39
source/libs/parser/src/parAstParser.c
View File

@ -275,19 +275,34 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pSubTables) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->dbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
if (pNode->type == QUERY_NODE_CREATE_SUBTABLE_CLAUSE) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->dbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code =
reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
} else {
SCreateSubTableFromFileClause* pClause = (SCreateSubTableFromFileClause*)pNode;
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->useDbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
}
if (TSDB_CODE_SUCCESS != code) {
break;
}

16
source/libs/parser/src/parAuthenticator.c
View File

@ -265,10 +265,18 @@ static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStm
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pSubTables) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = checkAuth(pCxt, pClause->dbName, NULL, AUTH_TYPE_WRITE, NULL);
if (TSDB_CODE_SUCCESS != code) {
break;
if (pNode->type == QUERY_NODE_CREATE_SUBTABLE_CLAUSE) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = checkAuth(pCxt, pClause->dbName, NULL, AUTH_TYPE_WRITE, NULL);
if (TSDB_CODE_SUCCESS != code) {
break;
}
} else {
SCreateSubTableFromFileClause* pClause = (SCreateSubTableFromFileClause*)pNode;
code = checkAuth(pCxt, pClause->useDbName, NULL, AUTH_TYPE_WRITE, NULL);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
}
return code;

73
source/libs/parser/src/parInsertSql.c
View File

@ -1708,37 +1708,31 @@ typedef union SRowsDataContext {
SStbRowsDataContext* pStbRowsCxt;
} SRowsDataContext;
static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken,
bool* pFoundCtbName) {
int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName) {
*pFoundCtbName = false;
int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == pToken->type) {
code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname");
}
if (code == TSDB_CODE_SUCCESS) {
if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) {
return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value");
}
if (pToken->n > 0) {
if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) {
for (int i = 0; i < pToken->n; ++i) {
if (pToken->z[i] == '.') {
return buildInvalidOperationMsg(&pCxt->msg, "tbname can not contain '.'");
} else {
pStbRowsCxt->ctbName.tname[i] = pToken->z[i];
}
}
pStbRowsCxt->ctbName.tname[pToken->n] = '\0';
*pFoundCtbName = true;
} else {
return buildInvalidOperationMsg(&pCxt->msg, "tbname is too long");
}
} else {
return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be empty");
}
if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) {
return buildInvalidOperationMsg(pMsgBuf, "tbname can not be null value");
}
return code;
if (pToken->n > 0) {
if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) {
for (int i = 0; i < pToken->n; ++i) {
if (pToken->z[i] == '.') {
return buildInvalidOperationMsg(pMsgBuf, "tbname can not contain '.'");
} else {
tname[i] = pToken->z[i];
}
}
tname[pToken->n] = '\0';
*pFoundCtbName = true;
} else {
return buildInvalidOperationMsg(pMsgBuf, "tbname is too long");
}
} else {
return buildInvalidOperationMsg(pMsgBuf, "tbname can not be empty");
}
return TSDB_CODE_SUCCESS;
}
static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt,
@ -1821,7 +1815,14 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
}
}
} else if (pCols->pColIndex[i] == tbnameIdx) {
code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName);
code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == pToken->type) {
code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname");
}
if (code == TSDB_CODE_SUCCESS) {
code = parseTbnameToken(&pCxt->msg, pStbRowsCxt->ctbName.tname, pToken, bFoundTbName);
}
}
if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) {
@ -2427,6 +2428,20 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
return buildSyntaxErrMsg(&pCxt->msg, "table_name is expected", pTbName->z);
}
// db.? situation，ensure that the only thing following the '.' mark is '?'
char *tbNameAfterDbName = strchr(pTbName->z, '.');
if ((tbNameAfterDbName != NULL) && (tbNameAfterDbName + 1 - pTbName->z == pTbName->n - 1) &&
(*(tbNameAfterDbName + 1) == '?')) {
char *tbName = NULL;
int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName);
if (TSDB_CODE_SUCCESS == code) {
pTbName->z = tbName;
pTbName->n = strlen(tbName);
} else {
return code;
}
}
*pHasData = true;
return TSDB_CODE_SUCCESS;
}

53
source/libs/parser/src/parInsertStmt.c
View File

@ -51,7 +51,8 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) {
return TSDB_CODE_SUCCESS;
}
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) {
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo) {
// merge according to vgId
return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo);
}
@ -70,18 +71,16 @@ int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDat
return code;
}
/*
int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t tbNum) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pVgDataBlocks = NULL;
SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t
tbNum) { int32_t code = TSDB_CODE_SUCCESS; SArray* pVgDataBlocks = NULL; SVnodeModifyOpStmt*
pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
// merge according to vgId
if (tbNum > 0) {
code = insMergeStmtTableDataCxt(pTbCtx, pBlockList, &pVgDataBlocks, true, tbNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks);
}
@ -102,7 +101,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
if (taosHashGetSize(pBlockHash) > 0) {
code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks, true);
}
if (TSDB_CODE_SUCCESS == code) {
code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks, false);
}
@ -217,6 +216,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName,
pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL);
pTag = NULL;
end:
for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
@ -227,6 +227,7 @@ end:
}
taosArrayDestroy(pTagArray);
taosArrayDestroy(tagName);
taosMemoryFree(pTag);
return code;
}
@ -243,7 +244,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
if (NULL == dst->length) {
dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num);
if (NULL == dst->buffer) {
if (NULL == dst->length) {
taosMemoryFreeClear(dst->buffer);
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -276,7 +277,8 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
return TSDB_CODE_SUCCESS;
}
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo* pBindInfos) {
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -293,20 +295,21 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
}
for (int c = 0; c < boundInfo->numOfBound; ++c) {
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
if (pColSchema->colId <= lastColId) {
colInOrder = false;
} else {
lastColId = pColSchema->colId;
}
//SColData* pCol = taosArrayGet(pCols, c);
// SColData* pCol = taosArrayGet(pCols, c);
if (bind[c].num != rowNum) {
code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
goto _return;
}
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) &&
bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
goto _return;
}
@ -325,10 +328,10 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
pBindInfos[c].bind = pBind;
pBindInfos[c].type = pColSchema->type;
//code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1);
//if (code) {
// goto _return;
//}
// code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes -
// VARSTR_HEADER_SIZE: -1); if (code) {
// goto _return;
// }
}
code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols);
@ -362,7 +365,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
goto _return;
}
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) &&
bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
goto _return;
}
@ -377,7 +381,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
pBind = bind + c;
}
code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1);
code = tColDataAddValueByBind(pCol, pBind,
IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
if (code) {
goto _return;
}
@ -393,9 +398,8 @@ _return:
return code;
}
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t rowNum) {
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -510,7 +514,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
}
int32_t qResetStmtColumns(SArray* pCols, bool deepClear) {
int32_t colNum = taosArrayGetSize(pCols);
int32_t colNum = taosArrayGetSize(pCols);
for (int32_t i = 0; i < colNum; ++i) {
SColData* pCol = (SColData*)taosArrayGet(pCols, i);
@ -524,7 +528,6 @@ int32_t qResetStmtColumns(SArray* pCols, bool deepClear) {
return TSDB_CODE_SUCCESS;
}
int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) {
STableDataCxt* pBlock = (STableDataCxt*)block;
int32_t colNum = taosArrayGetSize(pBlock->pData->aCol);
@ -589,7 +592,7 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese
pNewTb->pCreateTbReq = NULL;
pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL);
if (NULL == pNewTb) {
if (NULL == pNewTb->aCol) {
insDestroyTableDataCxt(*pDst);
return TSDB_CODE_OUT_OF_MEMORY;
}

79
source/libs/parser/src/parInsertUtil.c
View File

@ -115,8 +115,7 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c
if (dbName == NULL) {
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
if (name[0] == '\0')
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4);
if (name[0] == '\0') return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4);
code = tNameSetDbName(pName, acctId, dbName, strlen(dbName));
if (code != TSDB_CODE_SUCCESS) {
@ -487,12 +486,12 @@ int insColDataComp(const void* lp, const void* rp) {
return 0;
}
int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId, STableColsData* pTbData, SName* sname) {
int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId,
STableColsData* pTbData, SName* sname) {
if (*vgId >= 0 && taosHashGet(pAllVgHash, (const char*)vgId, sizeof(*vgId))) {
return TSDB_CODE_SUCCESS;
}
SVgroupInfo vgInfo = {0};
SRequestConnInfo conn = {.pTrans = pBuildInfo->transport,
.requestId = pBuildInfo->requestId,
@ -503,23 +502,23 @@ int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuild
if (TSDB_CODE_SUCCESS != code) {
return code;
}
code = taosHashPut(pAllVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo));
if (TSDB_CODE_SUCCESS != code) {
return code;
}
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData, uint64_t* uid, int32_t* vgId) {
int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData,
uint64_t* uid, int32_t* vgId) {
STableVgUid* pTbInfo = NULL;
int32_t code = 0;
int32_t code = 0;
if (pTbData->getFromHash) {
pTbInfo = (STableVgUid*)tSimpleHashGet(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName));
}
}
if (NULL == pTbInfo) {
SName sname;
@ -540,7 +539,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
if (TSDB_CODE_SUCCESS != code) {
return code;
}
*uid = pTableMeta->uid;
*vgId = pTableMeta->vgId;
@ -548,7 +547,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
tSimpleHashPut(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName), &tbInfo, sizeof(tbInfo));
code = insTryAddTableVgroupInfo(pAllVgHash, pBuildInfo, vgId, pTbData, &sname);
taosMemoryFree(pTableMeta);
} else {
*uid = pTbInfo->uid;
@ -558,7 +557,6 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
return code;
}
int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks) {
int32_t code = TSDB_CODE_SUCCESS;
SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
@ -570,20 +568,19 @@ int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDa
return code;
}
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo) {
int32_t code = TSDB_CODE_SUCCESS;
uint64_t uid;
int32_t vgId;
pTbCtx->pData->aRowP = pTbData->aCol;
code = insGetStmtTableVgUid(pAllVgHash, pBuildInfo, pTbData, &uid, &vgId);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
pTbCtx->pMeta->vgId = vgId;
pTbCtx->pMeta->uid = uid;
pTbCtx->pData->uid = uid;
@ -611,24 +608,24 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData,
} else {
pVgCxt = *(SVgroupDataCxt**)pp;
}
if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupDataCxt(pTbCtx, pVgCxt, false, false);
}
if (taosArrayGetSize(pVgCxt->pData->aSubmitTbData) >= 20000) {
code = qBuildStmtFinOutput1((SQuery*)pBuildInfo->pQuery, pAllVgHash, pBuildInfo->pVgroupList);
//taosArrayClear(pVgCxt->pData->aSubmitTbData);
// taosArrayClear(pVgCxt->pData->aSubmitTbData);
tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE);
//insDestroyVgroupDataCxt(pVgCxt);
// insDestroyVgroupDataCxt(pVgCxt);
}
return code;
}
/*
int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild, int32_t tbNum) {
SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild,
int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES);
if (NULL == pVgroupHash || NULL == pVgroupList) {
taosHashCleanup(pVgroupHash);
@ -644,7 +641,7 @@ int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, S
pTableCxt->pMeta->uid = pTableCols->uid;
pTableCxt->pData->uid = pTableCols->uid;
pTableCxt->pData->aCol = pTableCols->aCol;
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0);
if (pCol->nVal <= 0) {
continue;
@ -810,7 +807,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
if (taosArrayGetSize(src->pData->aSubmitTbData) <= 0) {
continue;
}
SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
if (NULL == dst) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
@ -875,7 +872,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
char* p = (char*)data;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length |
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
int32_t version = *(int32_t*)data;
p += sizeof(int32_t);
p += sizeof(int32_t);
@ -890,7 +888,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
p += sizeof(uint64_t);
int8_t* fields = p;
if(*fields >= TSDB_DATA_TYPE_MAX || *fields < 0){
if (*fields >= TSDB_DATA_TYPE_MAX || *fields < 0) {
uError("fields type error:%d", *fields);
ret = TSDB_CODE_INVALID_PARA;
goto end;
@ -911,7 +909,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end;
}
if (tFields != NULL && numFields > boundInfo->numOfBound) {
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
if (errstr != NULL)
snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -920,8 +919,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
if (errstr != NULL)
snprintf(errstr, errstrLen,
"column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name,
*(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -951,8 +953,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
if (strcmp(pColSchema->name, tFields[i].name) == 0) {
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
if (errstr != NULL)
snprintf(errstr, errstrLen,
"column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name,
*(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -986,7 +991,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
}
if(!hasTs){
if (!hasTs) {
if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data");
ret = TSDB_CODE_INVALID_PARA;
goto end;

5
source/libs/parser/src/parTokenizer.c
View File

@ -781,8 +781,9 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC
if ('.' == str[*i + t0.n]) {
len = tGetToken(&str[*i + t0.n + 1], &type);
// only id and string are valid
if (((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) || ((TK_NK_STRING != type) && (TK_NK_ID != type))) {
// only id、string and ? are valid
if (((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) ||
((TK_NK_STRING != type) && (TK_NK_ID != type) && (TK_NK_QUESTION != type))) {
t0.type = TK_NK_ILLEGAL;
t0.n = 0;

411
source/libs/parser/src/parTranslater.c
View File

@ -7254,6 +7254,7 @@ static int32_t buildCmdMsg(STranslateContext* pCxt, int16_t msgType, FSerializeF
pCxt->pCmdMsg->msgLen = func(NULL, 0, pReq);
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
taosMemoryFreeClear(pCxt->pCmdMsg);
return TSDB_CODE_OUT_OF_MEMORY;
}
func(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, pReq);
@ -8111,9 +8112,11 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt,
for (int32_t i = 1; i < num; ++i) {
SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i);
STranslateContext cxt = {0};
initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2,
1 == i ? &pReq->ast1Len : &pReq->ast2Len);
code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
if (TSDB_CODE_SUCCESS == code) {
code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2,
1 == i ? &pReq->ast1Len : &pReq->ast2Len);
}
destroyTranslateContext(&cxt);
if (TSDB_CODE_SUCCESS != code) {
break;
@ -12413,38 +12416,39 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
return code;
}
static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt,
const STag* pTag, uint64_t suid, const char* sTableNmae, SVgroupInfo* pVgInfo,
SArray* tagName, uint8_t tagNum) {
// char dbFName[TSDB_DB_FNAME_LEN] = {0};
// SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId};
// strcpy(name.dbname, pStmt->dbName);
// tNameGetFullDbName(&name, dbFName);
static int32_t addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* dbName, uint64_t suid,
const char* sTableName, const char* tableName, SArray* tagName, uint8_t tagNum,
const STag* pTag, int32_t ttl, const char* comment, bool ignoreExists,
SVgroupInfo* pVgInfo) {
struct SVCreateTbReq req = {0};
req.type = TD_CHILD_TABLE;
req.name = taosStrdup(pStmt->tableName);
req.ttl = pStmt->pOptions->ttl;
if (pStmt->pOptions->commentNull == false) {
req.comment = taosStrdup(pStmt->pOptions->comment);
req.commentLen = strlen(pStmt->pOptions->comment);
req.name = taosStrdup(tableName);
req.ttl = ttl;
if (comment != NULL) {
req.comment = taosStrdup(comment);
req.commentLen = strlen(comment);
} else {
req.commentLen = -1;
}
req.ctb.suid = suid;
req.ctb.tagNum = tagNum;
req.ctb.stbName = taosStrdup(sTableNmae);
req.ctb.stbName = taosStrdup(sTableName);
req.ctb.pTag = (uint8_t*)pTag;
req.ctb.tagName = taosArrayDup(tagName, NULL);
if (pStmt->ignoreExists) {
if (ignoreExists) {
req.flags |= TD_CREATE_IF_NOT_EXISTS;
}
if (!req.name || !req.ctb.stbName || !req.ctb.tagName || (comment && !req.comment)) {
tdDestroySVCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
if (pTableBatch == NULL) {
SVgroupCreateTableBatch tBatch = {0};
tBatch.info = *pVgInfo;
strcpy(tBatch.dbName, pStmt->dbName);
strcpy(tBatch.dbName, dbName);
tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
taosArrayPush(tBatch.req.pArray, &req);
@ -12453,11 +12457,8 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S
} else { // add to the correct vgroup
taosArrayPush(pTableBatch->req.pArray, &req);
}
}
static SDataType schemaToDataType(uint8_t precision, SSchema* pSchema) {
SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = precision, .scale = 0};
return dt;
return TSDB_CODE_SUCCESS;
}
static int32_t createCastFuncForTag(STranslateContext* pCxt, SNode* pNode, SDataType dt, SNode** pCast) {
@ -12646,8 +12647,10 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info);
}
if (TSDB_CODE_SUCCESS == code) {
addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid,
pStmt->useTableName, &info, tagName, pSuperTableMeta->tableInfo.numOfTags);
const char* comment = pStmt->pOptions->commentNull ? NULL : pStmt->pOptions->comment;
code = addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->dbName, pSuperTableMeta->uid, pStmt->useTableName,
pStmt->tableName, tagName, pSuperTableMeta->tableInfo.numOfTags, pTag,
pStmt->pOptions->ttl, comment, pStmt->ignoreExists, &info);
} else {
taosMemoryFree(pTag);
}
@ -12657,6 +12660,355 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
return code;
}
static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFileClause* pStmt,
STableMeta* pSuperTableMeta) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfTags = getNumOfTags(pSuperTableMeta);
SSchema* pSchema = getTableTagSchema(pSuperTableMeta);
SHashObj* pIdxHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (NULL == pIdxHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
bool tbnameFound = false;
SNode* pTagNode;
FOREACH(pTagNode, pStmt->pSpecificTags) {
int32_t idx = -1;
do {
if (QUERY_NODE_COLUMN == nodeType(pTagNode)) {
SColumnNode* pColNode = (SColumnNode*)pTagNode;
for (int32_t index = 0; index < numOfTags; index++) {
if (strlen(pSchema[index].name) == strlen(pColNode->colName) &&
strcmp(pColNode->colName, pSchema[index].name) == 0) {
idx = index;
break;
}
}
if (idx < 0) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pColNode->colName);
break;
}
if (NULL != taosHashGet(pIdxHash, &idx, sizeof(idx))) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TAG_NAME_DUPLICATED, pColNode->colName);
break;
}
} else if (QUERY_NODE_FUNCTION == nodeType(pTagNode)) {
SFunctionNode* funcNode = (SFunctionNode*)pTagNode;
if (strlen("tbname") != strlen(funcNode->functionName) || strcmp("tbname", funcNode->functionName) != 0) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, funcNode->functionName);
}
idx = numOfTags + 1;
tbnameFound = true;
if (NULL != taosHashGet(pIdxHash, &idx, sizeof(idx))) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TAG_NAME_DUPLICATED, funcNode->functionName);
break;
}
} else {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "invalid node type");
break;
}
} while (0);
if (code) break;
if (taosHashPut(pIdxHash, &idx, sizeof(idx), NULL, 0) < 0) {
code = terrno;
goto _OUT;
}
if (NULL == taosArrayPush(pStmt->aTagIndexs, &idx)) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
}
if (!tbnameFound) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_ERROR);
}
_OUT:
taosHashCleanup(pIdxHash);
return code;
}
typedef struct {
// refer
STableMeta* pSuperTableMeta;
SArray* pTagIndexs;
TdFilePtr fp;
// containers
SHashObj* pTbNameHash;
SArray* aTagNames;
bool tagNameFilled;
SArray* aTagVals;
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
// per line
const char* pSql;
STag* pTag;
SName ctbName;
SVgroupInfo vg;
} SParseFileContext;
static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgroupInfo* pVgInfo) {
SVgroupInfo vg;
SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter,
.requestId = pParseCxt->requestId,
.requestObjRefId = pParseCxt->requestRid,
.mgmtEps = pParseCxt->mgmtEpSet};
int32_t code = catalogGetTableHashVgroup(pParseCxt->pCatalog, &conn, pName, &vg);
if (code == TSDB_CODE_SUCCESS) {
*pVgInfo = vg;
} else {
parserError("0x%" PRIx64 " catalogGetTableHashVgroup error, code:%s, dbName:%s, tbName:%s", pParseCxt->requestId,
tstrerror(code), pName->dbname, pName->tname);
}
return code;
}
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCtx) {
int32_t code = TSDB_CODE_SUCCESS;
int sz = taosArrayGetSize(pParFileCtx->pTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCtx->pSuperTableMeta);
uint8_t precision = getTableInfo(pParFileCtx->pSuperTableMeta).precision;
SSchema* pSchemas = getTableTagSchema(pParFileCtx->pSuperTableMeta);
for (int i = 0; i < sz; i++) {
const char* pSql = pParFileCtx->pSql;
int32_t pos = 0;
SToken token = tStrGetToken(pSql, &pos, true, NULL);
pParFileCtx->pSql += pos;
if (TK_NK_RP == token.type) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
break;
}
int16_t index = *(int16_t*)taosArrayGet(pParFileCtx->pTagIndexs, i);
if (index < numOfTags) {
// parse tag
const SSchema* pTagSchema = &pSchemas[index];
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, pTagSchema->type);
if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected row value");
}
if (TSDB_CODE_SUCCESS == code) {
SArray* aTagNames = pParFileCtx->tagNameFilled ? NULL : pParFileCtx->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCtx->pSql, precision, (SSchema*)pTagSchema, &token,
aTagNames, pParFileCtx->aTagVals, &pParFileCtx->pTag);
}
} else {
// parse tbname
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected tbname");
}
if (TSDB_CODE_SUCCESS == code) {
bool bFoundTbName = false;
code = parseTbnameToken(pMsgBuf, pParFileCtx->ctbName.tname, &token, &bFoundTbName);
}
}
if (TSDB_CODE_SUCCESS != code) break;
}
if (TSDB_CODE_SUCCESS == code) {
pParFileCtx->tagNameFilled = true;
code = tTagNew(pParFileCtx->aTagVals, 1, false, &pParFileCtx->pTag);
}
return code;
}
typedef struct {
SName ctbName;
SArray* aTagNames;
STag* pTag;
SVgroupInfo vg;
} SCreateTableData;
static void clearTagValArrayFp(void *data) {
STagVal* p = (STagVal*)data;
if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFreeClear(p->pData);
}
}
static void clearCreateTbArrayFp(void *data) {
SCreateTableData* p = (SCreateTableData*)data;
taosMemoryFreeClear(p->pTag);
}
static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFileContext* pParseFileCtx,
SArray* aCreateTbData) {
int32_t code = TSDB_CODE_SUCCESS;
char* pLine = NULL;
int64_t readLen = 0;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pParseFileCtx->fp, &pLine)) != -1) {
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
pLine[--readLen] = '\0';
}
if (readLen == 0) continue;
if (pLine[0] == '#') continue;
strtolower(pLine, pLine);
pParseFileCtx->pSql = pLine;
code = parseOneStbRow(pMsgBuf, pParseFileCtx);
if (TSDB_CODE_SUCCESS == code) {
if (taosHashGet(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname,
strlen(pParseFileCtx->ctbName.tname) + 1) != NULL) {
taosMemoryFreeClear(pParseFileCtx->pTag);
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParseFileCtx->ctbName.tname);
break;
}
code = taosHashPut(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname,
strlen(pParseFileCtx->ctbName.tname) + 1, NULL, 0);
}
if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupInfo(pParseCxt, &pParseFileCtx->ctbName, &pParseFileCtx->vg);
}
if (TSDB_CODE_SUCCESS == code) {
SCreateTableData data = {.ctbName = pParseFileCtx->ctbName,
.aTagNames = pParseFileCtx->aTagNames,
.pTag = pParseFileCtx->pTag,
.vg = pParseFileCtx->vg};
taosArrayPush(aCreateTbData, &data);
} else {
taosMemoryFreeClear(pParseFileCtx->pTag);
}
pParseFileCtx->pTag = NULL;
taosArrayClearEx(pParseFileCtx->aTagVals, clearTagValArrayFp);
}
if (TSDB_CODE_SUCCESS != code) {
taosArrayClearEx(aCreateTbData, clearCreateTbArrayFp);
}
taosMemoryFree(pLine);
return code;
}
static int32_t prepareReadFromFile(SCreateSubTableFromFileClause* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pStmt->fp) {
pStmt->fp = taosOpenFile(pStmt->filePath, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == pStmt->fp) {
code = TAOS_SYSTEM_ERROR(errno);
goto _ERR;
}
}
if (NULL == pStmt->aCreateTbData) {
pStmt->aCreateTbData = taosArrayInit(16, sizeof(SCreateTableData));
if (NULL == pStmt->aCreateTbData) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pStmt->aTagIndexs) {
pStmt->aTagIndexs = taosArrayInit(pStmt->pSpecificTags->length, sizeof(int16_t));
if (!pStmt->aTagIndexs) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
return code;
_ERR:
taosCloseFile(&pStmt->fp);
taosArrayDestroy(pStmt->aCreateTbData);
taosArrayDestroy(pStmt->aTagIndexs);
return code;
}
static int32_t rewriteCreateSubTableFromFile(STranslateContext* pCxt, SCreateSubTableFromFileClause* pStmt,
SHashObj* pVgroupHashmap) {
int32_t code = 0;
STableMeta* pSuperTableMeta = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = prepareReadFromFile(pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildTagIndexForBindTags(&pCxt->msgBuf, pStmt, pSuperTableMeta);
}
SParseFileContext parseFileCtx = {
.pSuperTableMeta = pSuperTableMeta, .fp = pStmt->fp, .pTagIndexs = pStmt->aTagIndexs};
parseFileCtx.pTbNameHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
parseFileCtx.aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN);
parseFileCtx.tagNameFilled = false;
parseFileCtx.aTagVals = taosArrayInit(8, sizeof(STagVal));
parseFileCtx.pTag = NULL;
parseFileCtx.ctbName.type = TSDB_TABLE_NAME_T;
parseFileCtx.ctbName.acctId = pCxt->pParseCxt->acctId;
strcpy(parseFileCtx.ctbName.dbname, pStmt->useDbName);
if (NULL == parseFileCtx.aTagNames || NULL == parseFileCtx.aTagVals || NULL == parseFileCtx.pTbNameHash) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
if (TSDB_CODE_SUCCESS == code) {
code = parseCsvFile(&pCxt->msgBuf, pCxt->pParseCxt, &parseFileCtx, pStmt->aCreateTbData);
}
if (TSDB_CODE_SUCCESS == code) {
int sz = taosArrayGetSize(pStmt->aCreateTbData);
for (int i = 0; i < sz; i++) {
SCreateTableData* pData = taosArrayGet(pStmt->aCreateTbData, i);
code = collectUseTable(&pData->ctbName, pCxt->pTargetTables);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(pData->pTag);
}
code = addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->useDbName, pSuperTableMeta->uid, pStmt->useTableName,
pData->ctbName.tname, pData->aTagNames, pSuperTableMeta->tableInfo.numOfTags,
pData->pTag, TSDB_DEFAULT_TABLE_TTL, NULL, pStmt->ignoreExists, &pData->vg);
}
}
_OUT:
taosMemoryFreeClear(pSuperTableMeta);
taosHashCleanup(parseFileCtx.pTbNameHash);
taosArrayDestroy(parseFileCtx.aTagNames);
taosArrayDestroy(parseFileCtx.aTagVals);
return code;
}
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) {
SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
if (NULL == pBufArray) {
@ -12689,8 +13041,13 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode;
FOREACH(pNode, pStmt->pSubTables) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
if (pNode->type == QUERY_NODE_CREATE_SUBTABLE_CLAUSE) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
} else {
SCreateSubTableFromFileClause* pClause = (SCreateSubTableFromFileClause*)pNode;
code = rewriteCreateSubTableFromFile(pCxt, pClause, pVgroupHashmap);
}
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pVgroupHashmap);
return code;

10
source/libs/parser/src/parUtil.c
View File

@ -212,10 +212,16 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_COL_PK_TYPE:
return "primary key column must be of type int, uint, bigint, ubigint, and varchar";
case TSDB_CODE_PAR_INVALID_PK_OP:
return "primary key column can not be added, modified, and dropped";
return "primary key column can not be added, modified, and dropped";
case TSDB_CODE_TSMA_NAME_TOO_LONG:
return "Tsma name too long";
default:
case TSDB_CODE_PAR_TBNAME_ERROR:
return "Pseudo tag tbname not set";
case TSDB_CODE_PAR_TBNAME_DUPLICATED:
return "Table name:%s duplicated";
case TSDB_CODE_PAR_TAG_NAME_DUPLICATED:
return "Tag name:%s duplicated";
default:
return "Unknown error";
}
}

41
source/libs/parser/src/parser.c
View File

@ -48,6 +48,47 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
return false;
}
bool qParseDbName(const char* pStr, size_t length, char** pDbName) {
(void) length;
int32_t index = 0;
SToken t;
if (NULL == pStr) {
*pDbName = NULL;
return false;
}
t = tStrGetToken((char *) pStr, &index, false, NULL);
if (TK_INSERT != t.type && TK_IMPORT != t.type) {
*pDbName = NULL;
return false;
}
t = tStrGetToken((char *) pStr, &index, false, NULL);
if (TK_INTO != t.type) {
*pDbName = NULL;
return false;
}
t = tStrGetToken((char *) pStr, &index, false, NULL);
if (t.n == 0 || t.z == NULL) {
*pDbName = NULL;
return false;
}
char *dotPos = strnchr(t.z, '.', t.n, true);
if (dotPos != NULL) {
int dbNameLen = dotPos - t.z;
*pDbName = taosMemoryMalloc(dbNameLen + 1);
if (*pDbName == NULL) {
return false;
}
strncpy(*pDbName, t.z, dbNameLen);
(*pDbName)[dbNameLen] = '\0';
return true;
}
return false;
}
static int32_t analyseSemantic(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
int32_t code = authenticate(pCxt, pQuery, pMetaCache);

10489
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

47
source/libs/qworker/src/qworker.c
View File

@ -92,7 +92,6 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re
QW_RET(TSDB_CODE_SUCCESS);
}
int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
qTaskInfo_t taskHandle = ctx->taskHandle;
@ -201,7 +200,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
} else {
QW_TASK_DLOG("dyn task qExecTask done, useconds:%" PRIu64, useconds);
}
ctx->queryExecDone = true;
}
@ -251,7 +250,6 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) {
return false;
}
int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) {
int32_t taskNum = 0;
@ -340,7 +338,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
if (!ctx->dynamicTask) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
}
if (NULL == pRsp) {
QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp));
*pOutput = output;
@ -367,8 +365,8 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
// set the serialize start position
output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN);
((int32_t*) output.pData)[0] = len;
((int32_t*) output.pData)[1] = rawLen;
((int32_t *)output.pData)[0] = len;
((int32_t *)output.pData)[1] = rawLen;
output.pData += sizeof(int32_t) * 2;
taosEnableMemoryPoolUsage(gQueryPoolHandle, ctx->memPoolSession);
@ -463,8 +461,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
return TSDB_CODE_SUCCESS;
}
int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, int32_t code) {
int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32_t code) {
if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) {
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
void *rsp = NULL;
@ -490,13 +487,13 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, i
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
dataLen);
}
}
}
return TSDB_CODE_SUCCESS;
@ -511,7 +508,7 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg
if (!atomic_val_compare_exchange_8((int8_t*)&ctx->queryEnd, true, false)) {
QW_TASK_ELOG("dynamic task prev exec not finished, queryEnd:%d", ctx->queryEnd);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
}
#else
ctx->queryExecDone = false;
ctx->queryEnd = false;
@ -519,7 +516,7 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg
taosEnableMemoryPoolUsage(gQueryPoolHandle, ctx->memPoolSession);
dsReset(ctx->sinkHandle);
taosDisableMemoryPoolUsage() ;
taosDisableMemoryPoolUsage();
qUpdateOperatorParam(ctx->taskHandle, qwMsg->msg);
@ -537,7 +534,6 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg
return TSDB_CODE_SUCCESS;
}
int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
int32_t code = 0;
SQWTaskCtx *ctx = NULL;
@ -805,7 +801,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
ctx->queryExecDone = true;
ctx->queryEnd = true;
}
_return:
taosMemoryFree(sql);
@ -842,7 +838,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (!queryStop) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop));
}
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
SOutputData sOutput = {0};
QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput));
@ -911,9 +907,9 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
}
int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0;
int32_t dataLen = 0;
int32_t rawDataLen = 0;
int32_t code = 0;
int32_t dataLen = 0;
int32_t rawDataLen = 0;
bool locked = false;
SQWTaskCtx *ctx = NULL;
@ -995,8 +991,8 @@ _return:
qwFreeFetchRsp(rsp);
rsp = NULL;
}
} else {
//qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code);
} else {
// qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code);
}
QW_RET(TSDB_CODE_SUCCESS);
@ -1038,6 +1034,8 @@ _return:
if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
} else {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
}
}
@ -1046,6 +1044,10 @@ _return:
}
if (ctx) {
if (qwMsg->connInfo.handle != ctx->ctrlConnInfo.handle) {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
}
qwReleaseTaskCtx(mgmt, ctx);
}
@ -1073,7 +1075,7 @@ int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (ctx->explain && !ctx->explainRsped) {
QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx));
}
break;
break;
default:
QW_ELOG("Invalid task notify type %d", qwMsg->msgType);
QW_ERR_JRET(TSDB_CODE_INVALID_MSG);
@ -1100,7 +1102,6 @@ _return:
QW_RET(TSDB_CODE_SUCCESS);
}
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0;
SSchedulerHbRsp rsp = {0};

269
source/libs/scalar/src/sclfunc.c
View File

@ -1095,24 +1095,45 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
char fraction[20] = {0};
bool hasFraction = false;
NUM_TO_STRING(type, input, sizeof(fraction), fraction);
int32_t tsDigits = (int32_t)strlen(fraction);
int32_t fractionLen;
char buf[64] = {0};
int64_t timeVal;
char* format = NULL;
int64_t quot = 0;
long mod = 0;
GET_TYPED_DATA(timeVal, int64_t, type, input);
if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal / 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / ((int64_t)(1000 * 1000));
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / ((int64_t)(1000 * 1000 * 1000));
} else {
switch (pInput->columnData[0].info.precision) {
case TSDB_TIME_PRECISION_MILLI: {
quot = timeVal / 1000;
fractionLen = 5;
format = ".%03" PRId64;
mod = timeVal % 1000;
break;
}
case TSDB_TIME_PRECISION_MICRO: {
quot = timeVal / 1000000;
fractionLen = 8;
format = ".%06" PRId64;
mod = timeVal % 1000000;
break;
}
case TSDB_TIME_PRECISION_NANO: {
quot = timeVal / 1000000000;
fractionLen = 11;
format = ".%09" PRId64;
mod = timeVal % 1000000000;
break;
}
default: {
colDataSetNULL(pOutput->columnData, i);
continue;
}
hasFraction = true;
memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS);
}
// trans current timezone's unix ts to dest timezone
@ -1122,18 +1143,19 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
if (0 != offsetOfTimezone(tz, &offset)) {
goto _end;
}
timeVal -= offset + 3600 * ((int64_t)tsTimezone);
quot -= offset + 3600 * ((int64_t)tsTimezone);
struct tm tmInfo;
int32_t len = 0;
if (taosLocalTime((const time_t *)&timeVal, &tmInfo, buf) == NULL) {
if (taosLocalTime((const time_t *)&quot, &tmInfo, buf) == NULL) {
len = (int32_t)strlen(buf);
goto _end;
}
strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo);
len = (int32_t)strlen(buf);
len = (int32_t)strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo);
len += snprintf(buf + len, fractionLen, format, mod);
// add timezone string
if (tzLen > 0) {
@ -1141,32 +1163,6 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
len += tzLen;
}
if (hasFraction) {
int32_t fracLen = (int32_t)strlen(fraction) + 1;
char *tzInfo;
if (buf[len - 1] == 'z' || buf[len - 1] == 'Z') {
tzInfo = &buf[len - 1];
memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
} else {
tzInfo = strchr(buf, '+');
if (tzInfo) {
memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
} else {
// search '-' backwards
tzInfo = strrchr(buf, '-');
if (tzInfo) {
memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
}
}
}
char tmp[32] = {0};
sprintf(tmp, ".%s", fraction);
memcpy(tzInfo, tmp, fracLen);
len += fracLen;
}
_end:
memmove(buf + VARSTR_HEADER_SIZE, buf, len);
varDataSetLen(buf, len);
@ -1347,9 +1343,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData);
memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData));
int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
int64_t unit = timeUnit * 1000 / factor;
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) {
colDataSetNULL(pOutput->columnData, i);
@ -1359,201 +1352,27 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal);
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal);
if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i);
continue;
}
// If converted value is less than 10digits in second, use value in second instead
int64_t timeValSec = timeVal / 1000000000;
if (timeValSec < 1000000000) {
timeVal = timeValSec;
}
} else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */
GET_TYPED_DATA(timeVal, int64_t, type, input);
} else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/
GET_TYPED_DATA(timeVal, int64_t, type, input);
int64_t timeValSec = timeVal / factor;
if (timeValSec < 1000000000) {
timeVal = timeValSec;
}
}
char buf[20] = {0};
NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal, sizeof(buf), buf);
int32_t tsDigits = (int32_t)strlen(buf);
switch (unit) {
case 0: { /* 1u or 1b */
if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
if (timePrec == TSDB_TIME_PRECISION_NANO && timeUnit == 1) {
timeVal = timeVal * 1;
} else {
timeVal = timeVal / 1000 * 1000;
}
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
timeVal = timeVal * factor;
} else {
timeVal = timeVal * 1;
}
break;
}
case 1: { /* 1a */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal * 1;
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / 1000 * 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000000 * 1000000;
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
timeVal = timeVal * factor;
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
case 1000: { /* 1s */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal / 1000 * 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / 1000000 * 1000000;
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000000000 * 1000000000;
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
timeVal = timeVal * factor;
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
case 60000: { /* 1m */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal / 1000 / 60 * 60 * 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / 1000000 / 60 * 60 * 1000000;
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000000000 / 60 * 60 * 1000000000;
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
timeVal = timeVal * factor / factor / 60 * 60 * factor;
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
case 3600000: { /* 1h */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal / 1000 / 3600 * 3600 * 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / 1000000 / 3600 * 3600 * 1000000;
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000000000 / 3600 * 3600 * 1000000000;
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
timeVal = timeVal * factor / factor / 3600 * 3600 * factor;
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
case 86400000: { /* 1d */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)86400) * 1000);
} else {
timeVal = timeVal / 1000 / 86400 * 86400 * 1000;
}
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)86400) * 1000000);
} else {
timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000;
}
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)86400) * 1000000000);
} else {
timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000;
}
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
if (ignoreTz) {
timeVal = (timeVal - (timeVal + offsetFromTz(timezone, 1)) % (86400L)) * factor;
} else {
timeVal = timeVal * factor / factor / 86400 * 86400 * factor;
}
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
case 604800000: { /* 1w */
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)604800) * 1000);
} else {
timeVal = timeVal / 1000 / 604800 * 604800 * 1000;
}
} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)604800) * 1000000);
} else {
timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000;
}
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)604800) * 1000000000);
} else {
timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000;
}
} else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
if (ignoreTz) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1)) % (((int64_t)604800L) * factor);
} else {
timeVal = timeVal * factor / factor / 604800 * 604800 * factor;
}
} else {
colDataSetNULL(pOutput->columnData, i);
continue;
}
break;
}
default: {
timeVal = timeVal * 1;
break;
}
// truncate the timestamp to time_unit precision
int64_t seconds = timeUnit / TSDB_TICK_PER_SECOND(timePrec);
if (ignoreTz && (seconds == 604800 || seconds == 86400)) {
timeVal = timeVal - (timeVal + offsetFromTz(timezone, TSDB_TICK_PER_SECOND(timePrec))) % timeUnit;
} else {
timeVal = timeVal / timeUnit * timeUnit;
}
// truncate the timestamp to db precision
switch (timePrec) {
case TSDB_TIME_PRECISION_MILLI: {
if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal / 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000000;
}
break;
}
case TSDB_TIME_PRECISION_MICRO: {
if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
timeVal = timeVal / 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal * 1000;
}
break;
}
case TSDB_TIME_PRECISION_NANO: {
if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
timeVal = timeVal * 1000;
} else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
timeVal = timeVal * 1000000;
}
break;
}
}
colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false);
}

8
source/libs/stream/inc/streamInt.h
View File

@ -55,10 +55,10 @@ struct SActiveCheckpointInfo {
int64_t activeId; // current active checkpoint id
int64_t failedId;
bool dispatchTrigger;
SArray* pDispatchTriggerList; // SArray<STaskTriggerSendInfo>
SArray* pReadyMsgList; // SArray<STaskCheckpointReadyInfo*>
SArray* pDispatchTriggerList; // SArray<STaskTriggerSendInfo>
SArray* pReadyMsgList; // SArray<STaskCheckpointReadyInfo*>
int8_t allUpstreamTriggerRecv;
SArray* pCheckpointReadyRecvList; // SArray<STaskDownstreamReadyInfo>
SArray* pCheckpointReadyRecvList; // SArray<STaskDownstreamReadyInfo>
int32_t checkCounter;
tmr_h pChkptTriggerTmr;
int32_t sendReadyCheckCounter;
@ -100,7 +100,7 @@ typedef struct {
int32_t upstreamNodeId;
int32_t transId;
int32_t childId;
SRpcMsg msg; // for mnode checkpoint-source rsp
SRpcMsg msg; // for mnode checkpoint-source rsp
int64_t checkpointId;
int64_t recvTs;
int32_t sendCompleted;

43
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -140,7 +140,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest);
int32_t valueToString(void* k, char* buf);
int32_t valueIsStale(void* k, int64_t ts);
void destroyCompare(void* arg);
void destroyCompare(void* arg);
static void cleanDir(const char* pPath, const char* id);
static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len);
@ -194,9 +194,7 @@ int32_t getCfIdx(const char* cfName) {
return idx;
}
bool isValidCheckpoint(const char* dir) {
return true;
}
bool isValidCheckpoint(const char* dir) { return true; }
int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
// impl later
@ -486,9 +484,7 @@ _ERROR:
return code;
}
int32_t backendCopyFiles(const char* src, const char* dst) {
return backendFileCopyFilesImpl(src, dst);
}
int32_t backendCopyFiles(const char* src, const char* dst) { return backendFileCopyFilesImpl(src, dst); }
static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId,
const char* defaultPath) {
@ -540,7 +536,8 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
char* chkptPath = taosMemoryCalloc(1, pathLen);
if (chkptId > 0) {
snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId);
snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
chkptId);
code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
if (code != 0) {
@ -549,11 +546,12 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
if (code != 0) {
stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
tstrerror(code), defaultPath);
code = 0; // reset the error code
tstrerror(code), defaultPath);
code = 0; // reset the error code
}
} else { // no valid checkpoint id
stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key, defaultPath);
stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key,
defaultPath);
cleanDir(defaultPath, key);
}
@ -2143,13 +2141,18 @@ void taskDbDestroy(void* pDb, bool flush) {
rocksdb_flushoptions_destroy(flushOpt);
}
}
for (int i = 0; i < nCf; i++) {
if (wrapper->pCf[i] != NULL) {
rocksdb_column_family_handle_destroy(wrapper->pCf[i]);
if (wrapper->pCf != NULL) {
for (int i = 0; i < nCf; i++) {
if (wrapper->pCf[i] != NULL) {
rocksdb_column_family_handle_destroy(wrapper->pCf[i]);
}
}
}
if (wrapper->db) rocksdb_close(wrapper->db);
if (wrapper->db) {
rocksdb_close(wrapper->db);
}
rocksdb_options_destroy(wrapper->dbOpt);
rocksdb_readoptions_destroy(wrapper->readOpt);
@ -2205,7 +2208,8 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char
return code;
}
int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list, const char* idStr) {
int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list,
const char* idStr) {
int32_t code = 0;
SBkdMgt* p = (SBkdMgt*)bkdChkpMgt;
@ -2224,7 +2228,8 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64
return code;
}
int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list, const char* idStr) {
int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list,
const char* idStr) {
int32_t code = -1;
STaskDbWrapper* pDb = arg;
ECHECKPOINT_BACKUP_TYPE utype = type;
@ -3422,7 +3427,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const
size_t kLen = 0;
char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen);
winKeyDecode((void*)&curKey, keyStr);
if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) > 0) {
if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) < 0) {
return pCur;
}
rocksdb_iter_next(pCur->iter);
@ -3459,7 +3464,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const
size_t kLen = 0;
char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen);
winKeyDecode((void*)&curKey, keyStr);
if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) < 0) {
if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) > 0) {
return pCur;
}
rocksdb_iter_prev(pCur->iter);

18
source/libs/stream/src/streamCheckStatus.c
View File

@ -58,7 +58,8 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
}
if (pInfo->stage < stage) {
stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 ", prev:%" PRId64,
stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64
", prev:%" PRId64,
id, upstreamTaskId, vgId, stage, pInfo->stage);
// record the checkpoint failure id and sent to mnode
taosThreadMutexLock(&pTask->lock);
@ -70,7 +71,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
}
if (pInfo->stage != stage) {
taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state;
if (status == TASK_STATUS__CK) {
@ -168,7 +168,8 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
} else {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId);
if (pTask != NULL) {
pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
pRsp->status =
streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
SStreamTaskState* pState = streamTaskGetStatus(pTask);
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64
@ -183,7 +184,6 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status);
}
}
}
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
@ -402,7 +402,8 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) {
if (reqId != p->reqId) {
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded",
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED;
@ -435,7 +436,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
ASSERT(pInfo->startTs > 0);
stError("s-task:%s already in check procedure, checkTs:%" PRId64 ", start monitor check rsp failed", id,
pInfo->startTs);
pInfo->stopCheckProcess = 0; // disable auto stop of check process
pInfo->stopCheckProcess = 0; // disable auto stop of check process
return TSDB_CODE_FAILED;
}
@ -509,8 +510,8 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher;
setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id,
pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@ -760,4 +761,3 @@ void rspMonitorFn(void* param, void* tmrId) {
taosArrayDestroy(pNotReadyList);
taosArrayDestroy(pTimeoutList);
}

Some files were not shown because too many files have changed in this diff Show More