xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2024-06-03 15:56:02 +08:00
parent f13dc1858a 6f72fd4170
commit 927e2237de
46 changed files with 3179 additions and 1357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/zh/12-taos-sql/14-stream.md
View File

@ -190,7 +190,7 @@ T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的
TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定：
1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果
1. 增量计算，即 IGNORE EXPIRED 0。
2. 直接丢弃，即 IGNORE EXPIRED 1：默认配置，忽略过期数据

2
include/client/taos.h
View File

@ -282,7 +282,7 @@ typedef enum tmq_res_t {
TMQ_RES_INVALID = -1,
TMQ_RES_DATA = 1,
TMQ_RES_TABLE_META = 2,
TMQ_RES_METADATA = 3,
TMQ_RES_METADATA = 3
} tmq_res_t;
typedef struct tmq_topic_assignment {

1
include/common/tcommon.h
View File

@ -144,6 +144,7 @@ enum {
TMQ_MSG_TYPE__EP_RSP,
TMQ_MSG_TYPE__POLL_DATA_META_RSP,
TMQ_MSG_TYPE__WALINFO_RSP,
TMQ_MSG_TYPE__POLL_BATCH_META_RSP,
};
enum {

96
include/common/tmsg.h
View File

@ -158,19 +158,20 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_MAX,
} EShowType;
#define TSDB_ALTER_TABLE_ADD_TAG 1
#define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11
#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13
#define TSDB_ALTER_TABLE_ADD_TAG 1
#define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11
#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13
#define TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION 14
#define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1
@ -2313,10 +2314,10 @@ int32_t tDeserializeSVArbHeartBeatRsp(void* buf, int32_t bufLen, SVArbHeartBeatR
void tFreeSVArbHeartBeatRsp(SVArbHeartBeatRsp* pRsp);
typedef struct {
char* arbToken;
char* arbToken;
int64_t arbTerm;
char* member0Token;
char* member1Token;
char* member0Token;
char* member1Token;
} SVArbCheckSyncReq;
int32_t tSerializeSVArbCheckSyncReq(void* buf, int32_t bufLen, SVArbCheckSyncReq* pReq);
@ -2336,9 +2337,9 @@ int32_t tDeserializeSVArbCheckSyncRsp(void* buf, int32_t bufLen, SVArbCheckSyncR
void tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp* pRsp);
typedef struct {
char* arbToken;
int64_t arbTerm;
char* memberToken;
char* arbToken;
int64_t arbTerm;
char* memberToken;
} SVArbSetAssignedLeaderReq;
int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
@ -2707,8 +2708,8 @@ typedef struct {
int64_t lastTs;
SArray* pVgroupVerList;
// 3.3.0.0
SArray* pCols; // array of SField
int64_t smaId;
SArray* pCols; // array of SField
int64_t smaId;
} SCMCreateStreamReq;
typedef struct {
@ -2755,6 +2756,7 @@ typedef struct {
int32_t autoCommitInterval;
int8_t resetOffsetCfg;
int8_t enableReplay;
int8_t enableBatchMeta;
} SCMSubscribeReq;
static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@ -2775,6 +2777,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval);
tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg);
tlen += taosEncodeFixedI8(buf, pReq->enableReplay);
tlen += taosEncodeFixedI8(buf, pReq->enableBatchMeta);
return tlen;
}
@ -2799,6 +2802,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval);
buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg);
buf = taosDecodeFixedI8(buf, &pReq->enableReplay);
buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta);
return buf;
}
@ -2987,6 +2991,7 @@ typedef struct {
int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
int tDecodeSVCreateTbBatchReq(SDecoder* pCoder, SVCreateTbBatchReq* pReq);
void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq);
typedef struct {
int32_t code;
@ -3577,7 +3582,7 @@ static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t u
pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
pOffsetVal->uid = uid;
pOffsetVal->ts = ts;
if(IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)){
if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) {
taosMemoryFree(pOffsetVal->primaryKey.pData);
}
pOffsetVal->primaryKey = primaryKey;
@ -3677,10 +3682,10 @@ typedef struct {
char* ast;
int64_t deleteMark;
int64_t lastTs;
int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
SArray* pVgroupVerList;
int8_t recursiveTsma;
char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
} SMCreateSmaReq;
int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
@ -3905,11 +3910,12 @@ typedef struct {
STqOffsetVal reqOffset;
int8_t enableReplay;
int8_t sourceExcluded;
int8_t enableBatchMeta;
} SMqPollReq;
int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
int32_t tDeserializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
void tDestroySMqPollReq(SMqPollReq *pReq);
void tDestroySMqPollReq(SMqPollReq* pReq);
typedef struct {
int32_t vgId;
@ -3953,7 +3959,7 @@ typedef struct {
int32_t tEncodeMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp);
int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp);
void tDeleteMqMetaRsp(SMqMetaRsp *pRsp);
void tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
#define MQ_DATA_RSP_VERSION 100
@ -3990,6 +3996,20 @@ int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp);
int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp);
void tDeleteSTaosxRsp(void* pRsp);
typedef struct SMqBatchMetaRsp {
SMqRspHead head; // not serialize
STqOffsetVal rspOffset;
SArray* batchMetaLen;
SArray* batchMetaReq;
void* pMetaBuff; // not serialize
uint32_t metaBuffLen; // not serialize
} SMqBatchMetaRsp;
int32_t tEncodeMqBatchMetaRsp(SEncoder* pEncoder, const SMqBatchMetaRsp* pRsp);
int32_t tDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp);
int32_t tSemiDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp);
void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp* pRsp);
typedef struct {
SMqRspHead head;
char cgroup[TSDB_CGROUP_LEN];
@ -4300,9 +4320,9 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
bool fetchingWithTsmaName; // if we are fetching with tsma name
}STableTSMAInfoReq;
char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
bool fetchingWithTsmaName; // if we are fetching with tsma name
} STableTSMAInfoReq;
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq);
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq);
@ -4325,15 +4345,15 @@ typedef struct {
int32_t version;
int64_t interval;
int8_t unit;
SArray* pFuncs; // SArray<STableTSMAFuncInfo>
SArray* pTags; // SArray<SSchema>
SArray* pUsedCols; // SArray<SSchema>
SArray* pFuncs; // SArray<STableTSMAFuncInfo>
SArray* pTags; // SArray<SSchema>
SArray* pUsedCols; // SArray<SSchema>
char* ast;
int64_t streamUid;
int64_t reqTs;
int64_t rspTs;
int64_t delayDuration; // ms
int64_t delayDuration; // ms
bool fillHistoryFinished;
} STableTSMAInfo;
@ -4344,10 +4364,10 @@ void tFreeTableTSMAInfo(void* p);
void tFreeAndClearTableTSMAInfo(void* p);
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp);
#define STSMAHbRsp STableTSMAInfoRsp
#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
#define STSMAHbRsp STableTSMAInfoRsp
#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
#define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp
#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
typedef struct SStreamProgressReq {
int64_t streamId;
@ -4373,7 +4393,7 @@ int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgres
typedef struct SDropCtbWithTsmaSingleVgReq {
SVgroupInfo vgInfo;
SArray* pTbs; // SVDropTbReq
SArray* pTbs; // SVDropTbReq
} SMDropTbReqsOnSingleVg;
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq);
@ -4381,7 +4401,7 @@ int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg*
void tFreeSMDropTbReqOnSingleVg(void* p);
typedef struct SDropTbsReq {
SArray* pVgReqs; // SMDropTbReqsOnSingleVg
SArray* pVgReqs; // SMDropTbReqsOnSingleVg
} SMDropTbsReq;
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq);

750
include/common/ttokendef.h
View File

@ -16,381 +16,381 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_ADD 39
#define TK_DROP 40
#define TK_GRANT 41
#define TK_ON 42
#define TK_TO 43
#define TK_REVOKE 44
#define TK_FROM 45
#define TK_SUBSCRIBE 46
#define TK_READ 47
#define TK_WRITE 48
#define TK_NK_DOT 49
#define TK_WITH 50
#define TK_ENCRYPT_KEY 51
#define TK_DNODE 52
#define TK_PORT 53
#define TK_DNODES 54
#define TK_RESTORE 55
#define TK_NK_IPTOKEN 56
#define TK_FORCE 57
#define TK_UNSAFE 58
#define TK_CLUSTER 59
#define TK_LOCAL 60
#define TK_QNODE 61
#define TK_BNODE 62
#define TK_SNODE 63
#define TK_MNODE 64
#define TK_VNODE 65
#define TK_DATABASE 66
#define TK_USE 67
#define TK_FLUSH 68
#define TK_TRIM 69
#define TK_S3MIGRATE 70
#define TK_COMPACT 71
#define TK_IF 72
#define TK_NOT 73
#define TK_EXISTS 74
#define TK_BUFFER 75
#define TK_CACHEMODEL 76
#define TK_CACHESIZE 77
#define TK_COMP 78
#define TK_DURATION 79
#define TK_NK_VARIABLE 80
#define TK_MAXROWS 81
#define TK_MINROWS 82
#define TK_KEEP 83
#define TK_PAGES 84
#define TK_PAGESIZE 85
#define TK_TSDB_PAGESIZE 86
#define TK_PRECISION 87
#define TK_REPLICA 88
#define TK_VGROUPS 89
#define TK_SINGLE_STABLE 90
#define TK_RETENTIONS 91
#define TK_SCHEMALESS 92
#define TK_WAL_LEVEL 93
#define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_ROLL_PERIOD 97
#define TK_WAL_SEGMENT_SIZE 98
#define TK_STT_TRIGGER 99
#define TK_TABLE_PREFIX 100
#define TK_TABLE_SUFFIX 101
#define TK_S3_CHUNKSIZE 102
#define TK_S3_KEEPLOCAL 103
#define TK_S3_COMPACT 104
#define TK_KEEP_TIME_OFFSET 105
#define TK_ENCRYPT_ALGORITHM 106
#define TK_NK_COLON 107
#define TK_BWLIMIT 108
#define TK_START 109
#define TK_TIMESTAMP 110
#define TK_END 111
#define TK_TABLE 112
#define TK_NK_LP 113
#define TK_NK_RP 114
#define TK_STABLE 115
#define TK_COLUMN 116
#define TK_MODIFY 117
#define TK_RENAME 118
#define TK_TAG 119
#define TK_SET 120
#define TK_NK_EQ 121
#define TK_USING 122
#define TK_TAGS 123
#define TK_BOOL 124
#define TK_TINYINT 125
#define TK_SMALLINT 126
#define TK_INT 127
#define TK_INTEGER 128
#define TK_BIGINT 129
#define TK_FLOAT 130
#define TK_DOUBLE 131
#define TK_BINARY 132
#define TK_NCHAR 133
#define TK_UNSIGNED 134
#define TK_JSON 135
#define TK_VARCHAR 136
#define TK_MEDIUMBLOB 137
#define TK_BLOB 138
#define TK_VARBINARY 139
#define TK_GEOMETRY 140
#define TK_DECIMAL 141
#define TK_COMMENT 142
#define TK_MAX_DELAY 143
#define TK_WATERMARK 144
#define TK_ROLLUP 145
#define TK_TTL 146
#define TK_SMA 147
#define TK_DELETE_MARK 148
#define TK_FIRST 149
#define TK_LAST 150
#define TK_SHOW 151
#define TK_PRIVILEGES 152
#define TK_DATABASES 153
#define TK_TABLES 154
#define TK_STABLES 155
#define TK_MNODES 156
#define TK_QNODES 157
#define TK_ARBGROUPS 158
#define TK_FUNCTIONS 159
#define TK_INDEXES 160
#define TK_ACCOUNTS 161
#define TK_APPS 162
#define TK_CONNECTIONS 163
#define TK_LICENCES 164
#define TK_GRANTS 165
#define TK_FULL 166
#define TK_LOGS 167
#define TK_MACHINES 168
#define TK_ENCRYPTIONS 169
#define TK_QUERIES 170
#define TK_SCORES 171
#define TK_TOPICS 172
#define TK_VARIABLES 173
#define TK_BNODES 174
#define TK_SNODES 175
#define TK_TRANSACTIONS 176
#define TK_DISTRIBUTED 177
#define TK_CONSUMERS 178
#define TK_SUBSCRIPTIONS 179
#define TK_VNODES 180
#define TK_ALIVE 181
#define TK_VIEWS 182
#define TK_VIEW 183
#define TK_COMPACTS 184
#define TK_NORMAL 185
#define TK_CHILD 186
#define TK_LIKE 187
#define TK_TBNAME 188
#define TK_QTAGS 189
#define TK_AS 190
#define TK_SYSTEM 191
#define TK_TSMA 192
#define TK_INTERVAL 193
#define TK_RECURSIVE 194
#define TK_TSMAS 195
#define TK_FUNCTION 196
#define TK_INDEX 197
#define TK_COUNT 198
#define TK_LAST_ROW 199
#define TK_META 200
#define TK_ONLY 201
#define TK_TOPIC 202
#define TK_CONSUMER 203
#define TK_GROUP 204
#define TK_DESC 205
#define TK_DESCRIBE 206
#define TK_RESET 207
#define TK_QUERY 208
#define TK_CACHE 209
#define TK_EXPLAIN 210
#define TK_ANALYZE 211
#define TK_VERBOSE 212
#define TK_NK_BOOL 213
#define TK_RATIO 214
#define TK_NK_FLOAT 215
#define TK_OUTPUTTYPE 216
#define TK_AGGREGATE 217
#define TK_BUFSIZE 218
#define TK_LANGUAGE 219
#define TK_REPLACE 220
#define TK_STREAM 221
#define TK_INTO 222
#define TK_PAUSE 223
#define TK_RESUME 224
#define TK_PRIMARY 225
#define TK_KEY 226
#define TK_TRIGGER 227
#define TK_AT_ONCE 228
#define TK_WINDOW_CLOSE 229
#define TK_IGNORE 230
#define TK_EXPIRED 231
#define TK_FILL_HISTORY 232
#define TK_UPDATE 233
#define TK_SUBTABLE 234
#define TK_UNTREATED 235
#define TK_KILL 236
#define TK_CONNECTION 237
#define TK_TRANSACTION 238
#define TK_BALANCE 239
#define TK_VGROUP 240
#define TK_LEADER 241
#define TK_MERGE 242
#define TK_REDISTRIBUTE 243
#define TK_SPLIT 244
#define TK_DELETE 245
#define TK_INSERT 246
#define TK_NK_BIN 247
#define TK_NK_HEX 248
#define TK_NULL 249
#define TK_NK_QUESTION 250
#define TK_NK_ALIAS 251
#define TK_NK_ARROW 252
#define TK_ROWTS 253
#define TK_QSTART 254
#define TK_QEND 255
#define TK_QDURATION 256
#define TK_WSTART 257
#define TK_WEND 258
#define TK_WDURATION 259
#define TK_IROWTS 260
#define TK_ISFILLED 261
#define TK_CAST 262
#define TK_NOW 263
#define TK_TODAY 264
#define TK_TIMEZONE 265
#define TK_CLIENT_VERSION 266
#define TK_SERVER_VERSION 267
#define TK_SERVER_STATUS 268
#define TK_CURRENT_USER 269
#define TK_CASE 270
#define TK_WHEN 271
#define TK_THEN 272
#define TK_ELSE 273
#define TK_BETWEEN 274
#define TK_IS 275
#define TK_NK_LT 276
#define TK_NK_GT 277
#define TK_NK_LE 278
#define TK_NK_GE 279
#define TK_NK_NE 280
#define TK_MATCH 281
#define TK_NMATCH 282
#define TK_CONTAINS 283
#define TK_IN 284
#define TK_JOIN 285
#define TK_INNER 286
#define TK_LEFT 287
#define TK_RIGHT 288
#define TK_OUTER 289
#define TK_SEMI 290
#define TK_ANTI 291
#define TK_ASOF 292
#define TK_WINDOW 293
#define TK_WINDOW_OFFSET 294
#define TK_JLIMIT 295
#define TK_SELECT 296
#define TK_NK_HINT 297
#define TK_DISTINCT 298
#define TK_WHERE 299
#define TK_PARTITION 300
#define TK_BY 301
#define TK_SESSION 302
#define TK_STATE_WINDOW 303
#define TK_EVENT_WINDOW 304
#define TK_COUNT_WINDOW 305
#define TK_SLIDING 306
#define TK_FILL 307
#define TK_VALUE 308
#define TK_VALUE_F 309
#define TK_NONE 310
#define TK_PREV 311
#define TK_NULL_F 312
#define TK_LINEAR 313
#define TK_NEXT 314
#define TK_HAVING 315
#define TK_RANGE 316
#define TK_EVERY 317
#define TK_ORDER 318
#define TK_SLIMIT 319
#define TK_SOFFSET 320
#define TK_LIMIT 321
#define TK_OFFSET 322
#define TK_ASC 323
#define TK_NULLS 324
#define TK_ABORT 325
#define TK_AFTER 326
#define TK_ATTACH 327
#define TK_BEFORE 328
#define TK_BEGIN 329
#define TK_BITAND 330
#define TK_BITNOT 331
#define TK_BITOR 332
#define TK_BLOCKS 333
#define TK_CHANGE 334
#define TK_COMMA 335
#define TK_CONCAT 336
#define TK_CONFLICT 337
#define TK_COPY 338
#define TK_DEFERRED 339
#define TK_DELIMITERS 340
#define TK_DETACH 341
#define TK_DIVIDE 342
#define TK_DOT 343
#define TK_EACH 344
#define TK_FAIL 345
#define TK_FILE 346
#define TK_FOR 347
#define TK_GLOB 348
#define TK_ID 349
#define TK_IMMEDIATE 350
#define TK_IMPORT 351
#define TK_INITIALLY 352
#define TK_INSTEAD 353
#define TK_ISNULL 354
#define TK_MODULES 355
#define TK_NK_BITNOT 356
#define TK_NK_SEMI 357
#define TK_NOTNULL 358
#define TK_OF 359
#define TK_PLUS 360
#define TK_PRIVILEGE 361
#define TK_RAISE 362
#define TK_RESTRICT 363
#define TK_ROW 364
#define TK_STAR 365
#define TK_STATEMENT 366
#define TK_STRICT 367
#define TK_STRING 368
#define TK_TIMES 369
#define TK_VALUES 370
#define TK_VARIABLE 371
#define TK_WAL 372
#define TK_ENCODE 373
#define TK_COMPRESS 374
#define TK_LEVEL 375
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_ADD 39
#define TK_DROP 40
#define TK_GRANT 41
#define TK_ON 42
#define TK_TO 43
#define TK_REVOKE 44
#define TK_FROM 45
#define TK_SUBSCRIBE 46
#define TK_READ 47
#define TK_WRITE 48
#define TK_NK_DOT 49
#define TK_WITH 50
#define TK_ENCRYPT_KEY 51
#define TK_DNODE 52
#define TK_PORT 53
#define TK_DNODES 54
#define TK_RESTORE 55
#define TK_NK_IPTOKEN 56
#define TK_FORCE 57
#define TK_UNSAFE 58
#define TK_CLUSTER 59
#define TK_LOCAL 60
#define TK_QNODE 61
#define TK_BNODE 62
#define TK_SNODE 63
#define TK_MNODE 64
#define TK_VNODE 65
#define TK_DATABASE 66
#define TK_USE 67
#define TK_FLUSH 68
#define TK_TRIM 69
#define TK_S3MIGRATE 70
#define TK_COMPACT 71
#define TK_IF 72
#define TK_NOT 73
#define TK_EXISTS 74
#define TK_BUFFER 75
#define TK_CACHEMODEL 76
#define TK_CACHESIZE 77
#define TK_COMP 78
#define TK_DURATION 79
#define TK_NK_VARIABLE 80
#define TK_MAXROWS 81
#define TK_MINROWS 82
#define TK_KEEP 83
#define TK_PAGES 84
#define TK_PAGESIZE 85
#define TK_TSDB_PAGESIZE 86
#define TK_PRECISION 87
#define TK_REPLICA 88
#define TK_VGROUPS 89
#define TK_SINGLE_STABLE 90
#define TK_RETENTIONS 91
#define TK_SCHEMALESS 92
#define TK_WAL_LEVEL 93
#define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_ROLL_PERIOD 97
#define TK_WAL_SEGMENT_SIZE 98
#define TK_STT_TRIGGER 99
#define TK_TABLE_PREFIX 100
#define TK_TABLE_SUFFIX 101
#define TK_S3_CHUNKSIZE 102
#define TK_S3_KEEPLOCAL 103
#define TK_S3_COMPACT 104
#define TK_KEEP_TIME_OFFSET 105
#define TK_ENCRYPT_ALGORITHM 106
#define TK_NK_COLON 107
#define TK_BWLIMIT 108
#define TK_START 109
#define TK_TIMESTAMP 110
#define TK_END 111
#define TK_TABLE 112
#define TK_NK_LP 113
#define TK_NK_RP 114
#define TK_STABLE 115
#define TK_COLUMN 116
#define TK_MODIFY 117
#define TK_RENAME 118
#define TK_TAG 119
#define TK_SET 120
#define TK_NK_EQ 121
#define TK_USING 122
#define TK_TAGS 123
#define TK_BOOL 124
#define TK_TINYINT 125
#define TK_SMALLINT 126
#define TK_INT 127
#define TK_INTEGER 128
#define TK_BIGINT 129
#define TK_FLOAT 130
#define TK_DOUBLE 131
#define TK_BINARY 132
#define TK_NCHAR 133
#define TK_UNSIGNED 134
#define TK_JSON 135
#define TK_VARCHAR 136
#define TK_MEDIUMBLOB 137
#define TK_BLOB 138
#define TK_VARBINARY 139
#define TK_GEOMETRY 140
#define TK_DECIMAL 141
#define TK_COMMENT 142
#define TK_MAX_DELAY 143
#define TK_WATERMARK 144
#define TK_ROLLUP 145
#define TK_TTL 146
#define TK_SMA 147
#define TK_DELETE_MARK 148
#define TK_FIRST 149
#define TK_LAST 150
#define TK_SHOW 151
#define TK_PRIVILEGES 152
#define TK_DATABASES 153
#define TK_TABLES 154
#define TK_STABLES 155
#define TK_MNODES 156
#define TK_QNODES 157
#define TK_ARBGROUPS 158
#define TK_FUNCTIONS 159
#define TK_INDEXES 160
#define TK_ACCOUNTS 161
#define TK_APPS 162
#define TK_CONNECTIONS 163
#define TK_LICENCES 164
#define TK_GRANTS 165
#define TK_FULL 166
#define TK_LOGS 167
#define TK_MACHINES 168
#define TK_ENCRYPTIONS 169
#define TK_QUERIES 170
#define TK_SCORES 171
#define TK_TOPICS 172
#define TK_VARIABLES 173
#define TK_BNODES 174
#define TK_SNODES 175
#define TK_TRANSACTIONS 176
#define TK_DISTRIBUTED 177
#define TK_CONSUMERS 178
#define TK_SUBSCRIPTIONS 179
#define TK_VNODES 180
#define TK_ALIVE 181
#define TK_VIEWS 182
#define TK_VIEW 183
#define TK_COMPACTS 184
#define TK_NORMAL 185
#define TK_CHILD 186
#define TK_LIKE 187
#define TK_TBNAME 188
#define TK_QTAGS 189
#define TK_AS 190
#define TK_SYSTEM 191
#define TK_TSMA 192
#define TK_INTERVAL 193
#define TK_RECURSIVE 194
#define TK_TSMAS 195
#define TK_FUNCTION 196
#define TK_INDEX 197
#define TK_COUNT 198
#define TK_LAST_ROW 199
#define TK_META 200
#define TK_ONLY 201
#define TK_TOPIC 202
#define TK_CONSUMER 203
#define TK_GROUP 204
#define TK_DESC 205
#define TK_DESCRIBE 206
#define TK_RESET 207
#define TK_QUERY 208
#define TK_CACHE 209
#define TK_EXPLAIN 210
#define TK_ANALYZE 211
#define TK_VERBOSE 212
#define TK_NK_BOOL 213
#define TK_RATIO 214
#define TK_NK_FLOAT 215
#define TK_OUTPUTTYPE 216
#define TK_AGGREGATE 217
#define TK_BUFSIZE 218
#define TK_LANGUAGE 219
#define TK_REPLACE 220
#define TK_STREAM 221
#define TK_INTO 222
#define TK_PAUSE 223
#define TK_RESUME 224
#define TK_PRIMARY 225
#define TK_KEY 226
#define TK_TRIGGER 227
#define TK_AT_ONCE 228
#define TK_WINDOW_CLOSE 229
#define TK_IGNORE 230
#define TK_EXPIRED 231
#define TK_FILL_HISTORY 232
#define TK_UPDATE 233
#define TK_SUBTABLE 234
#define TK_UNTREATED 235
#define TK_KILL 236
#define TK_CONNECTION 237
#define TK_TRANSACTION 238
#define TK_BALANCE 239
#define TK_VGROUP 240
#define TK_LEADER 241
#define TK_MERGE 242
#define TK_REDISTRIBUTE 243
#define TK_SPLIT 244
#define TK_DELETE 245
#define TK_INSERT 246
#define TK_NK_BIN 247
#define TK_NK_HEX 248
#define TK_NULL 249
#define TK_NK_QUESTION 250
#define TK_NK_ALIAS 251
#define TK_NK_ARROW 252
#define TK_ROWTS 253
#define TK_QSTART 254
#define TK_QEND 255
#define TK_QDURATION 256
#define TK_WSTART 257
#define TK_WEND 258
#define TK_WDURATION 259
#define TK_IROWTS 260
#define TK_ISFILLED 261
#define TK_CAST 262
#define TK_NOW 263
#define TK_TODAY 264
#define TK_TIMEZONE 265
#define TK_CLIENT_VERSION 266
#define TK_SERVER_VERSION 267
#define TK_SERVER_STATUS 268
#define TK_CURRENT_USER 269
#define TK_CASE 270
#define TK_WHEN 271
#define TK_THEN 272
#define TK_ELSE 273
#define TK_BETWEEN 274
#define TK_IS 275
#define TK_NK_LT 276
#define TK_NK_GT 277
#define TK_NK_LE 278
#define TK_NK_GE 279
#define TK_NK_NE 280
#define TK_MATCH 281
#define TK_NMATCH 282
#define TK_CONTAINS 283
#define TK_IN 284
#define TK_JOIN 285
#define TK_INNER 286
#define TK_LEFT 287
#define TK_RIGHT 288
#define TK_OUTER 289
#define TK_SEMI 290
#define TK_ANTI 291
#define TK_ASOF 292
#define TK_WINDOW 293
#define TK_WINDOW_OFFSET 294
#define TK_JLIMIT 295
#define TK_SELECT 296
#define TK_NK_HINT 297
#define TK_DISTINCT 298
#define TK_WHERE 299
#define TK_PARTITION 300
#define TK_BY 301
#define TK_SESSION 302
#define TK_STATE_WINDOW 303
#define TK_EVENT_WINDOW 304
#define TK_COUNT_WINDOW 305
#define TK_SLIDING 306
#define TK_FILL 307
#define TK_VALUE 308
#define TK_VALUE_F 309
#define TK_NONE 310
#define TK_PREV 311
#define TK_NULL_F 312
#define TK_LINEAR 313
#define TK_NEXT 314
#define TK_HAVING 315
#define TK_RANGE 316
#define TK_EVERY 317
#define TK_ORDER 318
#define TK_SLIMIT 319
#define TK_SOFFSET 320
#define TK_LIMIT 321
#define TK_OFFSET 322
#define TK_ASC 323
#define TK_NULLS 324
#define TK_ABORT 325
#define TK_AFTER 326
#define TK_ATTACH 327
#define TK_BEFORE 328
#define TK_BEGIN 329
#define TK_BITAND 330
#define TK_BITNOT 331
#define TK_BITOR 332
#define TK_BLOCKS 333
#define TK_CHANGE 334
#define TK_COMMA 335
#define TK_CONCAT 336
#define TK_CONFLICT 337
#define TK_COPY 338
#define TK_DEFERRED 339
#define TK_DELIMITERS 340
#define TK_DETACH 341
#define TK_DIVIDE 342
#define TK_DOT 343
#define TK_EACH 344
#define TK_FAIL 345
#define TK_FILE 346
#define TK_FOR 347
#define TK_GLOB 348
#define TK_ID 349
#define TK_IMMEDIATE 350
#define TK_IMPORT 351
#define TK_INITIALLY 352
#define TK_INSTEAD 353
#define TK_ISNULL 354
#define TK_MODULES 355
#define TK_NK_BITNOT 356
#define TK_NK_SEMI 357
#define TK_NOTNULL 358
#define TK_OF 359
#define TK_PLUS 360
#define TK_PRIVILEGE 361
#define TK_RAISE 362
#define TK_RESTRICT 363
#define TK_ROW 364
#define TK_STAR 365
#define TK_STATEMENT 366
#define TK_STRICT 367
#define TK_STRING 368
#define TK_TIMES 369
#define TK_VALUES 370
#define TK_VARIABLE 371
#define TK_WAL 372
#define TK_ENCODE 373
#define TK_COMPRESS 374
#define TK_LEVEL 375
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

2
include/libs/executor/executor.h
View File

@ -201,7 +201,7 @@ void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded);
void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);

2
include/util/tbuffer.inc
View File

@ -177,7 +177,7 @@ static int32_t tBufferReaderInit(SBufferReader *reader, uint32_t offset, SBuffer
}
static FORCE_INLINE int32_t tBufferGet(SBufferReader *reader, uint32_t size, void *data) {
if (reader->offset < 0 || reader->offset + size > reader->buffer->size) {
if (reader->offset + size > reader->buffer->size) {
return TSDB_CODE_OUT_OF_RANGE;
}
if (data) {

1
include/util/tencode.h
View File

@ -77,6 +77,7 @@ typedef struct {
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tEncodeSize(E, S, SIZE, RET) \
do { \

2
include/util/tutil.h
View File

@ -119,7 +119,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \
do { \
if ((CODE)) { \
if (TSDB_CODE_SUCCESS != (CODE)) { \
LINO = __LINE__; \
goto LABEL; \
} \

15
source/client/inc/clientInt.h
View File

@ -44,6 +44,7 @@ enum {
RES_TYPE__TMQ,
RES_TYPE__TMQ_META,
RES_TYPE__TMQ_METADATA,
RES_TYPE__TMQ_BATCH_META,
};
#define SHOW_VARIABLES_RESULT_COLS 3
@ -51,10 +52,11 @@ enum {
#define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
#define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ)
#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA)
#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ)
#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA)
#define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)res == RES_TYPE__TMQ_BATCH_META)
typedef struct SAppInstInfo SAppInstInfo;
@ -241,6 +243,11 @@ typedef struct {
STaosxRsp rsp;
} SMqTaosxRspObj;
typedef struct {
SMqRspObjCommon common;
SMqBatchMetaRsp rsp;
} SMqBatchMetaRspObj;
typedef struct SReqRelInfo {
uint64_t userRefId;
uint64_t prevRefId;

34
source/client/src/clientMain.c
View File

@ -301,7 +301,7 @@ void taos_close(TAOS *taos) {
}
int taos_errno(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return terrno;
}
@ -313,7 +313,7 @@ int taos_errno(TAOS_RES *res) {
}
const char *taos_errstr(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return (const char *)tstrerror(terrno);
}
@ -354,6 +354,10 @@ void taos_free_result(TAOS_RES *res) {
SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res;
tDeleteMqMetaRsp(&pRspObj->metaRsp);
taosMemoryFree(pRspObj);
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj *pBtRspObj = (SMqBatchMetaRspObj *)res;
tDeleteMqBatchMetaRsp(&pBtRspObj->rsp);
taosMemoryFree(pBtRspObj);
}
}
@ -371,7 +375,7 @@ void taos_kill_query(TAOS *taos) {
}
int taos_field_count(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -382,7 +386,7 @@ int taos_field_count(TAOS_RES *res) {
int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res)) {
if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
@ -437,7 +441,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
pResultInfo->current += 1;
return pResultInfo->row;
}
} else if (TD_RES_TMQ_META(res)) {
} else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
} else {
// assert to avoid un-initialization error
@ -548,7 +552,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
}
int *taos_fetch_lengths(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
@ -557,7 +561,7 @@ int *taos_fetch_lengths(TAOS_RES *res) {
}
TAOS_ROW *taos_result_block(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -625,7 +629,7 @@ const char *taos_get_client_info() { return version; }
// return int32_t
int taos_affected_rows(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -636,7 +640,7 @@ int taos_affected_rows(TAOS_RES *res) {
// return int64_t
int64_t taos_affected_rows64(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -646,7 +650,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) {
}
int taos_result_precision(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return TSDB_TIME_PRECISION_MILLI;
}
@ -686,7 +690,7 @@ int taos_select_db(TAOS *taos, const char *db) {
}
void taos_stop_query(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return;
}
@ -694,7 +698,7 @@ void taos_stop_query(TAOS_RES *res) {
}
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return true;
}
SReqResultInfo *pResultInfo = tscGetCurResInfo(res);
@ -719,7 +723,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
}
int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -761,7 +765,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
*numOfRows = 0;
*pData = NULL;
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -797,7 +801,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
}
int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
if (res == NULL || TD_RES_TMQ_META(res)) {
if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return 0;
}

334
source/client/src/clientRawBlockWrite.c
View File

@ -26,9 +26,13 @@
#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
#define TMQ_META_VERSION "1.0"
static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
static cJSON* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
int8_t t, SColCmprWrapper* pColCmprRow) {
int8_t buildDefaultCompress = 0;
if (pColCmprRow->nCols <= 0) {
@ -122,9 +126,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
}
cJSON_AddItemToObject(json, "tags", tags);
string = cJSON_PrintUnformatted(json);
cJSON_Delete(json);
return string;
return json;
}
static int32_t setCompressOption(cJSON* json, uint32_t para) {
@ -151,7 +153,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
}
return 0;
}
static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
static cJSON* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
SMAlterStbReq req = {0};
cJSON* json = NULL;
char* string = NULL;
@ -199,6 +201,26 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
}
break;
}
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
SFieldWithOptions* field = taosArrayGet(req.pFields, 0);
cJSON* colName = cJSON_CreateString(field->name);
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(field->type);
cJSON_AddItemToObject(json, "colType", colType);
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY ||
field->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = field->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
} else if (field->type == TSDB_DATA_TYPE_NCHAR) {
int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
}
setCompressOption(json, field->compress);
break;
}
case TSDB_ALTER_TABLE_DROP_TAG:
case TSDB_ALTER_TABLE_DROP_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
@ -245,18 +267,16 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
default:
break;
}
string = cJSON_PrintUnformatted(json);
end:
cJSON_Delete(json);
tFreeSMAltertbReq(&req);
return string;
return json;
}
static char* processCreateStb(SMqMetaRsp* metaRsp) {
static cJSON* processCreateStb(SMqMetaRsp* metaRsp) {
SVCreateStbReq req = {0};
SDecoder coder;
char* string = NULL;
cJSON* pJson = NULL;
uDebug("create stable data:%p", metaRsp);
// decode and process req
@ -267,17 +287,17 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
goto _err;
}
string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr);
pJson = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr);
_err:
uDebug("create stable return, sql json:%s", string);
uDebug("create stable return, sql json:%s", cJSON_PrintUnformatted(pJson));
tDecoderClear(&coder);
return string;
return pJson;
}
static char* processAlterStb(SMqMetaRsp* metaRsp) {
static cJSON* processAlterStb(SMqMetaRsp* metaRsp) {
SVCreateStbReq req = {0};
SDecoder coder;
char* string = NULL;
cJSON* pJson = NULL;
uDebug("alter stable data:%p", metaRsp);
// decode and process req
@ -288,11 +308,11 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) {
if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
goto _err;
}
string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
pJson = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
_err:
uDebug("alter stable return, sql json:%s", string);
uDebug("alter stable return, sql json:%s", cJSON_PrintUnformatted(pJson));
tDecoderClear(&coder);
return string;
return pJson;
}
static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
@ -385,7 +405,7 @@ end:
taosArrayDestroy(pTagVals);
}
static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
static cJSON* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
char* string = NULL;
cJSON* json = cJSON_CreateObject();
if (json == NULL) {
@ -410,16 +430,14 @@ static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
cJSON_AddItemToArray(createList, create);
}
cJSON_AddItemToObject(json, "createList", createList);
string = cJSON_PrintUnformatted(json);
cJSON_Delete(json);
return string;
return json;
}
static char* processCreateTable(SMqMetaRsp* metaRsp) {
static cJSON* processCreateTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVCreateTbBatchReq req = {0};
SVCreateTbReq* pCreateReq;
char* string = NULL;
cJSON* pJson = NULL;
// decode
uDebug("create table data:%p", metaRsp);
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@ -433,25 +451,18 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
if (req.nReqs > 0) {
pCreateReq = req.pReqs;
if (pCreateReq->type == TSDB_CHILD_TABLE) {
string = buildCreateCTableJson(req.pReqs, req.nReqs);
pJson = buildCreateCTableJson(req.pReqs, req.nReqs);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
pJson = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
TSDB_NORMAL_TABLE, &pCreateReq->colCmpr);
}
}
_exit:
uDebug("create table return, sql json:%s", string);
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
taosMemoryFreeClear(pCreateReq->comment);
taosMemoryFreeClear(pCreateReq->sql);
if (pCreateReq->type == TSDB_CHILD_TABLE) {
taosArrayDestroy(pCreateReq->ctb.tagName);
}
}
uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(pJson));
tDeleteSVCreateTbBatchReq(&req);
tDecoderClear(&decoder);
return string;
return pJson;
}
static char* processAutoCreateTable(STaosxRsp* rsp) {
@ -480,7 +491,9 @@ static char* processAutoCreateTable(STaosxRsp* rsp) {
goto _exit;
}
}
string = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
cJSON* pJson = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
_exit:
uDebug("auto created table return, sql json:%s", string);
for (int i = 0; i < rsp->createTableNum; i++) {
@ -495,7 +508,7 @@ _exit:
return string;
}
static char* processAlterTable(SMqMetaRsp* metaRsp) {
static cJSON* processAlterTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVAlterTbReq vAlterTbReq = {0};
char* string = NULL;
@ -546,6 +559,25 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
}
break;
}
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
cJSON* colName = cJSON_CreateString(vAlterTbReq.colName);
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type);
cJSON_AddItemToObject(json, "colType", colType);
if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY ||
vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
} else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) {
int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
}
setCompressOption(json, vAlterTbReq.compress);
break;
}
case TSDB_ALTER_TABLE_DROP_COLUMN: {
cJSON* colName = cJSON_CreateString(vAlterTbReq.colName);
cJSON_AddItemToObject(json, "colName", colName);
@ -620,19 +652,16 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
default:
break;
}
string = cJSON_PrintUnformatted(json);
_exit:
uDebug("alter table return, sql json:%s", string);
cJSON_Delete(json);
uDebug("alter table return, sql json:%s", cJSON_PrintUnformatted(json));
tDecoderClear(&decoder);
return string;
return json;
}
static char* processDropSTable(SMqMetaRsp* metaRsp) {
static cJSON* processDropSTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVDropStbReq req = {0};
char* string = NULL;
cJSON* json = NULL;
uDebug("processDropSTable data:%p", metaRsp);
@ -657,18 +686,15 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
cJSON* tableName = cJSON_CreateString(req.name);
cJSON_AddItemToObject(json, "tableName", tableName);
string = cJSON_PrintUnformatted(json);
_exit:
uDebug("processDropSTable return, sql json:%s", string);
cJSON_Delete(json);
uDebug("processDropSTable return, sql json:%s", cJSON_PrintUnformatted(json));
tDecoderClear(&decoder);
return string;
return json;
}
static char* processDeleteTable(SMqMetaRsp* metaRsp) {
static cJSON* processDeleteTable(SMqMetaRsp* metaRsp) {
SDeleteRes req = {0};
SDecoder coder = {0};
cJSON* json = NULL;
char* string = NULL;
uDebug("processDeleteTable data:%p", metaRsp);
// decode and process req
@ -696,18 +722,15 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) {
cJSON* sqlJson = cJSON_CreateString(sql);
cJSON_AddItemToObject(json, "sql", sqlJson);
string = cJSON_PrintUnformatted(json);
_exit:
uDebug("processDeleteTable return, sql json:%s", string);
cJSON_Delete(json);
uDebug("processDeleteTable return, sql json:%s", cJSON_PrintUnformatted(json));
tDecoderClear(&coder);
return string;
return json;
}
static char* processDropTable(SMqMetaRsp* metaRsp) {
static cJSON* processDropTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVDropTbBatchReq req = {0};
char* string = NULL;
cJSON* json = NULL;
uDebug("processDropTable data:%p", metaRsp);
@ -741,12 +764,10 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
}
cJSON_AddItemToObject(json, "tableNameList", tableNameList);
string = cJSON_PrintUnformatted(json);
_exit:
uDebug("processDropTable return, json sql:%s", string);
cJSON_Delete(json);
uDebug("processDropTable return, json sql:%s", cJSON_PrintUnformatted(json));
tDecoderClear(&decoder);
return string;
return json;
}
static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
@ -1121,13 +1142,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
end:
uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
taosMemoryFreeClear(pCreateReq->comment);
if (pCreateReq->type == TSDB_CHILD_TABLE) {
taosArrayDestroy(pCreateReq->ctb.tagName);
}
}
tDeleteSVCreateTbBatchReq(&req);
taosHashCleanup(pVgroupHashmap);
destroyRequest(pRequest);
@ -1803,6 +1818,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
@ -1995,6 +2011,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
@ -2025,38 +2042,85 @@ end:
return code;
}
static cJSON* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) {
return processCreateStb(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_STB) {
return processAlterStb(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_DROP_STB) {
return processDropSTable(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_CREATE_TABLE) {
return processCreateTable(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_TABLE) {
return processAlterTable(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
return processDropTable(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
return processDropTable(pMetaRsp);
} else if (pMetaRsp->resMsgType == TDMT_VND_DELETE) {
return processDeleteTable(pMetaRsp);
}
return NULL;
}
static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
SDecoder coder;
SMqBatchMetaRsp rsp = {0};
tDecoderInit(&coder, pMsgRsp->pMetaBuff, pMsgRsp->metaBuffLen);
if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
goto _end;
}
cJSON* pJson = cJSON_CreateObject();
cJSON_AddStringToObject(pJson, "tmq_meta_version", TMQ_META_VERSION);
cJSON* pMetaArr = cJSON_CreateArray();
int32_t num = taosArrayGetSize(rsp.batchMetaReq);
for (int32_t i = 0; i < num; i++) {
int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
SDecoder metaCoder = {0};
SMqMetaRsp metaRsp = {0};
tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) {
goto _end;
}
cJSON* pItem = processSimpleMeta(&metaRsp);
tDeleteMqMetaRsp(&metaRsp);
cJSON_AddItemToArray(pMetaArr, pItem);
}
cJSON_AddItemToObject(pJson, "metas", pMetaArr);
tDeleteMqBatchMetaRsp(&rsp);
char* fullStr = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
return fullStr;
_end:
cJSON_Delete(pJson);
tDeleteMqBatchMetaRsp(&rsp);
return NULL;
}
char* tmq_get_json_meta(TAOS_RES* res) {
if (res == NULL) return NULL;
uDebug("tmq_get_json_meta res:%p", res);
if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) {
if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res) && !TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
return processAutoCreateTable(&pMetaDataRspObj->rsp);
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res;
return processBatchMetaToJson(&pBatchMetaRspObj->rsp);
}
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) {
return processCreateStb(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) {
return processAlterStb(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) {
return processDropSTable(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) {
return processCreateTable(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) {
return processAlterTable(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) {
return processDropTable(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) {
return processDropTable(&pMetaRspObj->metaRsp);
} else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) {
return processDeleteTable(&pMetaRspObj->metaRsp);
}
return NULL;
cJSON* pJson = processSimpleMeta(&pMetaRspObj->metaRsp);
char* string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
return string;
}
void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
@ -2147,6 +2211,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
}
raw->raw_type = RES_TYPE__TMQ_METADATA;
uDebug("tmq get raw type metadata:%p", raw);
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBtMetaRspObj = (SMqBatchMetaRspObj*)res;
raw->raw = pBtMetaRspObj->rsp.pMetaBuff;
raw->raw_len = pBtMetaRspObj->rsp.metaBuffLen;
raw->raw_type = RES_TYPE__TMQ_BATCH_META;
uDebug("tmq get raw batch meta:%p", raw);
} else {
uError("tmq get raw error type:%d", *(int8_t*)res);
terrno = TSDB_CODE_TMQ_INVALID_MSG;
@ -2163,32 +2233,74 @@ void tmq_free_raw(tmq_raw_data raw) {
memset(terrMsg, 0, ERR_MSG_LEN);
}
static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
if (type == TDMT_VND_CREATE_STB) {
return taosCreateStb(taos, buf, len);
} else if (type == TDMT_VND_ALTER_STB) {
return taosCreateStb(taos, buf, len);
} else if (type == TDMT_VND_DROP_STB) {
return taosDropStb(taos, buf, len);
} else if (type == TDMT_VND_CREATE_TABLE) {
return taosCreateTable(taos, buf, len);
} else if (type == TDMT_VND_ALTER_TABLE) {
return taosAlterTable(taos, buf, len);
} else if (type == TDMT_VND_DROP_TABLE) {
return taosDropTable(taos, buf, len);
} else if (type == TDMT_VND_DELETE) {
return taosDeleteData(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_METADATA) {
return tmqWriteRawMetaDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_BATCH_META) {
return tmqWriteBatchMetaDataImpl(taos, buf, len);
}
return TSDB_CODE_INVALID_PARA;
}
int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
if (!taos) {
goto end;
return TSDB_CODE_INVALID_PARA;
}
if (raw.raw_type == TDMT_VND_CREATE_STB) {
return taosCreateStb(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_ALTER_STB) {
return taosCreateStb(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_DROP_STB) {
return taosDropStb(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_CREATE_TABLE) {
return taosCreateTable(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_ALTER_TABLE) {
return taosAlterTable(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_DROP_TABLE) {
return taosDropTable(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == TDMT_VND_DELETE) {
return taosDeleteData(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len);
} else if (raw.raw_type == RES_TYPE__TMQ_METADATA) {
return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len);
}
end:
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
return writeRawImpl(taos, raw.raw, raw.raw_len, raw.raw_type);
}
static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
SMqBatchMetaRsp rsp = {0};
SDecoder coder;
int32_t code = TSDB_CODE_SUCCESS;
// decode and process req
tDecoderInit(&coder, meta, metaLen);
if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
code = TSDB_CODE_INVALID_PARA;
goto _end;
}
int32_t num = taosArrayGetSize(rsp.batchMetaReq);
for (int32_t i = 0; i < num; i++) {
int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
SDecoder metaCoder = {0};
SMqMetaRsp metaRsp = {0};
tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead));
if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) {
code = TSDB_CODE_INVALID_PARA;
goto _end;
}
code = writeRawImpl(taos, metaRsp.metaRsp, metaRsp.metaRspLen, metaRsp.resMsgType);
tDeleteMqMetaRsp(&metaRsp);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
}
_end:
tDeleteMqBatchMetaRsp(&rsp);
errno = code;
return code;
}

105
source/client/src/clientTmq.c
View File

@ -71,6 +71,7 @@ struct tmq_conf_t {
char* pass;
tmq_commit_cb* commitCb;
void* commitCbUserParam;
int8_t enableBatchMeta;
};
struct tmq_t {
@ -87,6 +88,7 @@ struct tmq_t {
uint64_t consumerId;
tmq_commit_cb* commitCb;
void* commitCbUserParam;
int8_t enableBatchMeta;
// status
SRWLatch lock;
@ -171,9 +173,10 @@ typedef struct {
uint64_t reqId;
SEpSet* pEpset;
union {
SMqDataRsp dataRsp;
SMqMetaRsp metaRsp;
STaosxRsp taosxRsp;
SMqDataRsp dataRsp;
SMqMetaRsp metaRsp;
STaosxRsp taosxRsp;
SMqBatchMetaRsp batchMetaRsp;
};
} SMqPollRspWrapper;
@ -268,6 +271,7 @@ tmq_conf_t* tmq_conf_new() {
conf->autoCommit = true;
conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL;
conf->resetOffset = TMQ_OFFSET__RESET_LATEST;
conf->enableBatchMeta = false;
return conf;
}
@ -397,6 +401,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
return TMQ_CONF_OK;
}
if (strcasecmp(key, "msg.enable.batchmeta") == 0) {
conf->enableBatchMeta = (taosStr2int64(value) != 0) ? true : false;
return TMQ_CONF_OK;
}
return TMQ_CONF_UNKNOWN;
}
@ -635,6 +644,11 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
pTopicName = pRspObj->common.topic;
vgId = pRspObj->common.vgId;
offsetVal = pRspObj->rsp.common.rspOffset;
} else if (TD_RES_TMQ_BATCH_META(pRes)) {
SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)pRes;
pTopicName = pBtRspObj->common.topic;
vgId = pBtRspObj->common.vgId;
offsetVal = pBtRspObj->rsp.rspOffset;
} else {
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
@ -934,7 +948,11 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
tDeleteSTaosxRsp(&pRsp->taosxRsp);
}
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
}
}
void tmqClearUnhandleMsg(tmq_t* tmq) {
@ -1112,6 +1130,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->resetOffsetCfg = conf->resetOffset;
pTmq->replayEnable = conf->replayEnable;
pTmq->sourceExcluded = conf->sourceExcluded;
pTmq->enableBatchMeta = conf->enableBatchMeta;
if (conf->replayEnable) {
pTmq->autoCommit = false;
}
@ -1189,6 +1208,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
req.autoCommitInterval = tmq->autoCommitInterval;
req.resetOffsetCfg = tmq->resetOffsetCfg;
req.enableReplay = tmq->replayEnable;
req.enableBatchMeta = tmq->enableBatchMeta;
for (int32_t i = 0; i < sz; i++) {
char* topic = taosArrayGetP(container, i);
@ -1418,6 +1438,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
}
tDecoderClear(&decoder);
memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
} else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
tDecoderClear(&decoder);
memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,
requestId);
} else { // invalid rspType
tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
}
@ -1600,6 +1633,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
pReq->reqId = generateRequestId();
pReq->enableReplay = tmq->replayEnable;
pReq->sourceExcluded = tmq->sourceExcluded;
pReq->enableBatchMeta = tmq->enableBatchMeta;
}
SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
@ -1616,6 +1650,20 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj;
}
SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj));
if(pRspObj == NULL) {
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META;
tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
pRspObj->common.vgId = pWrapper->vgHandle->vgId;
memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp));
tscDebug("build batchmeta Rsp from wrapper");
return pRspObj;
}
void changeByteEndian(char* pData){
char* p = pData;
@ -2006,6 +2054,42 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
}
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) {
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId);
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return NULL;
}
// build rsp
void* pRsp = NULL;
updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset,
pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever,
tmq->consumerId, true);
pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
} else {
tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->batchMetaRsp.head.epoch, consumerEpoch);
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
}
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
@ -2217,6 +2301,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
return TMQ_RES_TABLE_META;
} else if (TD_RES_TMQ_METADATA(res)) {
return TMQ_RES_METADATA;
} else if (TD_RES_TMQ_BATCH_META(res)) {
return TMQ_RES_TABLE_META;
} else {
return TMQ_RES_INVALID;
}
@ -2226,7 +2312,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
if (res == NULL) {
return NULL;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@ -2241,7 +2327,7 @@ const char* tmq_get_db_name(TAOS_RES* res) {
return NULL;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return strchr(((SMqRspObjCommon*)res)->db, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@ -2255,7 +2341,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
if (res == NULL) {
return -1;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
return ((SMqRspObjCommon*)res)->vgId;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@ -2282,6 +2368,11 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) {
return pRspObj->metaRsp.rspOffset.version;
}
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)res;
if (pBtRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) {
return pBtRspObj->rsp.rspOffset.version;
}
} else {
tscError("invalid tmq type:%d", *(int8_t*)res);
}

3
source/common/src/tcol.c
View File

@ -131,6 +131,7 @@ const char* columnEncodeStr(uint8_t type) {
encode = TSDB_COLUMN_ENCODE_DISABLED;
break;
default:
encode = TSDB_COLUMN_ENCODE_UNKNOWN;
break;
}
return encode;
@ -157,8 +158,8 @@ const char* columnCompressStr(uint16_t type) {
case TSDB_COLVAL_COMPRESS_DISABLED:
compress = TSDB_COLUMN_COMPRESS_DISABLED;
break;
default:
compress = TSDB_COLUMN_COMPRESS_UNKNOWN;
break;
}
return compress;

160
source/common/src/tmsg.c
View File

@ -768,11 +768,22 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
// if (pReq->alterType == )
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
SField *pField = taosArrayGet(pReq->pFields, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
SFieldWithOptions *pField = taosArrayGet(pReq->pFields, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
if (tEncodeU32(&encoder, pField->compress) < 0) return -1;
} else {
SField *pField = taosArrayGet(pReq->pFields, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
}
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
@ -802,13 +813,28 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
}
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
SField field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (taosArrayPush(pReq->pFields, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
taosArrayDestroy(pReq->pFields);
pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SFieldWithOptions));
SFieldWithOptions field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (tDecodeU32(&decoder, &field.compress) < 0) return -1;
if (taosArrayPush(pReq->pFields, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else {
SField field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (taosArrayPush(pReq->pFields, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
}
@ -5616,18 +5642,36 @@ int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *
}
int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
int32_t code = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tStartDecode(&decoder) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
goto _exit;
}
if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->compactId) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
goto _exit;
}
if (tDecodeI32(&decoder, &pReq->vgId) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
goto _exit;
}
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) {
code = TSDB_CODE_MSG_DECODE_ERROR;
goto _exit;
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return 0;
return code;
}
int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeConfigReq *pReq) {
@ -7238,6 +7282,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
pHead->vgId = htonl(pReq->head.vgId);
pHead->contLen = htonl(tlen + headLen);
}
if (tEncodeI8(&encoder, pReq->enableBatchMeta) < 0) return -1;
return tlen + headLen;
}
@ -7267,6 +7312,12 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
if (tDecodeI8(&decoder, &pReq->sourceExcluded) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->enableBatchMeta) < 0) return -1;
} else {
pReq->enableBatchMeta = false;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -8401,6 +8452,18 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
return 0;
}
void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq *pReq) {
for (int32_t iReq = 0; iReq < pReq->nReqs; iReq++) {
SVCreateTbReq *pCreateReq = pReq->pReqs + iReq;
taosMemoryFreeClear(pCreateReq->sql);
taosMemoryFreeClear(pCreateReq->comment);
if (pCreateReq->type == TSDB_CHILD_TABLE) {
taosArrayDestroy(pCreateReq->ctb.tagName);
pCreateReq->ctb.tagName = NULL;
}
}
}
int tEncodeSVCreateTbRsp(SEncoder *pCoder, const SVCreateTbRsp *pRsp) {
if (tStartEncode(pCoder) < 0) return -1;
@ -8755,6 +8818,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1;
break;
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
if (tEncodeI8(pEncoder, pReq->type) < 0) return -1;
if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1;
if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1;
if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1;
break;
default:
break;
}
@ -8810,6 +8880,12 @@ static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq)
if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1;
break;
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1;
if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1;
if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1;
default:
break;
}
@ -10596,3 +10672,55 @@ void tFreeFetchTtlExpiredTbsRsp(void *p) {
SVFetchTtlExpiredTbsRsp *pRsp = p;
taosArrayDestroy(pRsp->pExpiredTbs);
}
int32_t tEncodeMqBatchMetaRsp(SEncoder *pEncoder, const SMqBatchMetaRsp *pRsp) {
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
int32_t size = taosArrayGetSize(pRsp->batchMetaReq);
if (tEncodeI32(pEncoder, size) < 0) return -1;
if (size > 0) {
for (int32_t i = 0; i < size; i++) {
void *pMetaReq = taosArrayGetP(pRsp->batchMetaReq, i);
int32_t metaLen = *(int32_t *)taosArrayGet(pRsp->batchMetaLen, i);
if (tEncodeBinary(pEncoder, pMetaReq, metaLen) < 0) return -1;
}
}
return 0;
}
int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
for (int32_t i = 0; i < size; i++) {
void *pCreate = NULL;
uint64_t len = 0;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len;
taosArrayPush(pRsp->batchMetaReq, &pCreate);
taosArrayPush(pRsp->batchMetaLen, &l);
}
}
return 0;
}
int32_t tSemiDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
if (pDecoder->size < pDecoder->pos) {
return -1;
}
pRsp->metaBuffLen = TD_CODER_REMAIN_CAPACITY(pDecoder);
pRsp->pMetaBuff = taosMemoryCalloc(1, pRsp->metaBuffLen);
memcpy(pRsp->pMetaBuff, TD_CODER_CURRENT(pDecoder), pRsp->metaBuffLen);
return 0;
}
void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) {
taosMemoryFreeClear(pRsp->pMetaBuff);
taosArrayDestroyP(pRsp->batchMetaReq, taosMemoryFree);
taosArrayDestroy(pRsp->batchMetaLen);
pRsp->batchMetaReq = NULL;
pRsp->batchMetaLen = NULL;
}

76
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1779,7 +1779,8 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
return 0;
}
static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) {
static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols,
int8_t withCompress) {
if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) {
terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS;
return -1;
@ -1806,29 +1807,53 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
}
for (int32_t i = 0; i < ncols; i++) {
SField *pField = taosArrayGet(pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
return -1;
if (withCompress) {
SFieldWithOptions *pField = taosArrayGet(pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
return -1;
}
if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
return -1;
}
SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
pSchema->bytes = pField->bytes;
pSchema->type = pField->type;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = pNew->nextColId;
pNew->nextColId++;
SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
pCmpr->id = pSchema->colId;
pCmpr->alg = pField->compress;
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
} else {
SField *pField = taosArrayGet(pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
return -1;
}
if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
return -1;
}
SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
pSchema->bytes = pField->bytes;
pSchema->type = pField->type;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = pNew->nextColId;
pNew->nextColId++;
SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
pCmpr->id = pSchema->colId;
pCmpr->alg = createDefaultColCmprByType(pSchema->type);
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
}
if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
return -1;
}
SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
pSchema->bytes = pField->bytes;
pSchema->type = pField->type;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = pNew->nextColId;
pNew->nextColId++;
SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
pCmpr->id = pSchema->colId;
pCmpr->alg = createDefaultColCmprByType(pSchema->type);
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
}
pNew->colVer++;
@ -2461,7 +2486,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0);
break;
case TSDB_ALTER_TABLE_ADD_COLUMN:
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 0);
break;
case TSDB_ALTER_TABLE_DROP_COLUMN:
pField0 = taosArrayGet(pAlter->pFields, 0);
@ -2478,6 +2503,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
break;
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 1);
break;
default:
needRsp = false;
terrno = TSDB_CODE_OPS_NOT_SUPPORT;

2
source/dnode/vnode/src/inc/tq.h
View File

@ -113,7 +113,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
void tqDestroyTqHandle(void* data);
// tqRead
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset);
int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest);
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);

12
source/dnode/vnode/src/meta/metaTable.c
View File

@ -36,7 +36,7 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) {
int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) {
int32_t nCols = pWp->nCols;
int32_t ver = pWp->version;
if (add) {
@ -45,7 +45,7 @@ int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) {
SColCmpr *pCol = p + nCols;
pCol->id = pSchema->colId;
pCol->alg = createDefaultColCmprByType(pSchema->type);
pCol->alg = compress;
pWp->nCols = nCols + 1;
pWp->version = ver;
pWp->pColCmpr = p;
@ -1491,6 +1491,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
SSchema tScheam;
switch (pAlterTbReq->action) {
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
if (pColumn) {
terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS;
goto _err;
@ -1522,7 +1523,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
(void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type);
}
SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1];
updataTableColCmpr(&entry.colCmpr, pCol, 1);
uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type)
: pAlterTbReq->compress;
updataTableColCmpr(&entry.colCmpr, pCol, 1, compress);
freeColCmpr = true;
ASSERT(entry.colCmpr.nCols == pSchema->nCols);
break;
@ -1560,7 +1563,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
(void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey);
}
updataTableColCmpr(&entry.colCmpr, &tScheam, 0);
updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0);
ASSERT(entry.colCmpr.nCols == pSchema->nCols);
break;
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
@ -2269,6 +2272,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
pMeta->changed = true;
switch (pReq->action) {
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
case TSDB_ALTER_TABLE_DROP_COLUMN:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:

9
source/dnode/vnode/src/tq/tqRead.c
View File

@ -99,14 +99,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
taosArrayDestroy(reqNew.pArray);
}
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
taosMemoryFreeClear(pCreateReq->comment);
taosMemoryFreeClear(pCreateReq->sql);
if (pCreateReq->type == TSDB_CHILD_TABLE) {
taosArrayDestroy(pCreateReq->ctb.tagName);
}
}
tDeleteSVCreateTbBatchReq(&req);
} else if (msgType == TDMT_VND_ALTER_TABLE) {
SVAlterTbReq req = {0};

8
source/dnode/vnode/src/tq/tqScan.c
View File

@ -163,7 +163,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
return 0;
}
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) {
const STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task;
@ -218,10 +218,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
}
// get meta
SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
if (tmp->metaRspLen > 0) {
SMqBatchMetaRsp* tmp = qStreamExtractMetaMsg(task);
if (taosArrayGetSize(tmp->batchMetaReq) > 0) {
qStreamExtractOffset(task, &tmp->rspOffset);
*pMetaRsp = *tmp;
*pBatchMetaRsp = *tmp;
tqDebug("tmqsnap task get meta");
break;

138
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -17,6 +17,8 @@
static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
const SMqMetaRsp* pRsp, int32_t vgId);
static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
const SMqBatchMetaRsp* pRsp, int32_t vgId);
int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
tOffsetCopy(&pRsp->reqOffset, &pOffset);
@ -172,7 +174,7 @@ end : {
}
}
#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \
#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \
SDecoder decoder = {0};\
TYPE req = {0}; \
void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
@ -182,33 +184,37 @@ end : {
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \
pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
fetchVer++; \
DELETE_FUNC(&req); \
tDecoderClear(&decoder); \
continue; \
} \
DELETE_FUNC(&req); \
tDecoderClear(&decoder);
static void tDeleteCommon(void* parm) {
}
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
SMqMetaRsp metaRsp = {0};
STaosxRsp taosxRsp = {0};
int code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0};
tqInitTaosxRsp(&taosxRsp.common, *offset);
if (offset->type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset) < 0) {
code = -1;
goto end;
}
if (metaRsp.metaRspLen > 0) {
code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
if (taosArrayGetSize(btMetaRsp.batchMetaReq) > 0) {
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
",ts:%" PRId64,
pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
metaRsp.rspOffset.ts);
tDeleteMqMetaRsp(&metaRsp);
pRequest->consumerId, pHandle->subKey, vgId, btMetaRsp.rspOffset.type, btMetaRsp.rspOffset.uid,
btMetaRsp.rspOffset.ts);
tDeleteMqBatchMetaRsp(&btMetaRsp);
goto end;
}
@ -230,11 +236,18 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
uint64_t st = taosGetTimestampMs();
int totalRows = 0;
int32_t totalMetaRows = 0;
while (1) {
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
ASSERT(savedEpoch <= pRequest->epoch);
if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) {
if (totalMetaRows > 0) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
ASSERT(totalRows == 0);
goto end;
}
tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer);
code = tqSendDataRsp(
pHandle, pMsg, pRequest, &taosxRsp,
@ -258,22 +271,69 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
if (pHead->msgType == TDMT_VND_CREATE_TABLE) {
PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq)
PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq, tDeleteSVCreateTbBatchReq)
} else if (pHead->msgType == TDMT_VND_ALTER_TABLE) {
PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq)
PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq, tDeleteCommon)
} else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq, tDeleteCommon)
} else if (pHead->msgType == TDMT_VND_DELETE) {
PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes)
PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes, tDeleteCommon)
}
}
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s, enable batch meta:%d", pHead->version, vgId,
TMSG_INFO(pHead->msgType), pRequest->enableBatchMeta);
if (!pRequest->enableBatchMeta && !pRequest->useSnapshot) {
SMqMetaRsp metaRsp = {0};
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
goto end;
}
if (!btMetaRsp.batchMetaReq) {
btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
}
fetchVer++;
SMqMetaRsp tmpMetaRsp = {0};
tmpMetaRsp.resMsgType = pHead->msgType;
tmpMetaRsp.metaRspLen = pHead->bodyLen;
tmpMetaRsp.metaRsp = pHead->body;
uint32_t len = 0;
tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
if (TSDB_CODE_SUCCESS != code) {
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, metaBuff, len);
code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
if (code < 0) {
tEncoderClear(&encoder);
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
taosArrayPush(btMetaRsp.batchMetaReq, &tBuf);
taosArrayPush(btMetaRsp.batchMetaLen, &tLen);
totalMetaRows++;
if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > 1000)) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto end;
}
continue;
}
if (totalMetaRows > 0) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto end;
}
@ -304,7 +364,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
}
end:
tDeleteMqBatchMetaRsp(&btMetaRsp);
tDeleteSTaosxRsp(&taosxRsp);
return code;
}
@ -352,6 +412,40 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int
pMsgHead->walever = ever;
}
int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp,
int32_t vgId) {
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code);
if (code < 0) {
return -1;
}
int32_t tlen = sizeof(SMqRspHead) + len;
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
return -1;
}
int64_t sver = 0, ever = 0;
walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
initMqRspHead(buf, TMQ_MSG_TYPE__POLL_BATCH_META_RSP, pReq->epoch, pReq->consumerId, sver, ever);
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
tEncodeMqBatchMetaRsp(&encoder, pRsp);
tEncoderClear(&encoder);
SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
tmsgSendRsp(&resp);
tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", vgId,
pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type);
return 0;
}
int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp,
int32_t vgId) {
int32_t len = 0;

22
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -211,6 +211,11 @@ _exit:
return code;
}
static int32_t tsdbCommitCloseReader(SCommitter2 *committer) {
TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose);
return 0;
}
static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;
@ -255,13 +260,17 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
_exit:
if (code) {
tsdbCommitCloseReader(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbCommitCloseReader(SCommitter2 *committer) {
TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose);
static int32_t tsdbCommitCloseIter(SCommitter2 *committer) {
tsdbIterMergerClose(&committer->tombIterMerger);
tsdbIterMergerClose(&committer->dataIterMerger);
TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose);
TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose);
return 0;
}
@ -335,19 +344,12 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
_exit:
if (code) {
tsdbCommitCloseIter(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbCommitCloseIter(SCommitter2 *committer) {
tsdbIterMergerClose(&committer->tombIterMerger);
tsdbIterMergerClose(&committer->dataIterMerger);
TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose);
TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose);
return 0;
}
static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
int32_t code = 0;
int32_t lino = 0;

10
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -74,7 +74,7 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
pBlock->file.stt->level = reader->config->file.stt->level;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
@ -130,7 +130,7 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) {
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
@ -211,13 +211,13 @@ _exit:
return code;
}
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength,
encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += pDataBlock->dataLength;

2
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -44,7 +44,7 @@ static const struct {
};
void remove_file(const char *fname) {
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
tsdbInfo("file:%s is removed", fname);
}

17
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -90,6 +90,11 @@ _exit:
return code;
}
static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose);
return 0;
}
static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0;
@ -220,8 +225,10 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(merger->sttReaderArr, reader);
TSDB_CHECK_CODE(code, lino, _exit);
if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) {
tsdbSttFileReaderClose(&reader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
@ -232,6 +239,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
_exit:
if (code) {
tsdbMergeFileSetEndCloseReader(merger);
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
}
return code;
@ -375,11 +383,6 @@ static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) {
return 0;
}
static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose);
return 0;
}
static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0;

4
source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
View File

@ -321,10 +321,10 @@ int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartLis
if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
TARRAY2_APPEND(iList, r);
if (TARRAY2_APPEND(iList, r)) goto _err;
}
}
TARRAY2_APPEND(pList, p);
if (TARRAY2_APPEND(pList, p)) goto _err;
p = NULL;
}

35
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -57,6 +57,20 @@ struct STsdbSnapReader {
STombBlock tombBlock[1];
};
static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose);
tsdbDataFileReaderClose(&reader->dataReader);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
}
static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
@ -100,27 +114,16 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(reader->sttReaderArr, sttReader);
TSDB_CHECK_CODE(code, lino, _exit);
if ((code = TARRAY2_APPEND(reader->sttReaderArr, sttReader))) {
tsdbSttFileReaderClose(&sttReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
}
static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose);
tsdbDataFileReaderClose(&reader->dataReader);
_exit:
if (code) {
tsdbSnapReadFileSetCloseReader(reader);
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;

3
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -141,6 +141,7 @@ static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) {
_exit:
if (code) {
tsdbSnapRAWReadFileSetCloseReader(reader);
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
@ -546,7 +547,7 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD
int32_t lino = 0;
int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);

2
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -896,7 +896,7 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(writer->config->tsdb, writer->file, fname);
tsdbCloseFile(&writer->fd);
taosRemoveFile(fname);
(void)taosRemoveFile(fname);
return 0;
}

6
source/dnode/vnode/src/tsdb/tsdbUtil2.c
View File

@ -56,7 +56,8 @@ int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) {
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * idx, &tombBlock->buffers[i]);
tBufferGetI64(&br, &record->data[i]);
int32_t code = tBufferGetI64(&br, &record->data[i]);
if (code) return code;
}
return 0;
}
@ -182,7 +183,8 @@ int32_t tStatisBlockPut(STbStatisBlock *block, SRowInfo *row, int32_t maxRecords
if (block->numOfRecords > 0) {
int64_t lastUid;
SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * (block->numOfRecords - 1), &block->uids);
tBufferGetI64(&br, &lastUid);
int32_t code = tBufferGetI64(&br, &lastUid);
if (code) return code;
if (lastUid == row->uid) {
return tStatisBlockUpdate(block, row);

7
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1167,12 +1167,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
}
_exit:
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
taosMemoryFree(pCreateReq->sql);
taosMemoryFree(pCreateReq->comment);
taosArrayDestroy(pCreateReq->ctb.tagName);
}
tDeleteSVCreateTbBatchReq(&req);
taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp);
taosArrayDestroy(tbUids);
tDecoderClear(&decoder);

2
source/libs/executor/inc/querytask.h
View File

@ -58,7 +58,7 @@ typedef struct STaskStopInfo {
typedef struct {
STqOffsetVal currentOffset; // for tmq
SMqMetaRsp metaRsp; // for tmq fetching meta
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
int8_t sourceExcluded;
int64_t snapshotVer;
SSchemaWrapper* schema;

4
source/libs/executor/src/executor.c
View File

@ -1078,9 +1078,9 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
return pTaskInfo->streamInfo.tbName;
}
SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
return &pTaskInfo->streamInfo.metaRsp;
return &pTaskInfo->streamInfo.btMetaRsp;
}
void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {

76
source/libs/executor/src/scanoperator.c
View File

@ -2845,8 +2845,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
SStreamRawScanInfo* pInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta
pTaskInfo->streamInfo.metaRsp.metaRsp = NULL;
pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = NULL; // use batchMetaReq != NULL to judge if data is meta
pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = NULL;
qDebug("tmqsnap doRawScan called");
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
@ -2893,28 +2893,60 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
return NULL;
} else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
SSnapContext* sContext = pInfo->sContext;
void* data = NULL;
int32_t dataLen = 0;
int16_t type = 0;
int64_t uid = 0;
if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
qError("tmqsnap getTableInfoFromSnapshot error");
taosMemoryFreeClear(data);
return NULL;
}
for(int32_t i = 0; i < tmqRowSize; i++) {
void* data = NULL;
int32_t dataLen = 0;
int16_t type = 0;
int64_t uid = 0;
if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
qError("tmqsnap getTableInfoFromSnapshot error");
taosMemoryFreeClear(data);
break;
}
if (!sContext->queryMeta) { // change to get data next poll request
STqOffsetVal offset = {0};
SValue val = {0};
tqOffsetResetToData(&offset, 0, INT64_MIN, val);
qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
} else {
tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
pTaskInfo->streamInfo.metaRsp.resMsgType = type;
pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen;
pTaskInfo->streamInfo.metaRsp.metaRsp = data;
}
if (!sContext->queryMeta) { // change to get data next poll request
STqOffsetVal offset = {0};
SValue val = {0};
tqOffsetResetToData(&offset, 0, INT64_MIN, val);
qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
break;
} else {
tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
SMqMetaRsp tmpMetaRsp = {0};
tmpMetaRsp.resMsgType = type;
tmpMetaRsp.metaRspLen = dataLen;
tmpMetaRsp.metaRsp = data;
if (!pTaskInfo->streamInfo.btMetaRsp.batchMetaReq) {
pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
}
int32_t code = TSDB_CODE_SUCCESS;
uint32_t len = 0;
tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
if (TSDB_CODE_SUCCESS != code) {
qError("tmqsnap tEncodeMqMetaRsp error");
taosMemoryFreeClear(data);
break;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, metaBuff, len);
code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
if (code < 0) {
qError("tmqsnap tEncodeMqMetaRsp error");
tEncoderClear(&encoder);
taosMemoryFreeClear(tBuf);
taosMemoryFreeClear(data);
break;
}
taosMemoryFreeClear(data);
taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaReq, &tBuf);
taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaLen, &tLen);
}
}
return NULL;
}
return NULL;

90
source/libs/parser/inc/parAst.h
View File

@ -118,44 +118,46 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode);
SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode);
SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode);
SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName);
SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral);
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName);
SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral);
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pNode);
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pLeft,
SNode* pRight);
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight,
SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@ -170,7 +172,8 @@ SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit);
SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
@ -205,8 +208,12 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName,
SDataType dataType);
SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName,
SNode* pOptions);
SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
SToken* pColName, SDataType dataType, SNode* pOptions);
SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
SToken* pColName, SNode* pOptions);
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName);
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pOldColName,
SToken* pNewColName);
@ -217,7 +224,8 @@ SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind);
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type);
SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName,
EOperatorType tableCondType);
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType);
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName,
EOperatorType tableCondType);
SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type);
SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);

2
source/libs/parser/inc/sql.y
View File

@ -353,7 +353,7 @@ cmd ::= ALTER STABLE alter_table_clause(A).
alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableModifyOptions(pCxt, B, C); }
alter_table_clause(A) ::=
full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
full_table_name(B) ADD COLUMN column_name(C) type_name(D) column_options(E). { A = createAlterTableAddModifyColOptions2(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D, E); }
alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); }

43
source/libs/parser/src/parAstCreater.c
View File

@ -176,7 +176,8 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb)
static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
trimEscape(pTableName);
if (NULL != pTableName && pTableName->type != TK_NK_NIL && (pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) {
if (NULL != pTableName && pTableName->type != TK_NK_NIL &&
(pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z);
return false;
}
@ -185,7 +186,8 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) {
trimEscape(pColumnName);
if (NULL != pColumnName && pColumnName->type != TK_NK_NIL && (pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) {
if (NULL != pColumnName && pColumnName->type != TK_NK_NIL &&
(pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z);
return false;
}
@ -1893,6 +1895,38 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable,
pStmt->dataType = dataType;
return createAlterTableStmtFinalize(pRealTable, pStmt);
}
SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
SToken* pColName, SDataType dataType, SNode* pOptions) {
CHECK_PARSER_STATUS(pCxt);
if (!checkColumnName(pCxt, pColName)) {
return NULL;
}
SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName);
pStmt->dataType = dataType;
pStmt->pColOptions = (SColumnOptions*)pOptions;
if (pOptions != NULL) {
SColumnOptions* pOption = (SColumnOptions*)pOptions;
if (pOption->bPrimaryKey == false && pOption->commentNull == true) {
if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) {
pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
} else {
// pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
// "not support alter column with option except compress");
// return NULL;
}
} else {
pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
"not support alter column with option except compress");
return NULL;
}
}
return createAlterTableStmtFinalize(pRealTable, pStmt);
}
SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
SToken* pColName, SNode* pOptions) {
@ -2816,9 +2850,10 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI
return (SNode*)pStmt;
}
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
CHECK_PARSER_STATUS(pCxt);
SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
SBalanceVgroupLeaderStmt* pStmt =
(SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);

68
source/libs/parser/src/parTranslater.c
View File

@ -1003,7 +1003,7 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) {
SNodeList* pPartitionList = NULL;
SNode* pNode = NULL;
SNode* pNode = NULL;
if (pOrderList->length <= nodesNum) {
return NULL;
}
@ -1023,7 +1023,6 @@ SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesN
return pPartitionList;
}
static bool isTimeLineAlignedQuery(SNode* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
@ -1036,7 +1035,8 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
return true;
}
if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
SNodeList* pPartitionList =
buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
nodesDestroyList(pPartitionList);
@ -1049,7 +1049,8 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
SNodeList* pPartitionList =
buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
nodesDestroyList(pPartitionList);
@ -6083,7 +6084,7 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
}
}
}
pSelect->timeLineResMode = TIME_LINE_NONE;
}
@ -6237,8 +6238,8 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
}
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
bool isRightPrimTs = isPrimaryKeyImpl(pRight);
bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
bool isRightPrimTs = isPrimaryKeyImpl(pRight);
if (isLeftPrimTs && isRightPrimTs) {
SColumnNode* pFCol = (SColumnNode*)pProj;
@ -6288,9 +6289,9 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
pSetOperator->timeLineFromOrderBy = true;
return code;
}
}
}
}
pSetOperator->timeLineResMode = TIME_LINE_NONE;
}
return code;
@ -8156,9 +8157,9 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
}
switch (pStmt->alterType) {
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_ADD_TAG:
case TSDB_ALTER_TABLE_DROP_TAG:
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_DROP_COLUMN:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: {
@ -8193,6 +8194,31 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
taosArrayPush(pAlterReq->pFields, &field);
break;
}
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
taosArrayDestroy(pAlterReq->pFields);
pAlterReq->pFields = taosArrayInit(1, sizeof(SFieldWithOptions));
SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)};
// TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)};
strcpy(field.name, pStmt->colName);
if (pStmt->pColOptions != NULL) {
if (!checkColumnEncodeOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->encode))
return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnCompressOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compress))
return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
if (!checkColumnLevelOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compressLevel))
return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
int32_t code = setColCompressByOption(pStmt->dataType.type, columnEncodeVal(pStmt->pColOptions->encode),
columnCompressVal(pStmt->pColOptions->compress),
columnLevelVal(pStmt->pColOptions->compressLevel), false,
(uint32_t*)&field.compress);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
taosArrayPush(pAlterReq->pFields, &field);
break;
}
default:
break;
}
@ -10642,7 +10668,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId;
if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
if (pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
int32_t code =
buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
tFreeSBalanceVgroupLeaderReq(&req);
@ -11057,7 +11083,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
}
if (TSDB_CODE_SUCCESS == code) {
pReq->deleteMark = convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision);
pReq->deleteMark =
convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision);
code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen);
}
@ -12857,6 +12884,11 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW);
}
// only super and normal support
if (pStmt->pColOptions != NULL && TSDB_CHILD_TABLE == pTableMeta->tableType) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
}
pReq->colName = taosStrdup(pStmt->colName);
if (NULL == pReq->colName) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -12865,6 +12897,17 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
pReq->type = pStmt->dataType.type;
pReq->flags = COL_SMA_ON;
pReq->bytes = calcTypeBytes(pStmt->dataType);
if (pStmt->pColOptions != NULL) {
if (!checkColumnEncodeOrSetDefault(pReq->type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
if (!checkColumnCompressOrSetDefault(pReq->type, pStmt->pColOptions->compress))
return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
if (!checkColumnLevelOrSetDefault(pReq->type, pStmt->pColOptions->compressLevel))
return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode),
columnCompressVal(pStmt->pColOptions->compress),
columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress);
}
return TSDB_CODE_SUCCESS;
}
@ -13011,6 +13054,7 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
return buildUpdateTagValReq(pCxt, pStmt, pTableMeta, pReq);
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
return buildAddColReq(pCxt, pStmt, pTableMeta, pReq);
case TSDB_ALTER_TABLE_DROP_COLUMN:
return buildDropColReq(pCxt, pStmt, pTableMeta, pReq);

2161
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

33
source/libs/transport/src/transCli.c
View File

@ -200,7 +200,7 @@ static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst);
// process data read from server, add decompress etc later
static void cliHandleResp(SCliConn* conn);
// handle except about conn
static void cliHandleExcept(SCliConn* conn);
static void cliHandleExcept(SCliConn* conn, int32_t code);
static void cliReleaseUnfinishedMsg(SCliConn* conn);
static void cliHandleFastFail(SCliConn* pConn, int status);
@ -571,8 +571,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn);
transUnrefCliHandle(pConn);
}
void cliHandleExcept(SCliConn* conn) {
void cliHandleExcept(SCliConn* conn, int32_t code) {
tTrace("%s conn %p except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn));
if (code != TSDB_CODE_RPC_FQDN_ERROR) {
code = -1;
}
cliHandleExceptImpl(conn, -1);
}
@ -866,7 +869,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
while (transReadComplete(pBuf)) {
tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
if (pBuf->invalid) {
cliHandleExcept(conn);
cliHandleExcept(conn, -1);
break;
} else {
cliHandleResp(conn);
@ -886,7 +889,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
tDebug("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
T_REF_VAL_GET(conn));
conn->broken = true;
cliHandleExcept(conn);
cliHandleExcept(conn, -1);
}
}
@ -1037,7 +1040,7 @@ static void cliSendCb(uv_write_t* req, int status) {
} else {
if (!uv_is_closing((uv_handle_t*)&pConn->stream)) {
tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status));
cliHandleExcept(pConn);
cliHandleExcept(pConn, -1);
}
return;
}
@ -1113,7 +1116,7 @@ void cliSend(SCliConn* pConn) {
if (transQueueEmpty(&pConn->cliMsgs)) {
tError("%s conn %p not msg to send", pTransInst->label, pConn);
cliHandleExcept(pConn);
cliHandleExcept(pConn, -1);
return;
}
@ -1186,7 +1189,7 @@ void cliSend(SCliConn* pConn) {
if (status != 0) {
tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType),
uv_err_name(status));
cliHandleExcept(pConn);
cliHandleExcept(pConn, -1);
}
return;
_RETURN:
@ -1242,7 +1245,8 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) {
taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliHandleFastFail(conn, -1);
cliHandleFastFail(conn, terrno);
terrno = 0;
return;
}
struct sockaddr_in addr;
@ -1304,7 +1308,7 @@ static void cliSendBatchCb(uv_write_t* req, int status) {
tDebug("%s conn %p failed to send batch msg, batch size:%d, msgLen:%d, reason:%s", CONN_GET_INST_LABEL(conn), conn,
p->wLen, p->batchSize, uv_err_name(status));
if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn);
if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn, -1);
cliHandleBatchReq(nxtBatch, thrd);
} else {
@ -1362,7 +1366,7 @@ static void cliHandleFastFail(SCliConn* pConn, int status) {
cliDestroyBatch(pConn->pBatch);
pConn->pBatch = NULL;
}
cliHandleExcept(pConn);
cliHandleExcept(pConn, status);
}
void cliConnCb(uv_connect_t* req, int status) {
@ -1653,7 +1657,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
taosArrayPush(pThrd->timerList, &conn->timer);
conn->timer = NULL;
cliHandleExcept(conn);
cliHandleExcept(conn, terrno);
terrno = 0;
return;
}
@ -1667,20 +1672,20 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
if (fd == -1) {
tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn,
tstrerror(TAOS_SYSTEM_ERROR(errno)));
cliHandleExcept(conn);
cliHandleExcept(conn, -1);
errno = 0;
return;
}
int ret = uv_tcp_open((uv_tcp_t*)conn->stream, fd);
if (ret != 0) {
tGError("%s conn %p failed to set stream, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret));
cliHandleExcept(conn);
cliHandleExcept(conn, -1);
return;
}
ret = transSetConnOption((uv_tcp_t*)conn->stream, tsKeepAliveIdle);
if (ret != 0) {
tGError("%s conn %p failed to set socket opt, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret));
cliHandleExcept(conn);
cliHandleExcept(conn, -1);
return;
}

86
tests/army/community/query/subquery/subqueryBugs.py Normal file
View File

@ -0,0 +1,86 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
from frame.autogen import *
class TDTestCase(TBase):
def ts_30189(self):
tdLog.info("create database ts_30189")
tdSql.execute(f"create database ts_30189")
tdSql.execute(f"use ts_30189")
sqls = [
"CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256))",
"CREATE TABLE demo_201000008 USING demo (deviceid) TAGS ('201000008')",
"CREATE TABLE demo_K201000258 USING demo (deviceid) TAGS ('K201000258')",
"INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-11-30 23:59:27.255', 51412.900999999998021)",
"INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:11:28.179', 51458.900999999998021)",
"INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:12:28.180', 51458.800999999999476)",
"INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-31 23:59:36.108', 52855.400999999998021)",
"INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-11-30 23:59:00.365', 258839.234375000000000)",
"INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:00:00.381', 272188.843750000000000)",
"INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:01:00.600', 13.909012794494629)",
"INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-31 23:59:00.366', 1886.711303710937500)",
]
tdSql.executes(sqls)
sql1 = '''
SELECT ts, deviceid, faev FROM (
(
SELECT deviceid, ts, faev FROM (
SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev
FROM demo
WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00'
PARTITION BY deviceid
) WHERE diff_faev < 0
) UNION ALL
(
SELECT deviceid, ts, faev FROM (
SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev
FROM ( SELECT deviceid, _ts as ts , faev FROM demo
WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00'
ORDER BY ts desc ) PARTITION BY deviceid
) WHERE diff_faev > 0
)
UNION ALL
(
SELECT deviceid, LAST(_ts) AS ts, LAST(faev) AS faev FROM demo
WHERE deviceid in ('201000008') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00'
PARTITION BY deviceid INTERVAL(1n)
)
) order by ts
'''
tdSql.query(sql1)
tdSql.checkRows(4)
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
# TS-30189
self.ts_30189()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

1
tests/parallel_test/cases.task
View File

@ -27,6 +27,7 @@
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
,,n,army,python3 ./test.py -f community/query/show.py -N 3
,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/query/subquery/subqueryBugs.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3

11
tests/script/tsim/compress/compress_col.sim
View File

@ -21,4 +21,15 @@ sql create table txx2 (ts timestamp compress 'disabled' level 'h', f int compres
sql create table txx3 (ts timestamp compress 'disabled' level 'h', f int compress 'xz')
sql create table txx4 (ts timestamp compress 'disabled' level 'h', f int compress 'zstd')
sql alter table txx4 add column tt int compress 'xz' level 'h'
sql alter table txx4 drop column tt
sql create stable sp(ts timestamp, c int) tags(t int)
sql alter table sp add column c1 int compress 'zstd'
sql alter table sp drop column c1
sql alter stable sp add column c1 int compress 'zstd'
sql alter stable sp drop column c1
system sh/exec.sh -n dnode1 -s stop -x SIGINT

12
tests/system-test/2-query/avg.py
View File

@ -436,13 +436,13 @@ class TDTestCase:
tdSql.execute(f"use db")
tdSql.execute(f"create table db.st(ts timestamp, ibv bigint, ubv bigint unsigned) tags(area int)")
# insert t1 data
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,9223372036854775801,18446744073709551611)")
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,8223372036854775801,17446744073709551611)")
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,7223372036854775801,16446744073709551611)")
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 1s,9223372036854775801,18446744073709551611)")
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 2s,8223372036854775801,17446744073709551611)")
tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 3s,7223372036854775801,16446744073709551611)")
# insert t2 data
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,9223372036854775801,18446744073709551611)")
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,8223372036854775801,17446744073709551611)")
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,7223372036854775801,16446744073709551611)")
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 1s,9223372036854775801,18446744073709551611)")
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 2s,8223372036854775801,17446744073709551611)")
tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 3s,7223372036854775801,16446744073709551611)")
# check single table answer
tdSql.query(f"select avg(ibv), avg(ubv) from db.t1")

51
tests/system-test/7-tmq/tmq_taosx.py
View File

@ -278,6 +278,19 @@ class TDTestCase:
self.checkDropData(False)
return
def checkSnapshot1VgroupBtmeta(self):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -bt'%(buildPath, cfgPath)
tdLog.info(cmdStr)
os.system(cmdStr)
self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
self.checkData()
self.checkDropData(False)
return
def checkSnapshot1VgroupTable(self):
buildPath = tdCom.getBuildPath()
@ -290,6 +303,18 @@ class TDTestCase:
self.checkDataTable()
return
def checkSnapshot1VgroupTableBtmeta(self):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -t -bt'%(buildPath, cfgPath)
tdLog.info(cmdStr)
os.system(cmdStr)
self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
self.checkDataTable()
return
def checkSnapshotMultiVgroups(self):
buildPath = tdCom.getBuildPath()
@ -301,6 +326,17 @@ class TDTestCase:
self.checkDropData(False)
return
def checkSnapshotMultiVgroupsBtmeta(self):
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -bt'%(buildPath)
tdLog.info(cmdStr)
os.system(cmdStr)
self.checkData()
self.checkDropData(False)
return
def checkSnapshotMultiVgroupsWithDropTable(self):
buildPath = tdCom.getBuildPath()
@ -311,6 +347,16 @@ class TDTestCase:
self.checkDropData(True)
return
def checkSnapshotMultiVgroupsWithDropTableBtmeta(self):
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -d -bt'%(buildPath)
tdLog.info(cmdStr)
os.system(cmdStr)
self.checkDropData(True)
return
def consumeTest(self):
tdSql.execute(f'create database if not exists d1 vgroups 1')
@ -472,6 +518,11 @@ class TDTestCase:
self.checkSnapshotMultiVgroupsWithDropTable()
self.checkSnapshot1VgroupBtmeta()
self.checkSnapshot1VgroupTableBtmeta()
self.checkSnapshotMultiVgroupsBtmeta()
self.checkSnapshotMultiVgroupsWithDropTableBtmeta()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")

34
utils/test/c/tmq_taosx_ci.c
View File

@ -18,6 +18,7 @@
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "cJSON.h"
#include "taos.h"
#include "tmsg.h"
#include "types.h"
@ -32,6 +33,7 @@ typedef struct {
int srcVgroups;
int dstVgroups;
char dir[256];
bool btMeta;
} Config;
Config g_conf = {0};
@ -61,8 +63,23 @@ static void msg_process(TAOS_RES* msg) {
if (result) {
printf("meta result: %s\n", result);
if (g_fp && strcmp(result, "") != 0) {
taosFprintfFile(g_fp, result);
taosFprintfFile(g_fp, "\n");
// RES_TYPE__TMQ_BATCH_META
if ((*(int8_t*)msg) == 5) {
cJSON* pJson = cJSON_Parse(result);
cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas");
int32_t num = cJSON_GetArraySize(pJsonArray);
for (int32_t i = 0; i < num; i++) {
cJSON* pJsonItem = cJSON_GetArrayItem(pJsonArray, i);
char* itemStr = cJSON_PrintUnformatted(pJsonItem);
taosFprintfFile(g_fp, itemStr);
tmq_free_json_meta(itemStr);
taosFprintfFile(g_fp, "\n");
}
cJSON_Delete(pJson);
} else {
taosFprintfFile(g_fp, result);
taosFprintfFile(g_fp, "\n");
}
}
}
tmq_free_json_meta(result);
@ -584,6 +601,10 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
}
if (g_conf.btMeta) {
tmq_conf_set(conf, "msg.enable.batchmeta", "true");
}
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
assert(tmq);
@ -1064,7 +1085,7 @@ void testConsumeExcluded(int topic_type) {
char* topic = "create topic topic_excluded with meta as database db_taosx";
pRes = taos_query(pConn, topic);
if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
printf("failed to create topic topic_excluded1, reason:%s\n", taos_errstr(pRes));
taos_close(pConn);
return;
}
@ -1073,7 +1094,7 @@ void testConsumeExcluded(int topic_type) {
char* topic = "create topic topic_excluded as select * from stt";
pRes = taos_query(pConn, topic);
if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
printf("failed to create topic topic_excluded2, reason:%s\n", taos_errstr(pRes));
taos_close(pConn);
return;
}
@ -1115,7 +1136,7 @@ void testConsumeExcluded(int topic_type) {
assert(raw.raw_type != 2 && raw.raw_type != 4 && raw.raw_type != TDMT_VND_CREATE_STB &&
raw.raw_type != TDMT_VND_ALTER_STB && raw.raw_type != TDMT_VND_CREATE_TABLE &&
raw.raw_type != TDMT_VND_ALTER_TABLE && raw.raw_type != TDMT_VND_DELETE);
assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE);
assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE || raw.raw_type == 5);
} else if (topic_type == 2) {
assert(0);
}
@ -1138,6 +1159,7 @@ void testConsumeExcluded(int topic_type) {
taos_close(pConn);
return;
}
taos_close(pConn);
taos_free_result(pRes);
}
@ -1167,6 +1189,8 @@ int main(int argc, char* argv[]) {
g_conf.subTable = true;
} else if (strcmp(argv[i], "-onlymeta") == 0) {
g_conf.meta = 1;
} else if (strcmp(argv[i], "-bt") == 0) {
g_conf.btMeta = true;
}
}