xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into 3.0test/jcy

This commit is contained in:
jiacy-jcy 2022-06-23 10:16:38 +08:00
parent 01e670d3db 77f34c8fca
commit fe77051802
77 changed files with 5558 additions and 3947 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/tcommon.h
View File

@ -46,6 +46,7 @@ typedef enum EStreamType {
STREAM_INVALID,
STREAM_GET_ALL,
STREAM_DELETE,
STREAM_RETRIEVE,
} EStreamType;
typedef struct {

2
include/common/tglobal.h
View File

@ -137,6 +137,8 @@ extern bool tsSmlDataFormat;
// internal
extern int32_t tsTransPullupInterval;
extern int32_t tsMqRebalanceInterval;
extern int32_t tsTtlUnit;
extern int32_t tsTtlPushInterval;
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

7
include/common/tmsg.h
View File

@ -1210,6 +1210,13 @@ typedef struct {
char* data;
} STagData;
typedef struct {
int32_t useless; // useless
} SShowVariablesReq;
int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
/*
* sql: show tables like '%a_%'
* payload is the query condition, e.g., '%a_%'

5
include/common/tmsgdef.h
View File

@ -145,6 +145,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TTL_TIMER, "ttl-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "kill-trans", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "kill-query", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "kill-conn", NULL, NULL)
@ -159,6 +160,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_MERGE_VGROUP, "merge-vgroup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_REDISTRIBUTE_VGROUP, "redistribute-vgroup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SPLIT_VGROUP, "split-vgroup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW_VARIABLES, "show-variables", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG)
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@ -200,15 +202,14 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_QND_MSG)
//shared by snode and vnode
TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_SCH_MSG)
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)

410
include/common/ttokendef.h
View File

@ -49,214 +49,216 @@
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_PRIVILEGE 34
#define TK_DROP 35
#define TK_GRANT 36
#define TK_ON 37
#define TK_TO 38
#define TK_REVOKE 39
#define TK_FROM 40
#define TK_NK_COMMA 41
#define TK_READ 42
#define TK_WRITE 43
#define TK_NK_DOT 44
#define TK_DNODE 45
#define TK_PORT 46
#define TK_NK_INTEGER 47
#define TK_DNODES 48
#define TK_NK_IPTOKEN 49
#define TK_LOCAL 50
#define TK_QNODE 51
#define TK_BNODE 52
#define TK_SNODE 53
#define TK_MNODE 54
#define TK_DATABASE 55
#define TK_USE 56
#define TK_IF 57
#define TK_NOT 58
#define TK_EXISTS 59
#define TK_BUFFER 60
#define TK_CACHELAST 61
#define TK_COMP 62
#define TK_DURATION 63
#define TK_NK_VARIABLE 64
#define TK_FSYNC 65
#define TK_MAXROWS 66
#define TK_MINROWS 67
#define TK_KEEP 68
#define TK_PAGES 69
#define TK_PAGESIZE 70
#define TK_PRECISION 71
#define TK_REPLICA 72
#define TK_STRICT 73
#define TK_WAL 74
#define TK_VGROUPS 75
#define TK_SINGLE_STABLE 76
#define TK_RETENTIONS 77
#define TK_SCHEMALESS 78
#define TK_NK_COLON 79
#define TK_TABLE 80
#define TK_NK_LP 81
#define TK_NK_RP 82
#define TK_STABLE 83
#define TK_ADD 84
#define TK_COLUMN 85
#define TK_MODIFY 86
#define TK_RENAME 87
#define TK_TAG 88
#define TK_SET 89
#define TK_NK_EQ 90
#define TK_USING 91
#define TK_TAGS 92
#define TK_COMMENT 93
#define TK_BOOL 94
#define TK_TINYINT 95
#define TK_SMALLINT 96
#define TK_INT 97
#define TK_INTEGER 98
#define TK_BIGINT 99
#define TK_FLOAT 100
#define TK_DOUBLE 101
#define TK_BINARY 102
#define TK_TIMESTAMP 103
#define TK_NCHAR 104
#define TK_UNSIGNED 105
#define TK_JSON 106
#define TK_VARCHAR 107
#define TK_MEDIUMBLOB 108
#define TK_BLOB 109
#define TK_VARBINARY 110
#define TK_DECIMAL 111
#define TK_MAX_DELAY 112
#define TK_WATERMARK 113
#define TK_ROLLUP 114
#define TK_TTL 115
#define TK_SMA 116
#define TK_FIRST 117
#define TK_LAST 118
#define TK_SHOW 119
#define TK_DATABASES 120
#define TK_TABLES 121
#define TK_STABLES 122
#define TK_MNODES 123
#define TK_MODULES 124
#define TK_QNODES 125
#define TK_FUNCTIONS 126
#define TK_INDEXES 127
#define TK_ACCOUNTS 128
#define TK_APPS 129
#define TK_CONNECTIONS 130
#define TK_LICENCE 131
#define TK_GRANTS 132
#define TK_QUERIES 133
#define TK_SCORES 134
#define TK_TOPICS 135
#define TK_VARIABLES 136
#define TK_BNODES 137
#define TK_SNODES 138
#define TK_CLUSTER 139
#define TK_TRANSACTIONS 140
#define TK_DISTRIBUTED 141
#define TK_LIKE 142
#define TK_INDEX 143
#define TK_FULLTEXT 144
#define TK_ENABLE 34
#define TK_NK_INTEGER 35
#define TK_SYSINFO 36
#define TK_DROP 37
#define TK_GRANT 38
#define TK_ON 39
#define TK_TO 40
#define TK_REVOKE 41
#define TK_FROM 42
#define TK_NK_COMMA 43
#define TK_READ 44
#define TK_WRITE 45
#define TK_NK_DOT 46
#define TK_DNODE 47
#define TK_PORT 48
#define TK_DNODES 49
#define TK_NK_IPTOKEN 50
#define TK_LOCAL 51
#define TK_QNODE 52
#define TK_BNODE 53
#define TK_SNODE 54
#define TK_MNODE 55
#define TK_DATABASE 56
#define TK_USE 57
#define TK_IF 58
#define TK_NOT 59
#define TK_EXISTS 60
#define TK_BUFFER 61
#define TK_CACHELAST 62
#define TK_COMP 63
#define TK_DURATION 64
#define TK_NK_VARIABLE 65
#define TK_FSYNC 66
#define TK_MAXROWS 67
#define TK_MINROWS 68
#define TK_KEEP 69
#define TK_PAGES 70
#define TK_PAGESIZE 71
#define TK_PRECISION 72
#define TK_REPLICA 73
#define TK_STRICT 74
#define TK_WAL 75
#define TK_VGROUPS 76
#define TK_SINGLE_STABLE 77
#define TK_RETENTIONS 78
#define TK_SCHEMALESS 79
#define TK_NK_COLON 80
#define TK_TABLE 81
#define TK_NK_LP 82
#define TK_NK_RP 83
#define TK_STABLE 84
#define TK_ADD 85
#define TK_COLUMN 86
#define TK_MODIFY 87
#define TK_RENAME 88
#define TK_TAG 89
#define TK_SET 90
#define TK_NK_EQ 91
#define TK_USING 92
#define TK_TAGS 93
#define TK_COMMENT 94
#define TK_BOOL 95
#define TK_TINYINT 96
#define TK_SMALLINT 97
#define TK_INT 98
#define TK_INTEGER 99
#define TK_BIGINT 100
#define TK_FLOAT 101
#define TK_DOUBLE 102
#define TK_BINARY 103
#define TK_TIMESTAMP 104
#define TK_NCHAR 105
#define TK_UNSIGNED 106
#define TK_JSON 107
#define TK_VARCHAR 108
#define TK_MEDIUMBLOB 109
#define TK_BLOB 110
#define TK_VARBINARY 111
#define TK_DECIMAL 112
#define TK_MAX_DELAY 113
#define TK_WATERMARK 114
#define TK_ROLLUP 115
#define TK_TTL 116
#define TK_SMA 117
#define TK_FIRST 118
#define TK_LAST 119
#define TK_SHOW 120
#define TK_DATABASES 121
#define TK_TABLES 122
#define TK_STABLES 123
#define TK_MNODES 124
#define TK_MODULES 125
#define TK_QNODES 126
#define TK_FUNCTIONS 127
#define TK_INDEXES 128
#define TK_ACCOUNTS 129
#define TK_APPS 130
#define TK_CONNECTIONS 131
#define TK_LICENCE 132
#define TK_GRANTS 133
#define TK_QUERIES 134
#define TK_SCORES 135
#define TK_TOPICS 136
#define TK_VARIABLES 137
#define TK_BNODES 138
#define TK_SNODES 139
#define TK_CLUSTER 140
#define TK_TRANSACTIONS 141
#define TK_DISTRIBUTED 142
#define TK_LIKE 143
#define TK_INDEX 144
#define TK_FUNCTION 145
#define TK_INTERVAL 146
#define TK_TOPIC 147
#define TK_AS 148
#define TK_CONSUMER 149
#define TK_GROUP 150
#define TK_DESC 151
#define TK_DESCRIBE 152
#define TK_RESET 153
#define TK_QUERY 154
#define TK_CACHE 155
#define TK_EXPLAIN 156
#define TK_ANALYZE 157
#define TK_VERBOSE 158
#define TK_NK_BOOL 159
#define TK_RATIO 160
#define TK_NK_FLOAT 161
#define TK_COMPACT 162
#define TK_VNODES 163
#define TK_IN 164
#define TK_OUTPUTTYPE 165
#define TK_AGGREGATE 166
#define TK_BUFSIZE 167
#define TK_STREAM 168
#define TK_INTO 169
#define TK_TRIGGER 170
#define TK_AT_ONCE 171
#define TK_WINDOW_CLOSE 172
#define TK_KILL 173
#define TK_CONNECTION 174
#define TK_TRANSACTION 175
#define TK_BALANCE 176
#define TK_VGROUP 177
#define TK_MERGE 178
#define TK_REDISTRIBUTE 179
#define TK_SPLIT 180
#define TK_SYNCDB 181
#define TK_DELETE 182
#define TK_NULL 183
#define TK_NK_QUESTION 184
#define TK_NK_ARROW 185
#define TK_ROWTS 186
#define TK_TBNAME 187
#define TK_QSTARTTS 188
#define TK_QENDTS 189
#define TK_WSTARTTS 190
#define TK_WENDTS 191
#define TK_WDURATION 192
#define TK_CAST 193
#define TK_NOW 194
#define TK_TODAY 195
#define TK_TIMEZONE 196
#define TK_COUNT 197
#define TK_LAST_ROW 198
#define TK_BETWEEN 199
#define TK_IS 200
#define TK_NK_LT 201
#define TK_NK_GT 202
#define TK_NK_LE 203
#define TK_NK_GE 204
#define TK_NK_NE 205
#define TK_MATCH 206
#define TK_NMATCH 207
#define TK_CONTAINS 208
#define TK_JOIN 209
#define TK_INNER 210
#define TK_SELECT 211
#define TK_DISTINCT 212
#define TK_WHERE 213
#define TK_PARTITION 214
#define TK_BY 215
#define TK_SESSION 216
#define TK_STATE_WINDOW 217
#define TK_SLIDING 218
#define TK_FILL 219
#define TK_VALUE 220
#define TK_NONE 221
#define TK_PREV 222
#define TK_LINEAR 223
#define TK_NEXT 224
#define TK_HAVING 225
#define TK_RANGE 226
#define TK_EVERY 227
#define TK_ORDER 228
#define TK_SLIMIT 229
#define TK_SOFFSET 230
#define TK_LIMIT 231
#define TK_OFFSET 232
#define TK_ASC 233
#define TK_NULLS 234
#define TK_ID 235
#define TK_NK_BITNOT 236
#define TK_INSERT 237
#define TK_VALUES 238
#define TK_IMPORT 239
#define TK_NK_SEMI 240
#define TK_FILE 241
#define TK_WITH 149
#define TK_META 150
#define TK_CONSUMER 151
#define TK_GROUP 152
#define TK_DESC 153
#define TK_DESCRIBE 154
#define TK_RESET 155
#define TK_QUERY 156
#define TK_CACHE 157
#define TK_EXPLAIN 158
#define TK_ANALYZE 159
#define TK_VERBOSE 160
#define TK_NK_BOOL 161
#define TK_RATIO 162
#define TK_NK_FLOAT 163
#define TK_COMPACT 164
#define TK_VNODES 165
#define TK_IN 166
#define TK_OUTPUTTYPE 167
#define TK_AGGREGATE 168
#define TK_BUFSIZE 169
#define TK_STREAM 170
#define TK_INTO 171
#define TK_TRIGGER 172
#define TK_AT_ONCE 173
#define TK_WINDOW_CLOSE 174
#define TK_KILL 175
#define TK_CONNECTION 176
#define TK_TRANSACTION 177
#define TK_BALANCE 178
#define TK_VGROUP 179
#define TK_MERGE 180
#define TK_REDISTRIBUTE 181
#define TK_SPLIT 182
#define TK_SYNCDB 183
#define TK_DELETE 184
#define TK_NULL 185
#define TK_NK_QUESTION 186
#define TK_NK_ARROW 187
#define TK_ROWTS 188
#define TK_TBNAME 189
#define TK_QSTARTTS 190
#define TK_QENDTS 191
#define TK_WSTARTTS 192
#define TK_WENDTS 193
#define TK_WDURATION 194
#define TK_CAST 195
#define TK_NOW 196
#define TK_TODAY 197
#define TK_TIMEZONE 198
#define TK_COUNT 199
#define TK_LAST_ROW 200
#define TK_BETWEEN 201
#define TK_IS 202
#define TK_NK_LT 203
#define TK_NK_GT 204
#define TK_NK_LE 205
#define TK_NK_GE 206
#define TK_NK_NE 207
#define TK_MATCH 208
#define TK_NMATCH 209
#define TK_CONTAINS 210
#define TK_JOIN 211
#define TK_INNER 212
#define TK_SELECT 213
#define TK_DISTINCT 214
#define TK_WHERE 215
#define TK_PARTITION 216
#define TK_BY 217
#define TK_SESSION 218
#define TK_STATE_WINDOW 219
#define TK_SLIDING 220
#define TK_FILL 221
#define TK_VALUE 222
#define TK_NONE 223
#define TK_PREV 224
#define TK_LINEAR 225
#define TK_NEXT 226
#define TK_HAVING 227
#define TK_RANGE 228
#define TK_EVERY 229
#define TK_ORDER 230
#define TK_SLIMIT 231
#define TK_SOFFSET 232
#define TK_LIMIT 233
#define TK_OFFSET 234
#define TK_ASC 235
#define TK_NULLS 236
#define TK_ID 237
#define TK_NK_BITNOT 238
#define TK_INSERT 239
#define TK_VALUES 240
#define TK_IMPORT 241
#define TK_NK_SEMI 242
#define TK_FILE 243
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

7
include/libs/nodes/cmdnodes.h
View File

@ -179,13 +179,16 @@ typedef struct SCreateUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
int8_t sysinfo;
} SCreateUserStmt;
typedef struct SAlterUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
int8_t alterType;
char password[TSDB_USET_PASSWORD_LEN];
int8_t enable;
int8_t sysinfo;
} SAlterUserStmt;
typedef struct SDropUserStmt {
@ -268,7 +271,6 @@ typedef struct SDropIndexStmt {
ENodeType type;
bool ignoreNotExists;
char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
} SDropIndexStmt;
typedef struct SCreateComponentNodeStmt {
@ -287,6 +289,7 @@ typedef struct SCreateTopicStmt {
char subDbName[TSDB_DB_NAME_LEN];
char subSTbName[TSDB_TABLE_NAME_LEN];
bool ignoreExists;
bool withMeta;
SNode* pQuery;
} SCreateTopicStmt;

1
include/libs/nodes/nodes.h
View File

@ -226,6 +226,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
QUERY_NODE_PHYSICAL_PLAN_MERGE,
QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,

2
include/libs/nodes/plannodes.h
View File

@ -420,6 +420,8 @@ typedef struct SSortPhysiNode {
SNodeList* pTargets;
} SSortPhysiNode;
typedef SSortPhysiNode SGroupSortPhysiNode;
typedef struct SPartitionPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of partition_by_clause

1
include/libs/nodes/querynodes.h
View File

@ -385,6 +385,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp);
bool nodesIsComparisonOp(const SOperatorNode* pOp);
bool nodesIsJsonOp(const SOperatorNode* pOp);
bool nodesIsRegularOp(const SOperatorNode* pOp);
bool nodesIsBitwiseOp(const SOperatorNode* pOp);
bool nodesExprHasColumn(SNode* pNode);
bool nodesExprsHasColumn(SNodeList* pList);

58
include/libs/stream/tstream.h
View File

@ -77,7 +77,7 @@ typedef struct {
typedef struct {
int8_t type;
int32_t sourceVg;
int32_t srcVgId;
int64_t sourceVer;
SArray* blocks; // SArray<SSDataBlock*>
@ -145,11 +145,6 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit);
SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit);
#if 0
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
#endif
typedef struct {
char* qmsg;
// followings are not applicable to encoder and decoder
@ -234,26 +229,33 @@ enum {
TASK_TRIGGER_STATUS__ACTIVE,
};
typedef struct {
int32_t nodeId;
int32_t childId;
int32_t taskId;
SEpSet epSet;
} SStreamChildEpInfo;
struct SStreamTask {
int64_t streamId;
int32_t taskId;
int8_t inputType;
int8_t taskStatus;
int8_t execStatus;
int8_t isDataScan;
int8_t execType;
int8_t sinkType;
int8_t dispatchType;
int16_t dispatchMsgType;
int8_t dataScan;
int8_t taskStatus;
int8_t execStatus;
// node info
int32_t childId;
int32_t selfChildId;
int32_t nodeId;
SEpSet epSet;
// children info
SArray* childEpInfo; // SArray<SStreamChildEpInfo*>
// exec
STaskExec exec;
@ -291,6 +293,9 @@ struct SStreamTask {
SMsgCb* pMsgCb;
};
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
SStreamTask* tNewSStreamTask(int64_t streamId);
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
@ -369,9 +374,9 @@ typedef struct {
typedef struct {
int64_t streamId;
int32_t taskId;
int32_t sourceTaskId;
int32_t sourceVg;
int32_t sourceChildId;
int32_t dataSrcVgId;
int32_t upstreamTaskId;
int32_t upstreamChildId;
int32_t upstreamNodeId;
#if 0
int64_t sourceVer;
@ -387,6 +392,23 @@ typedef struct {
int8_t inputStatus;
} SStreamDispatchRsp;
typedef struct {
int64_t streamId;
int32_t srcTaskId;
int32_t srcNodeId;
int32_t dstTaskId;
int32_t dstNodeId;
int32_t retrieveLen;
SRetrieveTableRsp* pRetrieve;
} SStreamRetrieveReq;
typedef struct {
int64_t streamId;
int32_t childId;
int32_t rspFromTaskId;
int32_t rspToTaskId;
} SStreamRetrieveRsp;
typedef struct {
int64_t streamId;
int32_t taskId;
@ -401,6 +423,7 @@ typedef struct {
} SStreamTaskRecoverRsp;
int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);
int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId);
int32_t streamSetupTrigger(SStreamTask* pTask);
@ -411,6 +434,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp);
int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
#ifdef __cplusplus
}
#endif

8
include/libs/wal/wal.h
View File

@ -195,7 +195,6 @@ void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
// only for tq usage
// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
@ -211,13 +210,8 @@ void walCloseRef(SWalRef *);
int32_t walRefVer(SWalRef *, int64_t ver);
int32_t walUnrefVer(SWal *);
// deprecated
#if 0
int32_t walRead(SWal *, SWalHead **, int64_t ver);
int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
#endif
// lifecycle check
bool walIsEmpty(SWal *);
int64_t walGetFirstVer(SWal *);
int64_t walGetSnapshotVer(SWal *);
int64_t walGetLastVer(SWal *);

2
include/os/os.h
View File

@ -104,8 +104,6 @@ extern "C" {
#include "osTimezone.h"
#include "osEnv.h"
void osDefaultInit();
#ifdef __cplusplus
}
#endif

2
include/util/tdef.h
View File

@ -134,7 +134,7 @@ typedef enum EOperatorType {
OP_TYPE_MINUS,
OP_TYPE_ASSIGN,
// bit operator
// bitwise operator
OP_TYPE_BIT_AND,
OP_TYPE_BIT_OR,

9
source/common/src/tglobal.c
View File

@ -187,6 +187,9 @@ bool tsStartUdfd = true;
// internal
int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2;
int32_t tsTtlUnit = 86400;
int32_t tsTtlPushInterval = 60;
void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
@ -467,6 +470,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1;
if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
return 0;
@ -619,6 +624,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32;
tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
@ -631,7 +638,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
osDefaultInit();
if (tsCfg == NULL) osDefaultInit();
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;

25
source/common/src/tmsg.c
View File

@ -2733,6 +2733,31 @@ void tFreeSTableIndexInfo(void *info) {
taosMemoryFree(pInfo->expr);
}
int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);

2
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -101,6 +101,8 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -359,6 +359,8 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

1
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -580,6 +580,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;

55
source/dnode/mnode/impl/src/mndMain.c
View File

@ -65,6 +65,13 @@ static void mndPullupTrans(SMnode *pMnode) {
}
}
static void mndTtlTimer(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
@ -83,41 +90,6 @@ static void mndPullupTelem(SMnode *pMnode) {
}
}
static void mndPushTtlTime(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
int32_t t = taosGetTimestampSec();
*(int32_t *)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("failed to send ttl time seed msg, code:0x%x", code);
} else {
mInfo("send ttl time seed msg, time:%d", t);
}
sdbRelease(pSdb, pVgroup);
}
}
static void *mndThreadFp(void *param) {
SMnode *pMnode = param;
int64_t lastTime = 0;
@ -125,14 +97,13 @@ static void *mndThreadFp(void *param) {
while (1) {
lastTime++;
if (lastTime % (864000) == 0) { // sleep 1 day for ttl
mndPushTtlTime(pMnode);
}
taosMsleep(100);
if (mndGetStop(pMnode)) break;
if (lastTime % (tsTransPullupInterval * 10) == 1) {
mndTtlTimer(pMnode);
}
if (lastTime % (tsTransPullupInterval * 10) == 0) {
mndPullupTrans(pMnode);
}
@ -558,12 +529,12 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER) {
pMsg->msgType == TDMT_MND_TRANS_TIMER || TDMT_MND_TTL_TIMER) {
return -1;
}
const STraceId *trace = &pMsg->info.traceId;
mGError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
SEpSet epSet = {0};
mndGetMnodeEpSet(pMsg->info.node, &epSet);

40
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -37,7 +37,7 @@ extern bool tsSchedStreamToSnode;
static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
int32_t childId = taosArrayGetSize(pArray);
pTask->childId = childId;
pTask->selfChildId = childId;
taosArrayPush(pArray, &pTask);
return 0;
}
@ -271,7 +271,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
// source
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
pTask->isDataScan = 0;
// exec
pTask->execType = TASK_EXEC__NONE;
@ -306,6 +306,8 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
}
mndAddTaskToTaskSet(tasks, pTask);
ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId);
pTask->nodeId = pStream->fixedSinkVgId;
#if 0
SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
@ -315,8 +317,9 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
#endif
pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg);
// source
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
pTask->isDataScan = 0;
// exec
pTask->execType = TASK_EXEC__NONE;
@ -384,8 +387,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
pInnerTask = tNewSStreamTask(pStream->uid);
mndAddTaskToTaskSet(taskInnerLevel, pInnerTask);
// input
pInnerTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*));
// source
pInnerTask->isDataScan = 0;
// trigger
pInnerTask->triggerParam = pStream->triggerParam;
@ -446,10 +452,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
mndAddTaskToTaskSet(taskSourceLevel, pTask);
pTask->dataScan = 1;
// input
pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
// source
pTask->isDataScan = 1;
// add fixed vg dispatch
pTask->sinkType = TASK_SINK__NONE;
@ -467,6 +471,20 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
qDestroyQueryPlan(pPlan);
return -1;
}
SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo));
if (pEpInfo == NULL) {
ASSERT(0);
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan);
return -1;
}
pEpInfo->childId = pTask->selfChildId;
pEpInfo->epSet = pTask->epSet;
pEpInfo->nodeId = pTask->nodeId;
pEpInfo->taskId = pTask->taskId;
taosArrayPush(pInnerTask->childEpInfo, &pEpInfo);
}
}
@ -491,10 +509,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
mndAddTaskToTaskSet(taskOneLevel, pTask);
pTask->dataScan = 1;
// input
pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
pTask->isDataScan = 1;
// trigger
pTask->triggerParam = pStream->triggerParam;

39
source/dnode/mnode/impl/src/mndStb.c
View File

@ -37,6 +37,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb);
static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
static int32_t mndProcessTtlTimer(SRpcMsg *pReq);
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq);
static int32_t mndProcessAlterStbReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReq(SRpcMsg *pReq);
@ -63,6 +64,7 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb);
@ -799,6 +801,43 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
return 0;
}
static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
int32_t t = taosGetTimestampSec();
*(int32_t *)((char *)pHead + sizeof(SMsgHead)) = htonl(t);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("failed to send ttl time seed, code:0x%x", code);
} else {
mDebug("send ttl time seed success, time:%d", t);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;

1
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -56,6 +56,7 @@ static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->dep
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransReq(SRpcMsg *pReq);
static int32_t mndProcessTtl(SRpcMsg *pReq);
static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);

35
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -1553,10 +1553,11 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
void *pIter = NULL;
int32_t code = -1;
SSdb *pSdb = pMnode->pSdb;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
bool existInSrc = false;
@ -1568,13 +1569,15 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
}
if (!existInSrc || existInDst) {
sdbRelease(pMnode->pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
mndReleaseDb(pMnode, pDb);
sdbRelease(pMnode->pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
break;
}
@ -1593,15 +1596,25 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
while (1) {
taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
SDnodeObj *pSrc = taosArrayGet(pArray, 0);
SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SDnodeObj *pDnode = taosArrayGet(pArray, i);
mDebug("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
}
SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
SDnodeObj *pDst = taosArrayGet(pArray, 0);
float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
if (srcScore + 0.0001 < dstScore) {
mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
mDebug("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore,
pDst->id, dstScore);
if (srcScore > dstScore - 0.000001) {
code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
if (code == 0) {
pSrc->numOfVnodes--;
pDst->numOfVnodes++;
numOfVgroups++;
continue;
} else {
@ -1635,7 +1648,13 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
void *pIter = NULL;
int64_t curMs = taosGetTimestampMs();
mDebug("start to balance vgroup");
SBalanceVgroupReq req = {0};
if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("start to balance vgroup");
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER;

37
source/dnode/snode/src/snode.c
View File

@ -105,13 +105,14 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
ASSERT(pTask->execType != TASK_EXEC__NONE);
ASSERT(pTask->dataScan == 0);
ASSERT(pTask->isDataScan == 0);
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
ASSERT(pTask->exec.executor);
streamSetupTrigger(pTask);
qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId, pTask->childId);
qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId,
pTask->selfChildId);
taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));
@ -198,6 +199,34 @@ static int32_t sndProcessTaskDropReq(SSnode *pNode, SRpcMsg *pMsg) {
return code;
}
static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
SStreamMeta *pMeta = pNode->pMeta;
char *msgStr = pMsg->pCont;
char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
SStreamRetrieveReq req;
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req);
int32_t taskId = req.dstTaskId;
SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessRetrieveReq(pTask, &req, &rsp);
return 0;
}
static int32_t sndProcessTaskRetrieveRsp(SSnode *pNode, SRpcMsg *pMsg) {
//
return 0;
}
int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
// stream deploy
// stream stop/resume
@ -221,10 +250,14 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
return sndProcessTaskDispatchReq(pSnode, pMsg);
case TDMT_STREAM_TASK_RECOVER:
return sndProcessTaskRecoverReq(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE:
return sndProcessTaskRecoverReq(pSnode, pMsg);
case TDMT_STREAM_TASK_DISPATCH_RSP:
return sndProcessTaskDispatchRsp(pSnode, pMsg);
case TDMT_STREAM_TASK_RECOVER_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
case TDMT_STREAM_RETRIEVE_RSP:
return sndProcessTaskRecoverRsp(pSnode, pMsg);
default:
ASSERT(0);
}

1
source/dnode/vnode/inc/vnode.h
View File

@ -195,6 +195,7 @@ struct SVnodeCfg {
typedef struct {
TSKEY lastKey;
uint64_t uid;
uint64_t groupId;
} STableKeyInfo;
struct SMetaEntry {

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -149,6 +149,8 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId);

3
source/dnode/vnode/src/meta/metaTable.c
View File

@ -400,8 +400,7 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
if (ttlDays <= 0) return;
ttlKey->dtime = ctime / 1000 + ttlDays * 24 * 60 * 60;
// ttlKey->dtime = ctime / 1000 + ttlDays;
ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit;
ttlKey->uid = pME->uid;
}

33
source/dnode/vnode/src/tq/tq.c
View File

@ -426,6 +426,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
ASSERT(0);
}
tDecoderClear(&decoder);
ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1);
pTask->execStatus = TASK_EXEC_STATUS__IDLE;
@ -441,7 +442,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
// exec
if (pTask->execType != TASK_EXEC__NONE) {
// expand runners
if (pTask->dataScan) {
if (pTask->isDataScan) {
STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pStreamReader,
@ -476,7 +477,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
streamSetupTrigger(pTask);
tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->childId, TD_VID(pTq->pVnode));
tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));
taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));
@ -505,7 +506,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
continue;
}
if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
if (!pTask->isDataScan) continue;
if (!failed) {
if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) {
@ -616,3 +617,29 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
return code;
#endif
}
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
SStreamRetrieveReq req;
SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req);
int32_t taskId = req.dstTaskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
};
streamProcessRetrieveReq(pTask, &req, &rsp);
return 0;
}
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
//
return 0;
}

4
source/dnode/vnode/src/tq/tqRead.c
View File

@ -345,8 +345,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
while (1) {
pIter = taosHashIterate(pTq->pStreamTasks, pIter);
if (pIter == NULL) break;
SStreamTask* pTask = (SStreamTask*)pIter;
if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->isDataScan) {
int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
ASSERT(code == 0);
}

2
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2845,7 +2845,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
break;
}
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0};
taosArrayPush(list, &info);
}

6
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -265,10 +265,14 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return tqProcessTaskDispatchReq(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_RECOVER:
return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);
case TDMT_STREAM_RETRIEVE:
return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_DISPATCH_RSP:
return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_RECOVER_RSP:
return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);
case TDMT_STREAM_RETRIEVE_RSP:
return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in fetch queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
@ -311,7 +315,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
int32_t t = ntohl(*(int32_t *)pReq);
vError("rec ttl time:%d", t);
vDebug("vgId:%d, recv ttl msg, time:%d", pVnode->config.vgId, t);
int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
if (ret != 0) {
goto end;

14
source/libs/executor/inc/executorimpl.h
View File

@ -838,14 +838,18 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
uint64_t taskId, SNode* pTagCond);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
SNode* pTagCond);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey);
#ifdef __cplusplus
}
#endif

4
source/libs/executor/src/executil.c
View File

@ -239,7 +239,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
@ -247,7 +247,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
}
} else { // Create one table group.
STableKeyInfo info = {.lastKey = 0, .uid = tableUid};
STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}

7
source/libs/executor/src/executor.c
View File

@ -40,7 +40,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
SStreamBlockScanInfo* pInfo = pOperator->info;
pInfo->assignBlockUid = assignUid;
// the block type can not be changed in the streamscan operators
// no need to check
#if 0
if (pInfo->blockType == 0) {
pInfo->blockType = type;
@ -49,10 +49,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
return TSDB_CODE_QRY_APP_ERROR;
}
#endif
// rollup sma, the same qTaskInfo is used to insert data by SubmitReq and fetch result by SSDataBlock
if (pInfo->blockType != type) {
pInfo->blockType = type;
}
pInfo->blockType = type;
if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {

25
source/libs/executor/src/executorimpl.c
View File

@ -3968,14 +3968,16 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
}
}
}
int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len);
if (groupId) {
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t));
} else {
uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len);
if (!pGroupId) {
uint64_t tmpId = calcGroupId(keyBuf, len);
info->groupId = tmpId;
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t));
} else {
info->groupId = *pGroupId;
}
metaReaderClear(&mr);
@ -4021,14 +4023,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
SArray* dataReaders = taosArrayInit(8, POINTER_BYTES);
createMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId, pTagCond);
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond);
extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
taosArrayDestroy(groupKeys);
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, dataReaders, pHandle, pTaskInfo);
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
@ -4097,7 +4094,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
} else { // Create one table group.
STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid};
STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0};
taosArrayPush(pTableListInfo->pTableList, &info);
}
@ -4228,6 +4225,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
} else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) {
pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) {
pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;

250
source/libs/executor/src/scanoperator.c
View File

@ -537,7 +537,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
// taosSsleep(20);
SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
int32_t numOfCols = 0;
int32_t numOfCols = 0;
SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
@ -597,12 +597,12 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
pInfo->dataReader = pReadHandle;
// pInfo->prevGroupId = -1;
pOperator->name = "TableSeqScanOperator";
pOperator->name = "TableSeqScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL);
return pOperator;
@ -616,7 +616,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) {
metaGetTableEntryByUid(&mr, uid);
if (mr.me.type == TSDB_SUPER_TABLE) {
int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols;
for(int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes;
}
} else if (mr.me.type == TSDB_CHILD_TABLE) {
@ -624,12 +624,12 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) {
metaGetTableEntryByUid(&mr, suid);
int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols;
for(int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes;
}
} else if (mr.me.type == TSDB_NORMAL_TABLE) {
int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols;
for(int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
rowLen += mr.me.ntbEntry.schemaRow.pSchema[i].bytes;
}
}
@ -653,7 +653,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo);
@ -685,23 +685,23 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re
goto _error;
}
pInfo->pHandle = dataReader;
pInfo->pHandle = dataReader;
pInfo->readHandle = *readHandle;
pInfo->uid = uid;
pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
pInfo->uid = uid;
pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
int32_t numOfCols = 0;
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
pOperator->name = "DataBlockDistScanOperator";
pOperator->name = "DataBlockDistScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL,
@ -916,6 +916,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
}
size_t total = taosArrayGetSize(pInfo->pBlockLists);
// TODO: refactor
if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
if (pInfo->validBlockIndex >= total) {
/*doClearBufferedBlocks(pInfo);*/
@ -1878,26 +1879,25 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
SArray* colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
pInfo->pTableList = pTableListInfo;
pInfo->pColMatchInfo = colList;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
pInfo->pFilterNode = pPhyNode->node.pConditions;
pInfo->pTableList = pTableListInfo;
pInfo->pColMatchInfo = colList;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
pInfo->pFilterNode = pPhyNode->node.pConditions;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
initResultSizeInfo(pOperator, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
@ -1915,6 +1915,12 @@ _error:
}
typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
int32_t tableStartIndex;
int32_t tableEndIndex;
bool hasGroupId;
uint64_t groupId;
SArray* dataReaders; // array of tsdbReaderT*
SReadHandle readHandle;
@ -1927,11 +1933,9 @@ typedef struct STableMergeScanInfo {
SSDataBlock* pSortInputBlock;
int64_t startTs; // sort start time
bool hasGroupId;
uint64_t groupId;
STupleHandle* prefetchedTuple;
SArray* sortSourceParams;
SArray* sortSourceParams;
uint64_t queryId;
uint64_t taskId;
SFileBlockLoadRecorder readRecorder;
int64_t numOfRows;
@ -1959,29 +1963,42 @@ typedef struct STableMergeScanInfo {
// window to check if current data block needs to be loaded.
SSampleExecInfo sample; // sample execution info
int32_t curTWinIdx;
} STableMergeScanInfo;
int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
uint64_t taskId, SNode* pTagCond) {
int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
const STableKeyInfo* info1 = p1;
const STableKeyInfo* info2 = p2;
return info1->groupId - info2->groupId;
}
int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) {
int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
return code;
}
if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
goto _error;
return TSDB_CODE_SUCCESS;
}
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
if (groupKeys) {
taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
}
taosArrayDestroy(groupKeys);
return TSDB_CODE_SUCCESS;
}
int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
SQueryTableDataCond cond = {0};
code = initQueryTableDataCond(&cond, pTableScanNode);
int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
// TODO: free the sublist info and the table list in it
for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) {
STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
@ -1995,12 +2012,30 @@ int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHand
}
cleanupQueryTableDataCond(&cond);
return 0;
return TSDB_CODE_SUCCESS;
_error:
return code;
}
int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId);
taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(subListInfo->pTableList);
taosMemoryFree(subListInfo);
}
return TSDB_CODE_SUCCESS;
}
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -2180,22 +2215,47 @@ SArray* generateSortByTsInfo(int32_t order) {
return pList;
}
int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) {
int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (OPTR_IS_OPENED(pOperator)) {
return TSDB_CODE_SUCCESS;
{
size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
int32_t i = pInfo->tableStartIndex + 1;
for (; i < tableListSize; ++i) {
STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
if (tableKeyInfo->groupId != pInfo->groupId) {
break;
}
}
pInfo->tableEndIndex = i - 1;
}
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
int32_t tableStartIdx = pInfo->tableStartIndex;
int32_t tableEndIdx = pInfo->tableEndIndex;
STableListInfo* tableListInfo = pInfo->tableListInfo;
createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL);
size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
for (int32_t i = 0; i < numReaders; ++i) {
STableMergeScanSortSourceParam param = {0};
param.readerIdx = i;
param.pOperator = pOperator;
param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
taosArrayPush(pInfo->sortSourceParams, &param);
}
for (int32_t i = 0; i < numReaders; ++i) {
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
@ -2209,9 +2269,22 @@ int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, terrno);
}
pOperator->status = OP_RES_TO_RETURN;
return TSDB_CODE_SUCCESS;
}
int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
tsortDestroySortHandle(pInfo->pSortHandle);
taosArrayClear(pInfo->sortSourceParams);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->dataReaders); ++i) {
tsdbReaderT* reader = taosArrayGetP(pInfo->dataReaders, i);
tsdbCleanupReadHandle(reader);
}
taosArrayDestroy(pInfo->dataReaders);
OPTR_SET_OPENED(pOperator);
return TSDB_CODE_SUCCESS;
}
@ -2254,14 +2327,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows;
} else {
doSetOperatorCompleted(pOperator);
if (tableListSize == 0) {
doSetOperatorCompleted(pOperator);
return NULL;
}
pInfo->tableStartIndex = 0;
pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
SSDataBlock* pBlock = NULL;
while (pInfo->tableStartIndex < tableListSize) {
pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
if (pBlock != NULL) {
pBlock->info.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
} else {
stopGroupTableMergeScan(pOperator);
if (pInfo->tableEndIndex >= tableListSize - 1) {
doSetOperatorCompleted(pOperator);
break;
}
pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
pInfo->groupId =
((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
}
return pBlock;
}
@ -2269,17 +2366,10 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) {
STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
cleanupQueryTableDataCond(&pTableScanInfo->cond);
for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) {
tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, i);
tsdbCleanupReadHandle(reader);
}
taosArrayDestroy(pTableScanInfo->dataReaders);
if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo);
}
taosArrayDestroy(pTableScanInfo->sortSourceParams);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
@ -2305,8 +2395,9 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
uint64_t taskId) {
STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -2336,22 +2427,16 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->sample.seed = taosGetTimestampSec();
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
pInfo->dataReaders = dataReaders;
pInfo->tableListInfo = pTableListInfo;
pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColList;
pInfo->curTWinIdx = 0;
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
pInfo->queryId = queryId;
pInfo->taskId = taskId;
pInfo->sortSourceParams = taosArrayInit(taosArrayGetSize(dataReaders), sizeof(STableMergeScanSortSourceParam));
for (int32_t i = 0; i < taosArrayGetSize(dataReaders); ++i) {
STableMergeScanSortSourceParam* param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
param->readerIdx = i;
param->pOperator = pOperator;
param->inputBlock = createOneDataBlock(pInfo->pResBlock, false);
taosArrayPush(pInfo->sortSourceParams, param);
taosMemoryFree(param);
}
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
pInfo->pSortInfo = generateSortByTsInfo(pInfo->cond.order);
pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
@ -2359,14 +2444,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t rowSize = pInfo->pResBlock->info.rowSize;
pInfo->bufPageSize = getProperSortPageSize(rowSize);
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
pInfo->sortBufSize = pInfo->bufPageSize * (taosArrayGetSize(dataReaders) + 1);
pInfo->hasGroupId = false;
pInfo->prefetchedTuple = NULL;
pOperator->name = "TableMergeScanOperator";
// TODO : change it
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
@ -2376,8 +2454,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
initResultSizeInfo(pOperator, 1024);
pOperator->fpSet =
createOperatorFpSet(doOpenTableMergeScanOperator, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo,
NULL, NULL, getTableMergeScanExplainExecInfo);
createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo, NULL,
NULL, getTableMergeScanExplainExecInfo);
pOperator->cost.openCost = 0;
return pOperator;

30
source/libs/executor/src/sortoperator.c
View File

@ -424,10 +424,17 @@ int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
return TSDB_CODE_SUCCESS;
}
// TODO:
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode,
void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->pColMatchInfo);
}
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo) {
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) {
goto _error;
@ -452,8 +459,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysi
;
pInfo->pColMatchInfo = pColMatchColInfo;
pOperator->name = "GroupSortOperator";
// TODO
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
@ -461,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysi
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyOrderOperatorInfo, NULL,
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL,
NULL, getGroupSortExplainExecInfo);
int32_t code = appendDownstream(pOperator, &downstream, 1);
@ -478,18 +484,6 @@ _error:
return NULL;
}
void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->pColMatchInfo);
}
// TODO: sort group
// TODO: msortCompare compare group id in multiway merge sort.
// TODO: table merge scan, group first, then for each group, multiple readers
//=====================================================================================
// Multiway Sort Merge operator
typedef struct SMultiwaySortMergeOperatorInfo {

3
source/libs/executor/src/timewindowoperator.c
View File

@ -1656,10 +1656,9 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
return NULL;
}
return pBInfo->pRes;
return pBInfo->pRes->info.rows > 0 ? pBInfo->pRes : NULL;
}
int64_t st = taosGetTimestampUs();

11
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1357,6 +1357,17 @@ bool nodesIsRegularOp(const SOperatorNode* pOp) {
return false;
}
bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_BIT_AND:
case OP_TYPE_BIT_OR:
return true;
default:
break;
}
return false;
}
typedef struct SCollectColumnsCxt {
int32_t errCode;
const char* pTableAlias;

13
source/libs/parser/inc/parAst.h
View File

@ -162,7 +162,7 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo);
SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal);
SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
@ -171,12 +171,15 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName,
SToken* pTableName, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName);
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName);
SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createTopicOptions(SAstCreateContext* pCxt);
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery,
const SToken* pSubDbName, SNode* pRealTable);
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery);
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
const SToken* pSubDbName, bool withMeta);
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
const SToken* pTopicName);

37
source/libs/parser/inc/sql.y
View File

@ -80,11 +80,18 @@ alter_account_option ::= CONNS literal.
alter_account_option ::= STATE literal. { }
/************************************************ create/alter/drop user **********************************************/
cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
cmd ::= CREATE USER user_name(A) PASS NK_STRING(B) sysinfo_opt(C). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B, C); }
cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); }
//cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); }
cmd ::= ALTER USER user_name(A) ENABLE NK_INTEGER(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_ENABLE, &B); }
cmd ::= ALTER USER user_name(A) SYSINFO NK_INTEGER(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_SYSINFO, &B); }
cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
%type sysinfo_opt { int8_t }
%destructor sysinfo_opt { }
sysinfo_opt(A) ::= . { A = 1; }
sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B). { A = taosStr2Int8(B.z, NULL, 10); }
/************************************************ grant/revoke ********************************************************/
cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); }
cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); }
@ -396,9 +403,9 @@ from_db_opt(A) ::= FROM db_name(B).
/************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX not_exists_opt(D)
index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); }
cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D)
index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); }
cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); }
//cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D)
// index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); }
cmd ::= DROP INDEX exists_opt(B) index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, &A); }
index_options(A) ::= . { A = NULL; }
index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL
@ -414,10 +421,14 @@ func_list(A) ::= func_list(B) NK_COMMA func(C).
func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); }
/************************************************ create/drop topic ***************************************************/
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL, NULL); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C, NULL); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B)
AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, NULL, C); }
WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B)
AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); }
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B)
WITH META AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); }
cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); }
cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); }
@ -627,6 +638,16 @@ expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C).
SToken s = getTokenFromRawExprNode(pCxt, B);
A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, B), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C)));
}
expression(A) ::= expression(B) NK_BITAND expression(C). {
SToken s = getTokenFromRawExprNode(pCxt, B);
SToken e = getTokenFromRawExprNode(pCxt, C);
A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
}
expression(A) ::= expression(B) NK_BITOR expression(C). {
SToken s = getTokenFromRawExprNode(pCxt, B);
SToken e = getTokenFromRawExprNode(pCxt, C);
A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
}
%type expression_list { SNodeList* }
%destructor expression_list { nodesDestroyList($$); }

75
source/libs/parser/src/parAstCreater.c
View File

@ -1205,7 +1205,7 @@ SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId) {
return (SNode*)pStmt;
}
SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) {
SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo) {
CHECK_PARSER_STATUS(pCxt);
char password[TSDB_USET_PASSWORD_LEN] = {0};
if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) {
@ -1215,6 +1215,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName);
strcpy(pStmt->password, password);
pStmt->sysinfo = sysinfo;
return (SNode*)pStmt;
}
@ -1226,15 +1227,26 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al
SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName);
if (TSDB_ALTER_USER_PASSWD == alterType) {
char password[TSDB_USET_PASSWORD_LEN] = {0};
if (!checkPassword(pCxt, pVal, password)) {
nodesDestroyNode((SNode*)pStmt);
return NULL;
}
strcpy(pStmt->password, password);
}
pStmt->alterType = alterType;
switch (alterType) {
case TSDB_ALTER_USER_PASSWD: {
char password[TSDB_USET_PASSWORD_LEN] = {0};
if (!checkPassword(pCxt, pVal, password)) {
nodesDestroyNode((SNode*)pStmt);
return NULL;
}
strcpy(pStmt->password, password);
break;
}
case TSDB_ALTER_USER_ENABLE:
pStmt->enable = taosStr2Int8(pVal->z, NULL, 10);
break;
case TSDB_ALTER_USER_SYSINFO:
pStmt->sysinfo = taosStr2Int8(pVal->z, NULL, 10);
break;
default:
break;
}
return (SNode*)pStmt;
}
@ -1317,16 +1329,15 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
return (SNode*)pOptions;
}
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) {
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
if (!checkDbName(pCxt, NULL, true) || !checkIndexName(pCxt, pIndexName)) {
return NULL;
}
SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
COPY_STRING_FORM_ID_TOKEN(pStmt->indexName, pIndexName);
COPY_STRING_FORM_ID_TOKEN(pStmt->tableName, pTableName);
return (SNode*)pStmt;
}
@ -1346,22 +1357,40 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons
return (SNode*)pStmt;
}
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery,
const SToken* pSubDbName, SNode* pRealTable) {
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
pStmt->ignoreExists = ignoreExists;
if (NULL != pRealTable) {
strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName);
strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName);
nodesDestroyNode(pRealTable);
} else if (NULL != pSubDbName) {
COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName);
} else {
pStmt->pQuery = pQuery;
}
pStmt->pQuery = pQuery;
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
const SToken* pSubDbName, bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
pStmt->ignoreExists = ignoreExists;
COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName);
pStmt->withMeta = withMeta;
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
pStmt->ignoreExists = ignoreExists;
pStmt->withMeta = withMeta;
strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName);
strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName);
nodesDestroyNode(pRealTable);
return (SNode*)pStmt;
}

6
source/libs/parser/src/parTokenizer.c
View File

@ -79,6 +79,7 @@ static SKeyword keywordTable[] = {
{"DOUBLE", TK_DOUBLE},
{"DROP", TK_DROP},
{"DURATION", TK_DURATION},
{"ENABLE", TK_ENABLE},
{"EXISTS", TK_EXISTS},
{"EXPLAIN", TK_EXPLAIN},
{"EVERY", TK_EVERY},
@ -120,6 +121,7 @@ static SKeyword keywordTable[] = {
{"MAXROWS", TK_MAXROWS},
{"MAX_DELAY", TK_MAX_DELAY},
{"MERGE", TK_MERGE},
{"META", TK_META},
{"MINROWS", TK_MINROWS},
{"MINUS", TK_MINUS},
{"MNODE", TK_MNODE},
@ -146,7 +148,7 @@ static SKeyword keywordTable[] = {
{"PORT", TK_PORT},
{"PPS", TK_PPS},
{"PRECISION", TK_PRECISION},
{"PRIVILEGE", TK_PRIVILEGE},
// {"PRIVILEGE", TK_PRIVILEGE},
{"PREV", TK_PREV},
{"QNODE", TK_QNODE},
{"QNODES", TK_QNODES},
@ -187,6 +189,7 @@ static SKeyword keywordTable[] = {
{"STREAMS", TK_STREAMS},
{"STRICT", TK_STRICT},
{"SYNCDB", TK_SYNCDB},
{"SYSINFO", TK_SYSINFO},
{"TABLE", TK_TABLE},
{"TABLES", TK_TABLES},
{"TAG", TK_TAG},
@ -222,6 +225,7 @@ static SKeyword keywordTable[] = {
{"WATERMARK", TK_WATERMARK},
{"WHERE", TK_WHERE},
{"WINDOW_CLOSE", TK_WINDOW_CLOSE},
{"WITH", TK_WITH},
{"WRITE", TK_WRITE},
{"_C0", TK_ROWTS},
{"_QENDTS", TK_QENDTS},

68
source/libs/parser/src/parTranslater.c
View File

@ -1012,6 +1012,17 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO
return DEAL_RES_CONTINUE;
}
static EDealRes translateBitwiseOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
return DEAL_RES_CONTINUE;
}
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRef) {
SOperatorNode* pOp = *pOpRef;
@ -1030,6 +1041,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRe
return translateComparisonOperator(pCxt, pOp);
} else if (nodesIsJsonOp(pOp)) {
return translateJsonOperator(pCxt, pOp);
} else if (nodesIsBitwiseOp(pOp)) {
return translateBitwiseOperator(pCxt, pOp);
}
return DEAL_RES_CONTINUE;
}
@ -2148,9 +2161,6 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) {
if (COLUMN_TYPE_TAG == pCol->colType) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL);
}
if (TSDB_SUPER_TABLE == pCol->tableType) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE);
}
}
return DEAL_RES_CONTINUE;
}
@ -2381,6 +2391,8 @@ static SNode* createGroupingSet(SNode* pExpr) {
return (SNode*)pGroupingSet;
}
// from: select unique(expr), col1 + col2 from t where_clause partition_by_clause order_by_clause ...
// to: select expr, first(col1) + first(col2) from t where_clause partition_by_clause group by expr order_by_clause ...
static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasUniqueFunc) {
return TSDB_CODE_SUCCESS;
@ -2446,6 +2458,8 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) {
return (SNode*)pOrder;
}
// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ...
// to: select expr from t where_clause order by _rowts desc limit k offset f
static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasTailFunc) {
return TSDB_CODE_SUCCESS;
@ -3740,7 +3754,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt
strcpy(createReq.user, pStmt->useName);
createReq.createType = 0;
createReq.superUser = 0;
createReq.sysInfo = 1;
createReq.sysInfo = pStmt->sysinfo;
createReq.enable = 1;
strcpy(createReq.pass, pStmt->password);
@ -3752,6 +3766,8 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
strcpy(alterReq.user, pStmt->useName);
alterReq.alterType = pStmt->alterType;
alterReq.superUser = 0;
alterReq.enable = pStmt->enable;
alterReq.sysInfo = pStmt->sysinfo;
strcpy(alterReq.pass, pStmt->password);
if (NULL != pCxt->pParseCxt->db) {
strcpy(alterReq.dbname, pCxt->pParseCxt->db);
@ -3906,36 +3922,11 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
}
static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
SEncoder encoder = {0};
int32_t contLen = 0;
SVDropTSmaReq dropSmaReq = {0};
strcpy(dropSmaReq.indexName, pStmt->indexName);
pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t ret = 0;
tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
if (ret < 0) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
pCxt->pCmdMsg->msgLen = contLen;
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
void* pBuf = pCxt->pCmdMsg->pMsg;
if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) {
tEncoderClear(&encoder);
return TSDB_CODE_OUT_OF_MEMORY;
}
tEncoderClear(&encoder);
return TSDB_CODE_SUCCESS;
SMDropSmaReq dropSmaReq = {0};
SName name;
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), dropSmaReq.name);
dropSmaReq.igNotExists = pStmt->ignoreNotExists;
return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq);
}
static int16_t getCreateComponentNodeMsgType(ENodeType type) {
@ -3987,6 +3978,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
tNameGetFullDbName(&name, pReq->name);
pReq->igExists = pStmt->ignoreExists;
pReq->withMeta = pStmt->withMeta;
pReq->sql = strdup(pCxt->pParseCxt->pSql);
if (NULL == pReq->sql) {
@ -4342,6 +4334,11 @@ static int32_t translateSplitVgroup(STranslateContext* pCxt, SSplitVgroupStmt* p
return buildCmdMsg(pCxt, TDMT_MND_SPLIT_VGROUP, (FSerializeFunc)tSerializeSSplitVgroupReq, &req);
}
static int32_t translateShowVariables(STranslateContext* pCxt, SShowStmt* pStmt) {
SShowVariablesReq req = {0};
return buildCmdMsg(pCxt, TDMT_MND_SHOW_VARIABLES, (FSerializeFunc)tSerializeSShowVariablesReq, &req);
}
static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateDatabaseStmt* pStmt) {
pStmt->pCfg = taosMemoryCalloc(1, sizeof(SDbCfgInfo));
if (NULL == pStmt->pCfg) {
@ -4486,6 +4483,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_SPLIT_VGROUP_STMT:
code = translateSplitVgroup(pCxt, (SSplitVgroupStmt*)pNode);
break;
case QUERY_NODE_SHOW_VARIABLES_STMT:
code = translateShowVariables(pCxt, (SShowStmt*)pNode);
break;
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
code = translateShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pNode);
break;

6039
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

51
source/libs/parser/test/parInitialATest.cpp
View File

@ -121,7 +121,7 @@ TEST_F(ParserInitialATest, alterSTable) {
int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname);
expect.name[len] = '\0';
expect.alterType = alterType;
// expect.ttl = ttl;
// expect.ttl = ttl;
if (nullptr != pComment) {
expect.comment = strdup(pComment);
expect.commentLen = strlen(pComment);
@ -180,9 +180,9 @@ TEST_F(ParserInitialATest, alterSTable) {
tFreeSMAltertbReq(&req);
});
// setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
// run("ALTER TABLE st1 TTL 10");
// clearAlterStbReq();
// setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
// run("ALTER TABLE st1 TTL 10");
// clearAlterStbReq();
setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
run("ALTER TABLE st1 COMMENT 'test'");
@ -381,9 +381,48 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) {
TEST_F(ParserInitialATest, alterUser) {
useDb("root", "test");
run("ALTER user wxy PASS '123456'");
SAlterUserReq expect = {0};
run("ALTER user wxy privilege 'write'");
auto clearAlterUserReq = [&]() { memset(&expect, 0, sizeof(SAlterUserReq)); };
auto setAlterUserReq = [&](const char* pUser, int8_t alterType, const char* pPass = nullptr, int8_t sysInfo = 0,
int8_t enable = 0) {
strcpy(expect.user, pUser);
expect.alterType = alterType;
expect.superUser = 0;
expect.sysInfo = sysInfo;
expect.enable = enable;
if (nullptr != pPass) {
strcpy(expect.pass, pPass);
}
strcpy(expect.dbname, "test");
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_USER_STMT);
SAlterUserReq req = {0};
ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
ASSERT_EQ(req.alterType, expect.alterType);
ASSERT_EQ(req.superUser, expect.superUser);
ASSERT_EQ(req.sysInfo, expect.sysInfo);
ASSERT_EQ(req.enable, expect.enable);
ASSERT_EQ(std::string(req.user), std::string(expect.user));
ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
ASSERT_EQ(std::string(req.dbname), std::string(expect.dbname));
});
setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456");
run("ALTER USER wxy PASS '123456'");
clearAlterUserReq();
setAlterUserReq("wxy", TSDB_ALTER_USER_ENABLE, nullptr, 0, 1);
run("ALTER USER wxy ENABLE 1");
clearAlterUserReq();
setAlterUserReq("wxy", TSDB_ALTER_USER_SYSINFO, nullptr, 1);
run("ALTER USER wxy SYSINFO 1");
clearAlterUserReq();
}
TEST_F(ParserInitialATest, balanceVgroup) {

48
source/libs/parser/test/parInitialCTest.cpp
View File

@ -370,7 +370,7 @@ TEST_F(ParserInitialCTest, createStable) {
expect.delay2 = delay2;
expect.watermark1 = watermark1;
expect.watermark2 = watermark2;
// expect.ttl = ttl;
// expect.ttl = ttl;
if (nullptr != pComment) {
expect.comment = strdup(pComment);
expect.commentLen = strlen(pComment);
@ -414,7 +414,7 @@ TEST_F(ParserInitialCTest, createStable) {
ASSERT_EQ(req.ttl, expect.ttl);
ASSERT_EQ(req.numOfColumns, expect.numOfColumns);
ASSERT_EQ(req.numOfTags, expect.numOfTags);
// ASSERT_EQ(req.commentLen, expect.commentLen);
// ASSERT_EQ(req.commentLen, expect.commentLen);
ASSERT_EQ(req.ast1Len, expect.ast1Len);
ASSERT_EQ(req.ast2Len, expect.ast2Len);
@ -621,10 +621,11 @@ TEST_F(ParserInitialCTest, createTopic) {
auto clearCreateTopicReq = [&]() { memset(&expect, 0, sizeof(SCMCreateTopicReq)); };
auto setCreateTopicReqFunc = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst,
const char* pDbName = nullptr, const char* pTbname = nullptr) {
const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) {
snprintf(expect.name, sizeof(expect.name), "0.%s", pTopicName);
expect.igExists = igExists;
expect.sql = (char*)pSql;
expect.withMeta = withMeta;
if (nullptr != pTbname) {
expect.subType = TOPIC_SUB_TYPE__TABLE;
snprintf(expect.subStbName, sizeof(expect.subStbName), "0.%s.%s", pDbName, pTbname);
@ -647,6 +648,7 @@ TEST_F(ParserInitialCTest, createTopic) {
ASSERT_EQ(req.igExists, expect.igExists);
ASSERT_EQ(req.subType, expect.subType);
ASSERT_EQ(std::string(req.sql), std::string(expect.sql));
ASSERT_EQ(req.withMeta, expect.withMeta);
switch (expect.subType) {
case TOPIC_SUB_TYPE__DB:
ASSERT_EQ(std::string(req.subDbName), std::string(expect.subDbName));
@ -675,15 +677,55 @@ TEST_F(ParserInitialCTest, createTopic) {
run("CREATE TOPIC tp1 AS DATABASE test");
clearCreateTopicReq();
setCreateTopicReqFunc("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1);
run("CREATE TOPIC tp1 WITH META AS DATABASE test");
clearCreateTopicReq();
setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1");
run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1");
clearCreateTopicReq();
setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1);
run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1");
clearCreateTopicReq();
}
TEST_F(ParserInitialCTest, createUser) {
useDb("root", "test");
SCreateUserReq expect = {0};
auto clearCreateUserReq = [&]() { memset(&expect, 0, sizeof(SCreateUserReq)); };
auto setCreateUserReq = [&](const char* pUser, const char* pPass, int8_t sysInfo = 1) {
strcpy(expect.user, pUser);
strcpy(expect.pass, pPass);
expect.createType = 0;
expect.superUser = 0;
expect.sysInfo = sysInfo;
expect.enable = 1;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_USER_STMT);
SCreateUserReq req = {0};
ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
ASSERT_EQ(req.createType, expect.createType);
ASSERT_EQ(req.superUser, expect.superUser);
ASSERT_EQ(req.sysInfo, expect.sysInfo);
ASSERT_EQ(req.enable, expect.enable);
ASSERT_EQ(std::string(req.user), std::string(expect.user));
ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
});
setCreateUserReq("wxy", "123456");
run("CREATE USER wxy PASS '123456'");
clearCreateUserReq();
setCreateUserReq("wxy1", "a123456", 1);
run("CREATE USER wxy1 PASS 'a123456' SYSINFO 1");
clearCreateUserReq();
}
} // namespace ParserTest

26
source/libs/parser/test/parInitialDTest.cpp
View File

@ -132,7 +132,31 @@ TEST_F(ParserInitialDTest, dropDnode) {
TEST_F(ParserInitialDTest, dropIndex) {
useDb("root", "test");
run("DROP index index1 on t1");
SMDropSmaReq expect = {0};
auto clearDropSmaReq = [&]() { memset(&expect, 0, sizeof(SMDropSmaReq)); };
auto setDropSmaReq = [&](const char* pName, int8_t igNotExists = 0) {
sprintf(expect.name, "0.test.%s", pName);
expect.igNotExists = igNotExists;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_INDEX_STMT);
SMDropSmaReq req = {0};
ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMDropSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
ASSERT_EQ(std::string(req.name), std::string(expect.name));
ASSERT_EQ(req.igNotExists, expect.igNotExists);
});
setDropSmaReq("index1");
run("DROP INDEX index1");
clearDropSmaReq();
setDropSmaReq("index2", 1);
run("DROP INDEX IF EXISTS index2");
clearDropSmaReq();
}
TEST_F(ParserInitialDTest, dropMnode) {

2
source/libs/parser/test/parSelectTest.cpp
View File

@ -58,6 +58,8 @@ TEST_F(ParserSelectTest, expression) {
run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1");
run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1");
run("SELECT c1 | 10, c2 & 20, c4 | c5 FROM t1");
}
TEST_F(ParserSelectTest, condition) {

10
source/libs/planner/src/planOptimizer.c
View File

@ -79,7 +79,8 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func)
EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
*((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
// *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
*((bool*)pContext) = true;
return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
@ -98,11 +99,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) {
return false;
}
// todo: release after function splitting
if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType &&
SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) {
return false;
}
if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) &&
QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) &&
QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) {
@ -775,7 +771,7 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) {
}
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) {
if (NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) {
return false;
}

91
source/libs/planner/src/planSpliter.c
View File

@ -166,6 +166,31 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) {
return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild));
}
static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
if (WINDOW_TYPE_INTERVAL == pWindow->winType) {
return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
}
if (WINDOW_TYPE_SESSION == pWindow->winType) {
if (!streamQuery) {
return stbSplHasMultiTbScan(streamQuery, pNode);
} else {
return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
}
}
if (WINDOW_TYPE_STATE == pWindow->winType) {
if (!streamQuery) {
return stbSplHasMultiTbScan(streamQuery, pNode);
} else {
return false;
}
}
return false;
}
static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
switch (nodeType(pNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
@ -174,13 +199,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
return !(((SJoinLogicNode*)pNode)->isSingleTableJoin);
case QUERY_NODE_LOGIC_PLAN_AGG:
return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
case QUERY_NODE_LOGIC_PLAN_WINDOW: {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType)) {
return false;
}
return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
}
case QUERY_NODE_LOGIC_PLAN_WINDOW:
return stbSplNeedSplitWindow(streamQuery, pNode);
case QUERY_NODE_LOGIC_PLAN_SORT:
return stbSplHasMultiTbScan(streamQuery, pNode);
default:
@ -477,11 +497,64 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
return code;
}
static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
((SScanLogicNode*)pNode)->scanType = scanType;
} else {
if (1 == LIST_LENGTH(pNode->pChildren)) {
splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType);
}
}
}
static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
SLogicNode* pWindow = pInfo->pSplitNode;
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0);
SNodeList* pMergeKeys = NULL;
int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
(SNode*)splCreateScanSubplan(pCxt, pChild, SPLIT_FLAG_STABLE_SPLIT));
}
if (TSDB_CODE_SUCCESS == code) {
splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE);
++(pCxt->groupId);
}
if (TSDB_CODE_SUCCESS == code) {
pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
} else {
nodesDestroyList(pMergeKeys);
}
return code;
}
static int32_t stbSplSplitSession(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (pCxt->pPlanCxt->streamQuery) {
return stbSplSplitSessionForStream(pCxt, pInfo);
} else {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
}
}
static int32_t stbSplSplitStateForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (pCxt->pPlanCxt->streamQuery) {
return stbSplSplitStateForStream(pCxt, pInfo);
} else {
return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
}
}
@ -511,6 +584,8 @@ static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitI
return stbSplSplitInterval(pCxt, pInfo);
case WINDOW_TYPE_SESSION:
return stbSplSplitSession(pCxt, pInfo);
case WINDOW_TYPE_STATE:
return stbSplSplitState(pCxt, pInfo);
default:
break;
}

10
source/libs/planner/test/planSessionTest.cpp
View File

@ -34,3 +34,13 @@ TEST_F(PlanSessionTest, selectFunc) {
// select function along with the columns of select row, and with SESSION clause
run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)");
}
TEST_F(PlanSessionTest, stable) {
useDb("root", "test");
// select function for SESSION clause
run("SELECT MAX(c1), MIN(c1) FROM st1 SESSION(ts, 10s)");
// select function along with the columns of select row, and with SESSION clause
run("SELECT MAX(c1), c2 FROM st1 SESSION(ts, 10s)");
run("SELECT count(ts) FROM st1 PARTITION BY c1 SESSION(ts, 10s)");
}

9
source/libs/planner/test/planStateTest.cpp
View File

@ -40,3 +40,12 @@ TEST_F(PlanStateTest, selectFunc) {
// select function along with the columns of select row, and with STATE_WINDOW clause
run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)");
}
TEST_F(PlanStateTest, stable) {
useDb("root", "test");
// select function for STATE_WINDOW clause
run("SELECT MAX(c1), MIN(c1) FROM st1 STATE_WINDOW(c2)");
// select function along with the columns of select row, and with STATE_WINDOW clause
run("SELECT MAX(c1), c2 FROM st1 STATE_WINDOW(c2)");
}

5
source/libs/stream/inc/streamInc.h
View File

@ -33,8 +33,13 @@ static SStreamGlobalEnv streamEnv;
int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData);
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet);
int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock);
int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);
#ifdef __cplusplus
}
#endif

60
source/libs/stream/src/stream.c
View File

@ -112,7 +112,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
// enqueue
if (pData != NULL) {
pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
pData->sourceVg = pReq->sourceVg;
pData->srcVgId = pReq->dataSrcVgId;
// decode
/*pData->blocks = pReq->data;*/
/*pBlock->sourceVer = pReq->sourceVer;*/
@ -133,7 +133,42 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
pCont->inputStatus = status;
pCont->streamId = pReq->streamId;
pCont->taskId = pReq->sourceTaskId;
pCont->taskId = pReq->upstreamTaskId;
pRsp->pCont = buf;
pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
tmsgSendRsp(pRsp);
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
}
int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
int8_t status = TASK_INPUT_STATUS__NORMAL;
// enqueue
if (pData != NULL) {
pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
pData->srcVgId = 0;
// decode
/*pData->blocks = pReq->data;*/
/*pBlock->sourceVer = pReq->sourceVer;*/
streamRetrieveReqToData(pReq, pData);
if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) {
status = TASK_INPUT_STATUS__NORMAL;
} else {
status = TASK_INPUT_STATUS__FAILED;
}
} else {
/*streamTaskInputFail(pTask);*/
/*status = TASK_INPUT_STATUS__FAILED;*/
}
// rsp by input status
void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp));
((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId);
SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
pCont->streamId = pReq->streamId;
pCont->rspToTaskId = pReq->srcTaskId;
pCont->rspFromTaskId = pReq->dstTaskId;
pRsp->pCont = buf;
pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
tmsgSendRsp(pRsp);
@ -141,7 +176,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
}
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->sourceTaskId);
qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->upstreamTaskId);
// 1. handle input
streamTaskEnqueue(pTask, pReq, pRsp);
@ -208,3 +243,22 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp)
//
return 0;
}
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
qInfo("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId);
streamTaskEnqueueRetrieve(pTask, pReq, pRsp);
ASSERT(pTask->execType != TASK_EXEC__NONE);
streamExec(pTask, pTask->pMsgCb);
ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);
streamDispatch(pTask, pTask->pMsgCb);
return 0;
}
int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) {
//
return 0;
}

36
source/libs/stream/src/streamData.c
View File

@ -15,27 +15,6 @@
#include "streamInc.h"
#if 0
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pOutput->type);
tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
tlen += tEncodeDataBlocks(buf, pOutput->blocks);
return tlen;
}
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
buf = taosDecodeFixedI8(buf, &pInput->type);
buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
buf = tDecodeDataBlocks(buf, &pInput->blocks);
return (void*)buf;
}
#endif
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) {
int32_t blockNum = pReq->blockNum;
SArray* pArray = taosArrayInit(blockNum, sizeof(SSDataBlock));
@ -54,12 +33,25 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock
blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
// TODO: refactor
pDataBlock->info.type = pRetrieve->streamBlockType;
pDataBlock->info.childId = pReq->sourceChildId;
pDataBlock->info.childId = pReq->upstreamChildId;
}
pData->blocks = pArray;
return 0;
}
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) {
SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock));
if (pArray == NULL) {
return -1;
}
taosArraySetSize(pArray, 1);
SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
SSDataBlock* pBlock = taosArrayGet(pArray, 0);
blockCompressDecode(pBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
pData->blocks = pArray;
return 0;
}
SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
if (pDataSubmit == NULL) return NULL;

116
source/libs/stream/src/streamDispatch.c
View File

@ -19,9 +19,9 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->dataSrcVgId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
@ -40,9 +40,9 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->dataSrcVgId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
ASSERT(pReq->blockNum > 0);
@ -62,6 +62,102 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
return 0;
}
int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) {
//
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->srcTaskId) < 0) return -1;
if (tEncodeBinary(pEncoder, (const uint8_t*)&pReq->pRetrieve, pReq->retrieveLen) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) {
int32_t tlen = 0;
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1;
if (tDecodeBinary(pDecoder, (uint8_t**)&pReq->pRetrieve, &pReq->retrieveLen) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) {
SRetrieveTableRsp* pRetrieve = NULL;
void* buf = NULL;
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
pRetrieve = taosMemoryCalloc(1, dataStrLen);
if (pRetrieve == NULL) return -1;
pRetrieve->useconds = 0;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->streamBlockType = pBlock->info.type;
pRetrieve->numOfRows = htonl(pBlock->info.rows);
pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
SStreamRetrieveReq req = {
.streamId = pTask->streamId,
.srcNodeId = pTask->nodeId,
.srcTaskId = pTask->taskId,
.pRetrieve = pRetrieve,
};
int32_t sz = taosArrayGetSize(pTask->childEpInfo);
ASSERT(sz > 0);
for (int32_t i = 0; i < sz; i++) {
SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i);
req.dstNodeId = pEpInfo->nodeId;
req.dstTaskId = pEpInfo->taskId;
int32_t code;
int32_t len;
tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code);
if (code < 0) {
ASSERT(0);
return -1;
}
buf = rpcMallocCont(sizeof(SMsgHead) + len);
if (buf == NULL) {
goto FAIL;
}
((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId);
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
tEncodeStreamRetrieveReq(&encoder, &req);
SRpcMsg rpcMsg = {
.code = 0,
.msgType = TDMT_STREAM_RETRIEVE,
.pCont = buf,
.contLen = len,
};
if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
ASSERT(0);
return -1;
}
}
return 0;
FAIL:
if (pRetrieve) taosMemoryFree(pRetrieve);
if (buf) taosMemoryFree(buf);
return -1;
}
static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
@ -94,9 +190,9 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
SStreamDispatchReq req = {
.streamId = pTask->streamId,
.sourceTaskId = pTask->taskId,
.sourceVg = data->sourceVg,
.sourceChildId = pTask->childId,
.dataSrcVgId = data->srcVgId,
.upstreamTaskId = pTask->taskId,
.upstreamChildId = pTask->selfChildId,
.upstreamNodeId = pTask->nodeId,
.blockNum = blockNum,
};
@ -147,7 +243,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
ASSERT(vgId > 0 || vgId == SNODE_HANDLE);
req.taskId = downstreamTaskId;
qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->childId,
qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId,
downstreamTaskId, vgId);
// serialize

36
source/libs/stream/src/streamExec.c
View File

@ -13,8 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executor.h"
#include "tstream.h"
#include "streamInc.h"
static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
void* exec = pTask->exec.executor;
@ -25,13 +24,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
SStreamTrigger* pTrigger = (SStreamTrigger*)data;
qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->isDataScan);
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
ASSERT(pTask->inputType == STREAM_INPUT__DATA_SUBMIT);
qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK);
SArray* blocks = pBlock->blocks;
SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
} else if (pItem->type == STREAM_INPUT__DROP) {
// TODO exec drop
@ -46,9 +44,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
ASSERT(false);
}
if (output == NULL) break;
if (output->info.type == STREAM_RETRIEVE) {
if (streamBroadcastToChildren(pTask, output) < 0) {
// TODO
}
continue;
}
// TODO: do we need free memory?
SSDataBlock* outputCopy = createOneDataBlock(output, true);
outputCopy->info.childId = pTask->childId;
outputCopy->info.childId = pTask->selfChildId;
taosArrayPush(pRes, outputCopy);
}
return 0;
@ -82,17 +88,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
return NULL;
}
if (((SStreamQueueItem*)data)->type == STREAM_INPUT__TRIGGER) {
int8_t type = ((SStreamQueueItem*)data)->type;
if (type == STREAM_INPUT__TRIGGER) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data);
} else {
if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
} else {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
}
} else if (type == STREAM_INPUT__DATA_BLOCK) {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->isDataScan);
streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
}
streamQueueProcessSuccess(pTask->inputQueue);
return taosArrayInit(0, sizeof(SSDataBlock));

53
source/libs/stream/src/streamTask.c
View File

@ -30,23 +30,46 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
return pTask;
}
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) {
if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
return 0;
}
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
return 0;
}
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
/*if (tStartEncode(pEncoder) < 0) return -1;*/
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->dataScan) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->childId) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;
int32_t epSz = taosArrayGetSize(pTask->childEpInfo);
if (tEncodeI32(pEncoder, epSz) < 0) return -1;
for (int32_t i = 0; i < epSz; i++) {
SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->childEpInfo, i);
if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1;
}
if (pTask->execType != TASK_EXEC__NONE) {
if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1;
}
@ -83,19 +106,29 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
/*if (tStartDecode(pDecoder) < 0) return -1;*/
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->dataScan) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->childId) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;
int32_t epSz;
if (tDecodeI32(pDecoder, &epSz) < 0) return -1;
pTask->childEpInfo = taosArrayInit(epSz, sizeof(void*));
for (int32_t i = 0; i < epSz; i++) {
SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo));
if (pInfo == NULL) return -1;
if (tDecodeStreamEpInfo(pDecoder, pInfo) < 0) return -1;
taosArrayPush(pTask->childEpInfo, &pInfo);
}
if (pTask->execType != TASK_EXEC__NONE) {
if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1;
}

2
source/libs/wal/src/walMeta.c
View File

@ -19,6 +19,8 @@
#include "tref.h"
#include "walInt.h"
bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; }
int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }

15
source/libs/wal/src/walWrite.c
View File

@ -141,7 +141,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// validate offset
SWalHead head;
ASSERT(taosValidFile(pLogTFile));
int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
if (size != sizeof(SWalHead)) {
return -1;
}
@ -149,22 +149,33 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(code == 0);
if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (head.head.version != ver) {
// TODO
ASSERT(0);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
// truncate old files
code = taosFtruncateFile(pLogTFile, entry.offset);
if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosFtruncateFile(pIdxTFile, idxOff);
if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pWal->vers.lastVer = ver - 1;
if (pWal->vers.lastVer < pWal->vers.firstVer) {
ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1);
pWal->vers.firstVer = -1;
}
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
taosCloseFile(&pIdxTFile);

4
source/os/src/osLocale.c
View File

@ -88,11 +88,11 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
void taosGetSystemLocale(char *outLocale, char *outCharset) {
#ifdef WINDOWS
char *locale = setlocale(LC_CTYPE, "chs");
char *locale = setlocale(LC_CTYPE, "en_US.UTF-8");
if (locale != NULL) {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
}
strcpy(outCharset, "cp936");
strcpy(outCharset, "UTF-8");
#elif defined(_TD_DARWIN_64)
/*

302
tests/pytest/util/cluster.py Normal file
View File

@ -0,0 +1,302 @@
from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
import taos
import sys
import time
import os
import socket
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
class ClusterDnodes(TDDnodes):
"""rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
def __init__(self ,dnodes_lists):
super(ClusterDnodes,self).__init__()
self.dnodes = dnodes_lists # dnode must be TDDnode instance
self.simDeployed = False
self.testCluster = False
self.valgrind = 0
self.killValgrind = 1
self.independent = True
self.dnodeNums = 5
# def getTDDnodes(dnodeNums):
# return
class ConfigureyCluster:
"""configure dnodes and return TDDnodes list, it can """
def __init__(self):
self.dnodes = None
self.dnodes_nums = 5
self.independent = True
self.start_port = 6030
self.portStep = 100
hostname1= socket.gethostname()
def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1):
self.start_port=int(start_port)
self.portStep=int(portStep)
self.hostname=hostname
self.dnodes_nums = int(dnodes_nums)
self.independent = independent
self.dnodes = []
start_port_sec = 6130
for num in range(1, (self.dnodes_nums+1)):
dnode = TDDnode(num)
dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}")
dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}")
# dnode.addExtraCfg("monitorFqdn", hostname)
# dnode.addExtraCfg("monitorPort", 7043)
dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
# configure three dnoe don't support vnodes
if self.dnodes_nums > 4 :
if self.independent and (num < 4):
dnode.addExtraCfg("supportVnodes", 0)
# print(dnode)
self.dnodes.append(dnode)
return self.dnodes
def create_dnode(self,conn):
tdSql.init(conn.cursor())
for dnode in self.dnodes[1:]:
# print(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
print(dnode_id)
tdSql.execute(" create dnode '%s';"%dnode_id)
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show dnodes;")
# if tdSql.checkRows(self.dnodes_nums) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][4]=='leader' :
# if tdSql.queryResult[2][4]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
checkstatus=False
def check_dnode(self,conn):
tdSql.init(conn.cursor())
count=0
while count < 5:
tdSql.query("show dnodes")
# tdLog.debug(tdSql.queryResult)
status=0
for i in range(self.dnodes_nums):
if tdSql.queryResult[i][4] == "ready":
status+=1
tdLog.debug(status)
if status == self.dnodes_nums:
tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums)
break
count+=1
time.sleep(1)
else:
tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums)
return -1
cluster = ConfigureyCluster()
# def start(self ,dnodes_nums):
# self.TDDnodes = ClusterDnodes(dnodes)
# self.TDDnodes.init("")
# self.TDDnodes.setTestCluster(testCluster)
# self.TDDnodes.setValgrind(valgrind)
# self.TDDnodes.stopAll()
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.deploy(dnode.index,{})
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.starttaosd(dnode.index)
# # create cluster
# for dnode in self.TDDnodes.dnodes[1:]:
# # print(dnode.cfgDict)
# dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
# dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
# dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
# cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
# print(cmd)
# os.system(cmd)
# time.sleep(2)
# tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
# def buildcluster(self,dnodenumber):
# self.depoly_cluster(dnodenumber)
# self.master_dnode = self.TDDnodes.dnodes[0]
# self.host=self.master_dnode.cfgDict["fqdn"]
# conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
# tdSql.init(conn1.cursor())
# def checkdnodes(self,dnodenumber):
# count=0
# while count < 10:
# time.sleep(1)
# statusReadyBumber=0
# tdSql.query("show dnodes;")
# if tdSql.checkRows(dnodenumber) :
# print("dnode is %d nodes"%dnodenumber)
# for i in range(dnodenumber):
# if tdSql.queryResult[i][4] !='ready' :
# status=tdSql.queryResult[i][4]
# print("dnode:%d status is %s "%(i,status))
# break
# else:
# statusReadyBumber+=1
# print(statusReadyBumber)
# if statusReadyBumber == dnodenumber :
# print("all of %d mnodes is ready in 10s "%dnodenumber)
# return True
# break
# count+=1
# else:
# print("%d mnodes is not ready in 10s "%dnodenumber)
# return False
# def check3mnode(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("three mnodes is ready in 10s")
# break
# count+=1
# else:
# print("three mnodes is not ready in 10s ")
# return -1
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode1off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='offline' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# elif tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 1;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'offline')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode2off(self):
# count=0
# while count < 40:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='offline':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 2;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'offline')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'follower')
# tdSql.checkData(2,3,'ready')
# def check3mnode3off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[2][2]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
# tdSql.error("drop mnode on dnode 3;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'follower')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'offline')
# tdSql.checkData(2,3,'ready')

5
tests/script/jenkins/basic.txt
View File

@ -12,6 +12,8 @@
## ---- db
./test.sh -f tsim/db/create_all_options.sim
./test.sh -f tsim/db/alter_option.sim
./test.sh -f tsim/db/alter_replica_13.sim
#./test.sh -f tsim/db/alter_replica_31.sim
./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic2.sim
./test.sh -f tsim/db/basic3.sim
@ -21,6 +23,7 @@
./test.sh -f tsim/db/taosdlog.sim
# ---- dnode
./test.sh -f tsim/dnode/balance_replica1.sim
./test.sh -f tsim/dnode/create_dnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@ -28,6 +31,8 @@
#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
# ---- insert
./test.sh -f tsim/insert/basic0.sim

123
tests/script/tsim/dnode/balance_replica1.sim Normal file
View File

@ -0,0 +1,123 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sql connect
print =============== step1 create dnode2
# no enough vnodes
sql balance vgroup
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
$x = 0
step1:
$ = $x + 1
sleep 1000
if $x == 10 then
print ---> dnode not online!
return -1
endi
sql show dnodes
print ---> $data00 $data01 $data02 $data03 $data04 $data05
print ---> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != offline then
goto step1
endi
print =============== step2 create database
sql create database d1 vgroups 2
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql create table d1.c2 using st tags(1)
sql create table d1.c3 using st tags(1)
sql create table d1.c4 using st tags(1)
sql create table d1.c5 using st tags(1)
sql create table d1.c6 using st tags(1)
sql show d1.tables
if $rows != 6 then
return -1
endi
sql show d1.vgroups
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
if $rows != 2 then
return -1
endi
if $data(2)[3] != 2 then
return -1
endi
if $data(3)[3] != 2 then
return -1
endi
print =============== step3: balance vgroup
# has offline dnode
sql_error balance vgroup
system sh/exec.sh -n dnode3 -s start
$x = 0
step3:
$ = $x + 1
sleep 1000
if $x == 10 then
print ---> dnode not online!
return -1
endi
sql show dnodes
print ---> $data00 $data01 $data02 $data03 $data04 $data05
print ---> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step3
endi
if $data(2)[4] != ready then
goto step3
endi
if $data(3)[4] != ready then
goto step3
endi
print =============== step4: balance
sql balance vgroup
print show d1.vgroups
sql show d1.vgroups
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
if $rows != 2 then
return -1
endi
if $data(2)[3] != 3 then
return -1
endi
if $data(3)[3] != 2 then
return -1
endi
print =============== step7: select data
sql show d1.tables
print rows $rows
if $rows != 6 then
return -1
endi
return
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

9
tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
View File

@ -28,7 +28,7 @@ step1:
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
@ -72,6 +72,13 @@ if $data(2)[7] != 4 then
return -1
endi
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
return
print =============== step4: drop dnode 2
system sh/exec.sh -n dnode5 -s start
$x = 0

2
tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
View File

@ -176,6 +176,8 @@ if $rows != 1 then
return -1
endi
return
print =============== step33: move follower1
print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5
sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5

2
tests/script/wtest.bat
View File

@ -52,7 +52,7 @@ echo wal 0 >> %TAOS_CFG%
echo asyncLog 0 >> %TAOS_CFG%
echo locale en_US.UTF-8 >> %TAOS_CFG%
echo enableCoreFile 1 >> %TAOS_CFG%
echo charset cp65001 >> %TAOS_CFG%
echo charset UTF-8 >> %TAOS_CFG%
set "FILE_NAME=testSuite.sim"
if "%1" == "-f" set "FILE_NAME=%2"

473
tests/system-test/2-query/abs.py
View File

@ -6,14 +6,62 @@ import inspect
from util.log import *
from util.sql import *
from util.cases import *
import random
class TDTestCase:
updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 ,
"jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143,
"wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143}
updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143,
"jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143,
"wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.tb_nums = 10
self.row_nums = 20
self.ts = 1434938400000
self.time_step = 1000
def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ):
tdLog.info(" prepare datas for auto check abs function ")
tdSql.execute(" create database test ")
tdSql.execute(" use test ")
tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\
c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)")
for tbnum in range(tbnums):
tbname = "sub_tb_%d"%tbnum
tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum))
ts = self.ts
for row in range(rownums):
ts += time_step*row
c1 = random.randint(0,10000)
c2 = random.randint(0,100000)
c3 = random.randint(0,125)
c4 = random.randint(0,125)
c5 = random.random()/1.0
c6 = random.random()/1.0
c7 = "'true'"
c8 = "'binary_val'"
c9 = "'nchar_val'"
c10 = ts
tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})")
tdSql.execute("use test")
tbnames = ["stb", "sub_tb_1"]
support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
for tbname in tbnames:
tdSql.query("desc {}".format(tbname))
coltypes = tdSql.queryResult
for coltype in coltypes:
colname = coltype[0]
abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname)
origin_sql = "select {} from {} order by tbname".format(colname, tbname)
if coltype[1] in support_types:
self.check_result_auto(origin_sql , abs_sql)
def prepare_datas(self):
tdSql.execute(
@ -39,14 +87,21 @@ class TDTestCase:
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
@ -65,10 +120,10 @@ class TDTestCase:
'''
)
def prepare_tag_datas(self):
# prepare datas
tdSql.execute("create database if not exists testdb keep 3650 duration 1000")
# prepare datas
tdSql.execute(
"create database if not exists testdb keep 3650 duration 1000")
tdSql.execute(" use testdb ")
tdSql.execute(
'''create table stb1
@ -84,7 +139,8 @@ class TDTestCase:
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
tdSql.execute(
f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
for i in range(9):
tdSql.execute(
@ -93,14 +149,21 @@ class TDTestCase:
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
@ -119,18 +182,18 @@ class TDTestCase:
'''
)
def check_result_auto(self ,origin_query , abs_query):
def check_result_auto(self, origin_query, abs_query):
abs_result = tdSql.getResult(abs_query)
origin_result = tdSql.getResult(origin_query)
auto_result =[]
auto_result = []
for row in origin_result:
row_check = []
for elem in row:
if elem == None:
elem = None
elif elem >=0:
elif elem >= 0:
elem = elem
else:
elem = -elem
@ -138,21 +201,23 @@ class TDTestCase:
auto_result.append(row_check)
check_status = True
for row_index , row in enumerate(abs_result):
for col_index , elem in enumerate(row):
for row_index, row in enumerate(abs_result):
for col_index, elem in enumerate(row):
if auto_result[row_index][col_index] != elem:
check_status = False
if not check_status:
tdLog.notice("abs function value has not as expected , sql is \"%s\" "%abs_query )
tdLog.notice(
"abs function value has not as expected , sql is \"%s\" " % abs_query)
sys.exit(1)
else:
tdLog.info("abs value check pass , it work as expected ,sql is \"%s\" "%abs_query )
tdLog.info(
"abs value check pass , it work as expected ,sql is \"%s\" " % abs_query)
def test_errors(self):
tdSql.execute("use testdb")
error_sql_lists = [
"select abs from t1",
# "select abs(-+--+c1) from t1",
"select abs(-+--+c1) from t1",
# "select +-abs(c1) from t1",
# "select ++-abs(c1) from t1",
# "select ++--abs(c1) from t1",
@ -185,86 +250,18 @@ class TDTestCase:
def support_types(self):
tdSql.execute("use testdb")
type_error_sql_lists = [
"select abs(ts) from t1" ,
"select abs(t0) from t1" ,
"select abs(c7) from t1",
"select abs(c8) from t1",
"select abs(c9) from t1",
"select abs(t7) from t1",
"select abs(t8) from t1",
"select abs(t9) from t1",
"select abs(ts) from ct1" ,
"select abs(c7) from ct1",
"select abs(c8) from ct1",
"select abs(c9) from ct1",
"select abs(ts) from ct3" ,
"select abs(c7) from ct3",
"select abs(c8) from ct3",
"select abs(c9) from ct3",
"select abs(ts) from ct4" ,
"select abs(c7) from ct4",
"select abs(c8) from ct4",
"select abs(c9) from ct4",
"select abs(ts) from stb1" ,
"select abs(c7) from stb1",
"select abs(c8) from stb1",
"select abs(c9) from stb1" ,
"select abs(ts) from stbbb1" ,
"select abs(c7) from stbbb1",
"select abs(ts) from tbname",
"select abs(c9) from tbname"
]
for type_sql in type_error_sql_lists:
tdSql.error(type_sql)
type_sql_lists = [
"select abs(c1) from t1",
"select abs(c2) from t1",
"select abs(c3) from t1",
"select abs(c4) from t1",
"select abs(c5) from t1",
"select abs(c6) from t1",
"select abs(t1) from ct1",
"select abs(t2) from ct1",
"select abs(t3) from ct1",
"select abs(t4) from ct1",
"select abs(t5) from ct1",
"select abs(t6) from ct1",
"select abs(c1) from ct1",
"select abs(c2) from ct1",
"select abs(c3) from ct1",
"select abs(c4) from ct1",
"select abs(c5) from ct1",
"select abs(c6) from ct1",
"select abs(c1) from ct3",
"select abs(c2) from ct3",
"select abs(c3) from ct3",
"select abs(c4) from ct3",
"select abs(c5) from ct3",
"select abs(c6) from ct3",
"select abs(c1) from stb1",
"select abs(c2) from stb1",
"select abs(c3) from stb1",
"select abs(c4) from stb1",
"select abs(c5) from stb1",
"select abs(c6) from stb1",
"select abs(c6) as alisb from stb1",
"select abs(c6) alisb from stb1",
]
for type_sql in type_sql_lists:
tdSql.query(type_sql)
tbnames = ["stb1", "t1", "ct1", "ct2"]
support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
for tbname in tbnames:
tdSql.query("desc {}".format(tbname))
coltypes = tdSql.queryResult
for coltype in coltypes:
colname = coltype[0]
abs_sql = "select abs({}) from {}".format(colname, tbname)
if coltype[1] in support_types:
tdSql.query(abs_sql)
else:
tdSql.error(abs_sql)
def basic_abs_function(self):
@ -292,9 +289,9 @@ class TDTestCase:
# used for regular table
tdSql.query("select abs(c1) from t1")
tdSql.checkData(0, 0, None)
tdSql.checkData(1 , 0, 1)
tdSql.checkData(3 , 0, 3)
tdSql.checkData(5 , 0, None)
tdSql.checkData(1, 0, 1)
tdSql.checkData(3, 0, 3)
tdSql.checkData(5, 0, None)
tdSql.query("select c1, c2, c3 , c4, c5 from t1")
tdSql.checkData(1, 4, 1.11000)
@ -305,26 +302,30 @@ class TDTestCase:
tdSql.checkData(3, 4, 33)
tdSql.checkData(5, 5, None)
self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1")
self.check_result_auto("select c1, c2, c3 , c4, c5 from t1",
"select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1")
# used for sub table
tdSql.query("select abs(c1) from ct1")
tdSql.checkData(0, 0, 8)
tdSql.checkData(1 , 0, 7)
tdSql.checkData(3 , 0, 5)
tdSql.checkData(5 , 0, 4)
tdSql.checkData(1, 0, 7)
tdSql.checkData(3, 0, 5)
tdSql.checkData(5, 0, 4)
tdSql.query("select abs(c1) from ct1")
self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1")
self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" )
self.check_result_auto("select c1, c2, c3 , c4, c5 from ct1",
"select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1")
self.check_result_auto(
"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;", "select c1 from ct1")
# used for stable table
tdSql.query("select abs(c1) from stb1")
tdSql.checkRows(25)
self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4")
self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" )
self.check_result_auto("select c1, c2, c3 , c4, c5 from ct4 ",
"select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4")
self.check_result_auto(
"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;", "select c1 from ct4")
# used for not exists table
tdSql.error("select abs(c1) from stbbb1")
@ -333,34 +334,34 @@ class TDTestCase:
# mix with common col
tdSql.query("select c1, abs(c1) from ct1")
tdSql.checkData(0 , 0 ,8)
tdSql.checkData(0 , 1 ,8)
tdSql.checkData(4 , 0 ,0)
tdSql.checkData(4 , 1 ,0)
tdSql.checkData(0, 0, 8)
tdSql.checkData(0, 1, 8)
tdSql.checkData(4, 0, 0)
tdSql.checkData(4, 1, 0)
tdSql.query("select c1, abs(c1) from ct4")
tdSql.checkData(0 , 0 , None)
tdSql.checkData(0 , 1 ,None)
tdSql.checkData(4 , 0 ,5)
tdSql.checkData(4 , 1 ,5)
tdSql.checkData(5 , 0 ,None)
tdSql.checkData(5 , 1 ,None)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(4, 0, 5)
tdSql.checkData(4, 1, 5)
tdSql.checkData(5, 0, None)
tdSql.checkData(5, 1, None)
tdSql.query("select c1, abs(c1) from ct4 ")
tdSql.checkData(0 , 0 ,None)
tdSql.checkData(0 , 1 ,None)
tdSql.checkData(4 , 0 ,5)
tdSql.checkData(4 , 1 ,5)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(4, 0, 5)
tdSql.checkData(4, 1, 5)
# mix with common functions
tdSql.query("select c1, abs(c1),c5, floor(c5) from ct4 ")
tdSql.checkData(0 , 0 ,None)
tdSql.checkData(0 , 1 ,None)
tdSql.checkData(0 , 2 ,None)
tdSql.checkData(0 , 3 ,None)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
tdSql.checkData(0, 3, None)
tdSql.checkData(3 , 0 , 6)
tdSql.checkData(3 , 1 , 6)
tdSql.checkData(3 , 2 ,6.66000)
tdSql.checkData(3 , 3 ,6.00000)
tdSql.checkData(3, 0, 6)
tdSql.checkData(3, 1, 6)
tdSql.checkData(3, 2, 6.66000)
tdSql.checkData(3, 3, 6.00000)
tdSql.query("select c1, abs(c1),c5, floor(c5) from stb1 ")
@ -377,16 +378,15 @@ class TDTestCase:
tdSql.query("select max(c5), count(c5) from stb1")
tdSql.query("select max(c5), count(c5) from ct1")
# bug fix for count
tdSql.query("select count(c1) from ct4 ")
tdSql.checkData(0,0,9)
tdSql.checkData(0, 0, 9)
tdSql.query("select count(*) from ct4 ")
tdSql.checkData(0,0,12)
tdSql.checkData(0, 0, 12)
tdSql.query("select count(c1) from stb1 ")
tdSql.checkData(0,0,22)
tdSql.checkData(0, 0, 22)
tdSql.query("select count(*) from stb1 ")
tdSql.checkData(0,0,25)
tdSql.checkData(0, 0, 25)
# bug fix for compute
tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ")
@ -405,43 +405,44 @@ class TDTestCase:
tdSql.checkData(1, 1, 8.000000000)
tdSql.checkData(1, 2, 7.900000000)
def abs_func_filter(self):
tdSql.execute("use db")
tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ")
tdSql.query(
"select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ")
tdSql.checkRows(3)
tdSql.checkData(0,0,8)
tdSql.checkData(0,1,8.000000000)
tdSql.checkData(0,2,8.000000000)
tdSql.checkData(0,3,7.900000000)
tdSql.checkData(0,4,3.000000000)
tdSql.checkData(0, 0, 8)
tdSql.checkData(0, 1, 8.000000000)
tdSql.checkData(0, 2, 8.000000000)
tdSql.checkData(0, 3, 7.900000000)
tdSql.checkData(0, 4, 3.000000000)
tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ")
tdSql.query(
"select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ")
tdSql.checkRows(1)
tdSql.checkData(0,0,5)
tdSql.checkData(0,1,5.000000000)
tdSql.checkData(0,2,5.000000000)
tdSql.checkData(0,3,4.900000000)
tdSql.checkData(0,4,2.000000000)
tdSql.checkData(0, 0, 5)
tdSql.checkData(0, 1, 5.000000000)
tdSql.checkData(0, 2, 5.000000000)
tdSql.checkData(0, 3, 4.900000000)
tdSql.checkData(0, 4, 2.000000000)
tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ")
tdSql.query(
"select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ")
tdSql.checkRows(1)
tdSql.checkData(0,0,5)
tdSql.checkData(0,1,5.000000000)
tdSql.checkData(0,2,5.000000000)
tdSql.checkData(0,3,4.900000000)
tdSql.checkData(0,4,2.000000000)
tdSql.checkData(0, 0, 5)
tdSql.checkData(0, 1, 5.000000000)
tdSql.checkData(0, 2, 5.000000000)
tdSql.checkData(0, 3, 4.900000000)
tdSql.checkData(0, 4, 2.000000000)
tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ")
tdSql.query(
"select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ")
tdSql.checkRows(1)
tdSql.checkData(0,0,8)
tdSql.checkData(0,1,88888)
tdSql.checkData(0,2,8.000000000)
tdSql.checkData(0,3,8.000000000)
tdSql.checkData(0,4,7.900000000)
tdSql.checkData(0,5,3.000000000)
tdSql.checkData(0, 0, 8)
tdSql.checkData(0, 1, 88888)
tdSql.checkData(0, 2, 8.000000000)
tdSql.checkData(0, 3, 8.000000000)
tdSql.checkData(0, 4, 7.900000000)
tdSql.checkData(0, 5, 3.000000000)
def abs_Arithmetic(self):
pass
@ -457,70 +458,80 @@ class TDTestCase:
)
tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )')
tdSql.execute(
f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.error(
f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound")
self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound")
self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;" , "select abs(c1) from sub1_bound" )
f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
self.check_result_auto("select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ",
"select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound")
self.check_result_auto("select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ",
"select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound")
self.check_result_auto(
"select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;", "select abs(c1) from sub1_bound")
# check basic elem for table per row
tdSql.query("select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ")
tdSql.checkData(0,0,2147483647)
tdSql.checkData(0,1,9223372036854775807)
tdSql.checkData(0,2,32767)
tdSql.checkData(0,3,127)
tdSql.checkData(0,4,339999995214436424907732413799364296704.00000)
tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(1,0,2147483647)
tdSql.checkData(1,1,9223372036854775807)
tdSql.checkData(1,2,32767)
tdSql.checkData(1,3,127)
tdSql.checkData(1,4,339999995214436424907732413799364296704.00000)
tdSql.checkData(1,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(3,0,2147483646)
tdSql.checkData(3,1,9223372036854775806)
tdSql.checkData(3,2,32766)
tdSql.checkData(3,3,126)
tdSql.checkData(3,4,339999995214436424907732413799364296704.00000)
tdSql.checkData(3,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.query(
"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ")
tdSql.checkData(0, 0, 2147483647)
tdSql.checkData(0, 1, 9223372036854775807)
tdSql.checkData(0, 2, 32767)
tdSql.checkData(0, 3, 127)
tdSql.checkData(0, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(1, 0, 2147483647)
tdSql.checkData(1, 1, 9223372036854775807)
tdSql.checkData(1, 2, 32767)
tdSql.checkData(1, 3, 127)
tdSql.checkData(1, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(1, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(3, 0, 2147483646)
tdSql.checkData(3, 1, 9223372036854775806)
tdSql.checkData(3, 2, 32766)
tdSql.checkData(3, 3, 126)
tdSql.checkData(3, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(3, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
# check + - * / in functions
tdSql.query("select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
tdSql.checkData(0,0,2147483648.000000000)
tdSql.checkData(0,1,9223372036854775807)
tdSql.checkData(0,2,32767.000000000)
tdSql.checkData(0,3,63.500000000)
tdSql.checkData(0,4,169999997607218212453866206899682148352.000000000)
tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.query(
"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
tdSql.checkData(0, 0, 2147483648.000000000)
tdSql.checkData(0, 1, 9223372036854775807)
tdSql.checkData(0, 2, 32767.000000000)
tdSql.checkData(0, 3, 63.500000000)
tdSql.checkData(
0, 4, 169999997607218212453866206899682148352.000000000)
tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(1, 0, 2147483646.000000000)
tdSql.checkData(1, 1, 9223372036854775808.000000000)
tdSql.checkData(1, 2, 32767.000000000)
tdSql.checkData(1, 3, 63.500000000)
tdSql.checkData(
1, 4, 169999997607218212453866206899682148352.000000000)
tdSql.checkData(1,0,2147483646.000000000)
tdSql.checkData(1,1,9223372036854775808.000000000)
tdSql.checkData(1,2,32767.000000000)
tdSql.checkData(1,3,63.500000000)
tdSql.checkData(1,4,169999997607218212453866206899682148352.000000000)
self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound",
"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
def test_tag_compute_for_scalar_function(self):
tdSql.execute("use testdb")
self.check_result_auto( "select c1, t2, t3 , t4, t5 from ct4 ", "select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4")
self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from ct4 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4")
self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1")
self.check_result_auto("select c1, t2, t3 , t4, t5 from ct4 ",
"select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4")
self.check_result_auto("select c1+2, t2+2, t3 , t4, t5 from ct4 ",
"select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4")
self.check_result_auto("select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ",
"select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1")
# bug need fix
@ -532,7 +543,7 @@ class TDTestCase:
# tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)")
# tdSql.checkData(0,0,61)
# tdSql.query("select distinct(c1) ,t1 from stb1")
# tdSql.checkRows(11)
# tdSql.checkRows(20)
# tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
# tdSql.checkData(0,3,33333)
@ -543,9 +554,8 @@ class TDTestCase:
tdSql.checkRows(1)
# tdSql.query("select t1 from stb1 where abs(t1+c1)=1")
# tdSql.checkRows(1)
tdSql.query("select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
tdSql.query(
"select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -579,9 +589,14 @@ class TDTestCase:
self.test_tag_compute_for_scalar_function()
tdLog.printNoPrefix("==========step7: check result of query ============")
self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

149
tests/system-test/6-cluster/5dnode3mnodeStop.py
View File

@ -7,30 +7,24 @@ import os
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
from util.cluster import *
from test import tdDnodes
import time
import socket
import subprocess
from multiprocessing import Process
class MyDnodes(TDDnodes):
def __init__(self ,dnodes_lists):
super(MyDnodes,self).__init__()
self.dnodes = dnodes_lists # dnode must be TDDnode instance
self.simDeployed = False
class TDTestCase:
def init(self,conn ,logSql):
tdLog.debug(f"start to excute {__file__}")
self.TDDnodes = None
def buildcluster(self,dnodenumber):
self.depoly_cluster(dnodenumber)
self.master_dnode = self.TDDnodes.dnodes[0]
self.host=self.master_dnode.cfgDict["fqdn"]
conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
tdSql.init(conn1.cursor())
tdSql.init(conn.cursor())
self.host = socket.gethostname()
def getBuildPath(self):
@ -70,45 +64,6 @@ class TDTestCase:
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
def depoly_cluster(self ,dnodes_nums):
testCluster = False
valgrind = 0
hostname = socket.gethostname()
dnodes = []
start_port = 6030
for num in range(1, dnodes_nums+1):
dnode = TDDnode(num)
dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
dnode.addExtraCfg("monitorFqdn", hostname)
dnode.addExtraCfg("monitorPort", 7043)
dnodes.append(dnode)
self.TDDnodes = MyDnodes(dnodes)
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.starttaosd(dnode.index)
# create cluster
for dnode in self.TDDnodes.dnodes[1:]:
# tdLog.debug(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
tdLog.debug(cmd)
os.system(cmd)
time.sleep(2)
tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
def check3mnode(self):
count=0
@ -117,25 +72,26 @@ class TDTestCase:
tdSql.query("show mnodes;")
if tdSql.checkRows(3) :
tdLog.debug("mnode is three nodes")
else:
tdLog.exit("mnode number is correct")
if tdSql.queryResult[0][2]=='leader' :
if tdSql.queryResult[1][2]=='follower':
if tdSql.queryResult[2][2]=='follower':
tdLog.debug("three mnodes is ready in 10s")
break
elif tdSql.queryResult[0][2]=='follower' :
if tdSql.queryResult[1][2]=='leader':
elif tdSql.queryResult[1][2]=='leader' :
if tdSql.queryResult[0][2]=='follower':
if tdSql.queryResult[2][2]=='follower':
tdLog.debug("three mnodes is ready in 10s")
break
elif tdSql.queryResult[0][2]=='follower' :
elif tdSql.queryResult[2][2]=='leader' :
if tdSql.queryResult[1][2]=='follower':
if tdSql.queryResult[2][2]=='leader':
if tdSql.queryResult[0][2]=='follower':
tdLog.debug("three mnodes is ready in 10s")
break
count+=1
else:
tdLog.debug("three mnodes is not ready in 10s ")
return -1
tdLog.exit("three mnodes is not ready in 10s ")
tdSql.query("show mnodes;")
tdSql.checkRows(3)
@ -153,6 +109,8 @@ class TDTestCase:
tdSql.query("show mnodes;")
if tdSql.checkRows(3) :
tdLog.debug("mnode is three nodes")
else:
tdLog.exit("mnode number is correct")
if tdSql.queryResult[0][2]=='offline' :
if tdSql.queryResult[1][2]=='leader':
if tdSql.queryResult[2][2]=='follower':
@ -164,8 +122,8 @@ class TDTestCase:
break
count+=1
else:
tdLog.debug("stop mnodes on dnode 2 failed in 10s ")
return -1
tdLog.exit("stop mnodes on dnode 2 failed in 10s ")
tdSql.error("drop mnode on dnode 1;")
tdSql.query("show mnodes;")
@ -185,6 +143,8 @@ class TDTestCase:
tdSql.query("show mnodes;")
if tdSql.checkRows(3) :
tdLog.debug("mnode is three nodes")
else:
tdLog.exit("mnode number is correct")
if tdSql.queryResult[0][2]=='leader' :
if tdSql.queryResult[1][2]=='offline':
if tdSql.queryResult[2][2]=='follower':
@ -192,8 +152,8 @@ class TDTestCase:
break
count+=1
else:
tdLog.debug("stop mnodes on dnode 2 failed in 10s ")
return -1
tdLog.exit("stop mnodes on dnode 2 failed in 10s ")
tdSql.error("drop mnode on dnode 2;")
tdSql.query("show mnodes;")
@ -215,6 +175,8 @@ class TDTestCase:
tdSql.query("show mnodes;")
if tdSql.checkRows(3) :
tdLog.debug("mnode is three nodes")
else:
tdLog.exit("mnode number is correct")
if tdSql.queryResult[0][2]=='leader' :
if tdSql.queryResult[2][2]=='offline':
if tdSql.queryResult[1][2]=='follower':
@ -222,8 +184,8 @@ class TDTestCase:
break
count+=1
else:
tdLog.debug("stop mnodes on dnode 3 failed in 10s")
return -1
tdLog.exit("stop mnodes on dnode 3 failed in 10s")
tdSql.error("drop mnode on dnode 3;")
tdSql.query("show mnodes;")
tdSql.checkRows(3)
@ -238,14 +200,29 @@ class TDTestCase:
tdSql.checkData(2,3,'ready')
def check_dnodes_status(self,dnodeNumbers):
count=0
while count < 5:
tdSql.query("show dnodes")
# tdLog.debug(tdSql.queryResult)
status=0
for i in range(dnodeNumbers):
if tdSql.queryResult[i][4] == "ready":
status+=1
tdLog.debug(status)
if status == dnodeNumbers:
tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %dnodeNumbers)
break
count+=1
time.sleep(1)
else:
tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%dnodeNumbers)
def five_dnode_three_mnode(self,dnodenumber):
tdSql.query("show dnodes;")
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(4,1,'%s:6430'%self.host)
tdSql.checkData(0,4,'ready')
tdSql.checkData(4,4,'ready')
self.check_dnodes_status(5)
tdSql.query("show mnodes;")
tdLog.debug(self.host)
tdSql.checkRows(1)
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(0,2,'leader')
@ -262,26 +239,32 @@ class TDTestCase:
tdSql.error("create mnode on dnode 2")
tdSql.query("show dnodes;")
tdLog.debug(tdSql.queryResult)
# tdLog.debug(tdSql.queryResult)
tdLog.debug("stop and follower of mnode")
self.TDDnodes.stoptaosd(2)
tdDnodes=cluster.dnodes
# tdLog.debug(tdDnodes[0])
tdDnodes[1].stoptaosd()
self.check3mnode2off()
self.TDDnodes.starttaosd(2)
tdDnodes[1].starttaosd()
self.check3mnode()
self.TDDnodes.stoptaosd(3)
tdDnodes[2].stoptaosd()
self.check3mnode3off()
self.TDDnodes.starttaosd(3)
tdDnodes[2].starttaosd()
self.check3mnode()
self.TDDnodes.stoptaosd(1)
tdDnodes[0].stoptaosd()
self.check3mnode1off()
self.TDDnodes.starttaosd(1)
tdDnodes[0].starttaosd()
self.check3mnode()
self.check3mnode()
stopcount =0
while stopcount <= 2:
for i in range(dnodenumber):
self.TDDnodes.stoptaosd(i+1)
tdDnodes[i].stoptaosd()
# if i == 1 :
# self.check3mnode2off()
# elif i == 2 :
@ -289,22 +272,12 @@ class TDTestCase:
# elif i == 0:
# self.check3mnode1off()
self.TDDnodes.starttaosd(i+1)
tdDnodes[i].starttaosd()
# self.check3mnode()
stopcount+=1
self.check3mnode()
def getConnection(self, dnode):
host = dnode.cfgDict["fqdn"]
port = dnode.cfgDict["serverPort"]
config_dir = dnode.cfgDir
return taos.connect(host=host, port=int(port), config=config_dir)
def run(self):
# tdLog.debug(self.master_dnode.cfgDict)
self.buildcluster(5)
self.five_dnode_three_mnode(5)
def stop(self):

182
tests/system-test/7-tmq/tmqCheckData.py Normal file
View File

@ -0,0 +1,182 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
'dropFlag': 1,
'event': '',
'vgroups': 4,
'stbName': 'stb',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'ctbPrefix': 'ctb',
'ctbNum': 1,
'rowsPerTbl': 10000,
'batchNum': 10,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 10,
'showMsg': 1,
'showRow': 1}
topicNameList = ['topic1', 'topic2', 'topic3']
expectRowsList = []
tmqCom.initConsumerTable()
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
tdLog.info("create stb")
tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
tdLog.info("create ctb")
tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
tdLog.info("insert data")
tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
tdLog.info("create topics from stb with filter")
queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
expectRowsList.append(tdSql.getRows())
# init consume info, and start tmq_sim, then check consume result
tdLog.info("insert consume info to consume processor")
consumerId = 0
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
topicList = topicNameList[0]
ifcheckdata = 1
ifManualCommit = 1
keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("wait the consume result")
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
if expectRowsList[0] != resultList[0]:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
tdLog.exit("0 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
expectRowsList.append(tdSql.getRows())
consumerId = 1
topicList = topicNameList[1]
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("wait the consume result")
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
if expectRowsList[1] != resultList[0]:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
tdLog.exit("1 tmq consume rows error!")
self.checkFileContent(consumerId, queryString)
# reinit consume info, and start tmq_sim, then check consume result
tmqCom.initConsumerTable()
queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137)
sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
expectRowsList.append(tdSql.getRows())
consumerId = 2
topicList = topicNameList[2]
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("wait the consume result")
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
# if expectRowsList[2] != resultList[0]:
# tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
# tdLog.exit("2 tmq consume rows error!")
# self.checkFileContent(consumerId, queryString)
time.sleep(10)
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):
tdSql.prepare()
self.tmqCase1()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

5
tests/system-test/fulltest.sh
View File

@ -112,8 +112,8 @@ python3 ./test.py -f 2-query/twa.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py
#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py
#python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5
# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
python3 ./test.py -f 7-tmq/basic5.py
@ -132,3 +132,4 @@ python3 ./test.py -f 7-tmq/db.py
python3 ./test.py -f 7-tmq/tmqError.py
python3 ./test.py -f 7-tmq/schema.py
python3 ./test.py -f 7-tmq/stbFilter.py
python3 ./test.py -f 7-tmq/tmqCheckData.py

44
tests/system-test/test.py
View File

@ -28,6 +28,7 @@ sys.path.append("../pytest")
from util.log import *
from util.dnodes import *
from util.cases import *
from util.cluster import *
import taos
@ -58,10 +59,11 @@ if __name__ == "__main__":
logSql = True
stop = 0
restart = False
dnodeNums = 1
updateCfgDict = {}
execCmd = ""
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd'])
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums'])
for key, value in opts:
if key in ['-h', '--help']:
tdLog.printNoPrefix(
@ -77,6 +79,8 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-d update cfg dict, base64 json str')
tdLog.printNoPrefix('-k not kill valgrind processer')
tdLog.printNoPrefix('-e eval str to run')
tdLog.printNoPrefix('-N create dnodes numbers clusters')
sys.exit(0)
if key in ['-r', '--restart']:
@ -126,6 +130,9 @@ if __name__ == "__main__":
print('updateCfgDict convert fail.')
sys.exit(0)
if key in ['-N', '--dnodeNums']:
dnodeNums = value
if not execCmd == "":
tdDnodes.init(deployPath)
print(execCmd)
@ -232,10 +239,35 @@ if __name__ == "__main__":
updateCfgDict = ucase.updatecfgDict
except:
pass
tdDnodes.deploy(1,updateCfgDict)
tdDnodes.start(1)
if dnodeNums == 1 :
tdDnodes.deploy(1,updateCfgDict)
tdDnodes.start(1)
tdCases.logSql(logSql)
else :
print("start cluster and dnodes number")
dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True)
tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.stopAll()
for dnode in tdDnodes.dnodes:
tdDnodes.deploy(dnode.index,{})
for dnode in tdDnodes.dnodes:
tdDnodes.starttaosd(dnode.index)
tdCases.logSql(logSql)
conn = taos.connect(
host,
config=tdDnodes.getSimCfgPath())
print(tdDnodes.getSimCfgPath(),host)
cluster.create_dnode(conn)
try:
if cluster.check_dnode(conn) :
print("check dnode ready")
except Exception as r:
print(r)
tdCases.logSql(logSql)
if testCluster:
tdLog.info("Procedures for testing cluster")
@ -248,10 +280,12 @@ if __name__ == "__main__":
conn = taos.connect(
host,
config=tdDnodes.getSimCfgPath())
if fileName == "all":
tdCases.runAllLinux(conn)
else:
tdCases.runOneLinux(conn, fileName)
if restart:
if fileName == "all":
tdLog.info("not need to query ")

166
tests/test/c/tmqSim.c
View File

@ -24,6 +24,8 @@
#include "taos.h"
#include "taoserror.h"
#include "tlog.h"
#include "taosdef.h"
#include "types.h"
#define GREEN "\033[1;32m"
#define NC "\033[0m"
@ -49,6 +51,7 @@ typedef struct {
// char autoCommit[8]; // true, false
// char autoOffsetRest[16]; // none, earliest, latest
TdFilePtr pConsumeRowsFile;
int32_t ifCheckData;
int64_t expectMsgCnt;
@ -129,7 +132,6 @@ void initLogFile() {
char tmpString[128];
sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString));
// sprintf(filename, "%s/../log/tmqlog.txt", configDir);
#ifdef WINDOWS
for (int i = 2; i < sizeof(filename); i++) {
if (filename[i] == ':') filename[i] = '-';
@ -296,35 +298,165 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) {
return 0;
}
static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
//if (shell.args.is_raw_time) {
// sprintf(buf, "%" PRId64, val);
// return buf;
//}
time_t tt;
int32_t ms = 0;
if (precision == TSDB_TIME_PRECISION_NANO) {
tt = (time_t)(val / 1000000000);
ms = val % 1000000000;
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
tt = (time_t)(val / 1000000);
ms = val % 1000000;
} else {
tt = (time_t)(val / 1000);
ms = val % 1000;
}
/*
comment out as it make testcases like select_with_tags.sim fail.
but in windows, this may cause the call to localtime crash if tt < 0,
need to find a better solution.
if (tt < 0) {
tt = 0;
}
*/
#ifdef WINDOWS
if (tt < 0) tt = 0;
#endif
if (tt <= 0 && ms < 0) {
tt--;
if (precision == TSDB_TIME_PRECISION_NANO) {
ms += 1000000000;
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
ms += 1000000;
} else {
ms += 1000;
}
}
struct tm *ptm = taosLocalTime(&tt, NULL);
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", ms);
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
sprintf(buf + pos, ".%06d", ms);
} else {
sprintf(buf + pos, ".%03d", ms);
}
return buf;
}
static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) {
if (val == NULL) {
taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
return;
}
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
taosFprintfFile(pFile, "%d", *((int8_t *)val));
break;
case TSDB_DATA_TYPE_UTINYINT:
taosFprintfFile(pFile, "%u", *((uint8_t *)val));
break;
case TSDB_DATA_TYPE_SMALLINT:
taosFprintfFile(pFile, "%d", *((int16_t *)val));
break;
case TSDB_DATA_TYPE_USMALLINT:
taosFprintfFile(pFile, "%u", *((uint16_t *)val));
break;
case TSDB_DATA_TYPE_INT:
taosFprintfFile(pFile, "%d", *((int32_t *)val));
break;
case TSDB_DATA_TYPE_UINT:
taosFprintfFile(pFile, "%u", *((uint32_t *)val));
break;
case TSDB_DATA_TYPE_BIGINT:
taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
break;
case TSDB_DATA_TYPE_UBIGINT:
taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
if (n > TMAX(25, length)) {
taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val));
} else {
taosFprintfFile(pFile, "%s", buf);
}
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
memcpy(buf, val, length);
buf[length] = 0;
taosFprintfFile(pFile, "\'%s\'", buf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
shellFormatTimestamp(buf, *(int64_t *)val, precision);
taosFprintfFile(pFile, "'%s'", buf);
break;
default:
break;
}
}
static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, int32_t precision) {
for (int32_t i = 0; i < num_fields; i++) {
if (i > 0) {
taosFprintfFile(pFile, "\n");
}
shellDumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
}
taosFprintfFile(pFile, "\n");
}
static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) {
char buf[1024];
int32_t totalRows = 0;
// printf("topic: %s\n", tmq_get_topic_name(msg));
int32_t vgroupId = tmq_get_vgroup_id(msg);
const char* dbName = tmq_get_db_name(msg);
taosFprintfFile(g_fp, "msg index:%" PRId64 ", consumerId: %d\n", msgIndex, pInfo->consumerId);
// taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId,
// tmq_get_table_name(msg));
taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), vgroupId);
taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex);
taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId);
while (1) {
TAOS_ROW row = taos_fetch_row(msg);
if (row == NULL) break;
TAOS_FIELD* fields = taos_fetch_fields(msg);
TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg);
int32_t* length = taos_fetch_lengths(msg);
int32_t precision = taos_result_precision(msg);
const char* tbName = tmq_get_table_name(msg);
dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision);
taos_print_row(buf, row, fields, numOfFields);
const char* tbName = tmq_get_table_name(msg);
if (0 != g_stConfInfo.showRowFlag) {
taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf);
if (0 != g_stConfInfo.saveRowFlag) {
saveConsumeContentToTbl(pInfo, buf);
}
//if (0 != g_stConfInfo.saveRowFlag) {
// saveConsumeContentToTbl(pInfo, buf);
//}
}
totalRows++;
@ -464,6 +596,18 @@ void loop_consume(SThreadInfo* pInfo) {
pInfo->ts = taosGetTimestampMs();
if (pInfo->ifCheckData) {
char filename[256] = {0};
char tmpString[128];
//sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, getCurrentTimeString(tmpString));
sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId);
pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);
if (pInfo->pConsumeRowsFile == NULL) {
taosFprintfFile(g_fp, "%s create file fail for save rows content\n", getCurrentTimeString(tmpString));
return;
}
}
while (running) {
TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
if (tmqMsg) {

62
tools/shell/src/shellCommand.c
View File

@ -21,6 +21,7 @@
#define UP 3
#define DOWN 4
#define PSIZE shell.info.promptSize
#define SHELL_INPUT_MAX_COMMAND_SIZE 10000
typedef struct {
char *buffer;
@ -227,6 +228,7 @@ void shellPrintChar(char c, int32_t times) {
}
void shellPositionCursor(int32_t step, int32_t direction) {
#ifndef WINDOWS
if (step > 0) {
if (direction == LEFT) {
fprintf(stdout, "\033[%dD", step);
@ -239,6 +241,7 @@ void shellPositionCursor(int32_t step, int32_t direction) {
}
fflush(stdout);
}
#endif
}
void shellUpdateBuffer(SShellCmd *cmd) {
@ -330,10 +333,14 @@ void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) {
int32_t command_x = ecmd_pos / ws_col;
shellPositionCursor(cursor_y, LEFT);
shellPositionCursor(command_x - cursor_x, DOWN);
#ifndef WINDOWS
fprintf(stdout, "\033[2K");
#endif
for (int32_t i = 0; i < command_x; i++) {
shellPositionCursor(1, UP);
#ifndef WINDOWS
fprintf(stdout, "\033[2K");
#endif
}
fflush(stdout);
}
@ -394,6 +401,41 @@ void shellShowOnScreen(SShellCmd *cmd) {
fflush(stdout);
}
char taosGetConsoleChar() {
#ifdef WINDOWS
static void *console = NULL;
if (console == NULL) {
console = GetStdHandle(STD_INPUT_HANDLE);
}
static TdWchar buf[SHELL_INPUT_MAX_COMMAND_SIZE];
static char mbStr[5];
static unsigned long bufLen = 0;
static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0;
if (bufLen == 0) {
ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL);
bufIndex = 0;
}
if (mbStrLen == 0){
if (buf[bufIndex] == '\r') {
bufIndex++;
}
mbStrLen = WideCharToMultiByte(CP_UTF8, 0, &buf[bufIndex], 1, mbStr, sizeof(mbStr), NULL, NULL);
mbStrIndex = 0;
bufIndex++;
}
mbStrIndex++;
if (mbStrIndex == mbStrLen) {
mbStrLen = 0;
if (bufIndex == bufLen) {
bufLen = 0;
}
}
return mbStr[mbStrIndex-1];
#else
return (char)getchar(); // getchar() return an 'int32_t' value
#endif
}
int32_t shellReadCommand(char *command) {
SShellHistory *pHistory = &shell.history;
SShellCmd cmd = {0};
@ -407,7 +449,7 @@ int32_t shellReadCommand(char *command) {
// Read input.
char c;
while (1) {
c = (char)getchar(); // getchar() return an 'int32_t' value
c = taosGetConsoleChar();
if (c == EOF) {
return c;
@ -417,7 +459,7 @@ int32_t shellReadCommand(char *command) {
int32_t count = shellCountPrefixOnes(c);
utf8_array[0] = c;
for (int32_t k = 1; k < count; k++) {
c = (char)getchar();
c = taosGetConsoleChar();
utf8_array[k] = c;
}
shellInsertChar(&cmd, utf8_array, count);
@ -472,10 +514,10 @@ int32_t shellReadCommand(char *command) {
break;
}
} else if (c == '\033') {
c = (char)getchar();
c = taosGetConsoleChar();
switch (c) {
case '[':
c = (char)getchar();
c = taosGetConsoleChar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != pHistory->hstart) {
@ -502,35 +544,35 @@ int32_t shellReadCommand(char *command) {
shellMoveCursorLeft(&cmd);
break;
case '1':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// Home key
shellPositionCursorHome(&cmd);
}
break;
case '2':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// Delete key
shellDeleteChar(&cmd);
}
break;
case '4':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// End key
shellPositionCursorEnd(&cmd);
}
break;
case '5':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = (char)getchar()) == '~') {
if ((c = taosGetConsoleChar()) == '~') {
// Page down key
}
break;

4
tools/shell/src/shellEngine.c
View File

@ -393,15 +393,11 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
break;
}
int w = 0;
#ifdef WINDOWS
w = bytes;
#else
if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
w = bytes;
}else{
w = taosWcharWidth(wc);
}
#endif
pos += bytes;
if (w <= 0) {

2
tools/taos-tools

@ -1 +1 @@
Subproject commit d3c29fb492514cbaf08cb533976121bff5d94dea
Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd