Merge branch '3.0' into feat/compress_3.0
This commit is contained in:
Yihao Deng
commit
ea9385912f
|
|
@ -952,7 +952,7 @@ FIRST(expr)
|
|||
|
||||
**More explanation**:
|
||||
|
||||
- FIRST(\*) can be used to get the first non-null value of all columns
|
||||
- FIRST(\*) can be used to get the first non-null value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- NULL will be returned if all the values of the specified column are all NULL
|
||||
- A result will NOT be returned if all the columns in the result set are all NULL
|
||||
|
||||
|
|
@ -1014,7 +1014,7 @@ LAST(expr)
|
|||
|
||||
**More explanation**:
|
||||
|
||||
- LAST(\*) can be used to get the last non-NULL value of all columns
|
||||
- LAST(\*) can be used to get the last non-NULL value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned.
|
||||
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
|
||||
|
||||
|
|
@ -1035,6 +1035,7 @@ LAST_ROW(expr)
|
|||
|
||||
**More explanations**:
|
||||
|
||||
- LAST_ROW(\*) can be used to get the last value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST_ROW(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
|
||||
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
|
||||
- Can't be used with `INTERVAL`.
|
||||
|
||||
|
|
|
|||
|
|
@ -231,6 +231,16 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo
|
|||
| Default Value | 0 |
|
||||
| Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. |
|
||||
|
||||
### multiResultFunctionStarReturnTags
|
||||
|
||||
| Attribute | Description |
|
||||
| ------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| Applicable | Client only |
|
||||
| Meaning | When querying a super table, whether last(\*)/last_row(\*)/first(\*) returns tags is affected by this parameter. When querying a normal table or subtable, this parameter has no effect. |
|
||||
| Value Range | 0: do not return tags, 1: return tags |
|
||||
| Default Value | 0 |
|
||||
| Notes | When this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the columns of the super table; When it is 1, return the columns and tags of the super table. |
|
||||
|
||||
## Locale Parameters
|
||||
|
||||
### timezone
|
||||
|
|
|
|||
|
|
@ -954,7 +954,7 @@ FIRST(expr)
|
|||
|
||||
**使用说明**:
|
||||
|
||||
- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；
|
||||
- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，FIRST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；
|
||||
- 如果结果集中所有列全部为 NULL 值，则不返回结果。
|
||||
|
||||
|
|
@ -1006,7 +1006,7 @@ LAST(expr)
|
|||
|
||||
**使用说明**:
|
||||
|
||||
- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；
|
||||
- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。
|
||||
- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。
|
||||
|
||||
|
|
@ -1026,7 +1026,7 @@ LAST_ROW(expr)
|
|||
**适用于**：表和超级表。
|
||||
|
||||
**使用说明**：
|
||||
|
||||
- 如果要返回各个列的最后一条记录（时间戳最大），可以使用 LAST_ROW(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST_ROW(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。
|
||||
- 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。
|
||||
- 不能与 INTERVAL 一起使用。
|
||||
|
||||
|
|
|
|||
|
|
@ -230,6 +230,16 @@ taos -C
|
|||
| 缺省值 | 1 |
|
||||
| 补充说明 | 该参数设置为 1 时，如果查询中含有 GROUP BY，PARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果 |
|
||||
|
||||
### multiResultFunctionStarReturnTags
|
||||
|
||||
| 属性 | 说明 |
|
||||
| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 适用范围 | 仅客户端适用 |
|
||||
| 含义 | 查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响。 |
|
||||
| 取值范围 | 0：不返回标签列，1：返回标签列 |
|
||||
| 缺省值 | 0 |
|
||||
| 补充说明 | 该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列 |
|
||||
|
||||
## 区域相关
|
||||
|
||||
### timezone
|
||||
|
|
|
|||
|
|
@ -33,10 +33,12 @@ extern int32_t tsS3UploadDelaySec;
|
|||
|
||||
int32_t s3Init();
|
||||
void s3CleanUp();
|
||||
int32_t s3CheckCfg();
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object);
|
||||
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
|
||||
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
|
||||
void s3DeleteObjectsByPrefix(const char *prefix);
|
||||
void s3DeleteObjects(const char *object_name[], int nobject);
|
||||
int32_t s3DeleteObjects(const char *object_name[], int nobject);
|
||||
bool s3Exists(const char *object_name);
|
||||
bool s3Get(const char *object_name, const char *path);
|
||||
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
|
||||
|
|
@ -45,6 +47,8 @@ void s3EvictCache(const char *path, long object_size);
|
|||
long s3Size(const char *object_name);
|
||||
int32_t s3GetObjectToFile(const char *object_name, char *fileName);
|
||||
|
||||
#define S3_DATA_CHUNK_PAGES (256 * 1024 * 1024)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -116,9 +116,9 @@ extern int32_t tsMonitorIntervalForBasic;
|
|||
extern bool tsMonitorForceV2;
|
||||
|
||||
// audit
|
||||
extern bool tsEnableAudit;
|
||||
extern bool tsEnableAuditCreateTable;
|
||||
extern int32_t tsAuditInterval;
|
||||
extern bool tsEnableAudit;
|
||||
extern bool tsEnableAuditCreateTable;
|
||||
extern int32_t tsAuditInterval;
|
||||
|
||||
// telem
|
||||
extern bool tsEnableTelem;
|
||||
|
|
@ -126,9 +126,9 @@ extern int32_t tsTelemInterval;
|
|||
extern char tsTelemServer[];
|
||||
extern uint16_t tsTelemPort;
|
||||
extern bool tsEnableCrashReport;
|
||||
extern char * tsTelemUri;
|
||||
extern char * tsClientCrashReportUri;
|
||||
extern char * tsSvrCrashReportUri;
|
||||
extern char *tsTelemUri;
|
||||
extern char *tsClientCrashReportUri;
|
||||
extern char *tsSvrCrashReportUri;
|
||||
|
||||
// query buffer management
|
||||
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
|
||||
|
|
@ -158,6 +158,7 @@ extern int32_t tsMetaCacheMaxSize;
|
|||
extern int32_t tsSlowLogThreshold;
|
||||
extern int32_t tsSlowLogScope;
|
||||
extern int32_t tsTimeSeriesThreshold;
|
||||
extern bool tsMultiResultFunctionStarReturnTags;
|
||||
|
||||
// client
|
||||
extern int32_t tsMinSlidingTime;
|
||||
|
|
@ -215,6 +216,8 @@ extern int32_t tsTtlUnit;
|
|||
extern int32_t tsTtlPushIntervalSec;
|
||||
extern int32_t tsTtlBatchDropNum;
|
||||
extern int32_t tsTrimVDbIntervalSec;
|
||||
extern int32_t tsS3MigrateIntervalSec;
|
||||
extern bool tsS3MigrateEnabled;
|
||||
extern int32_t tsGrantHBInterval;
|
||||
extern int32_t tsUptimeInterval;
|
||||
|
||||
|
|
|
|||
|
|
@ -304,7 +304,8 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_GRANT_STMT,
|
||||
QUERY_NODE_REVOKE_STMT,
|
||||
QUERY_NODE_ALTER_CLUSTER_STMT,
|
||||
// placeholder for [153, 180]
|
||||
QUERY_NODE_S3MIGRATE_DATABASE_STMT,
|
||||
// placeholder for [154, 180]
|
||||
QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
|
||||
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
|
||||
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
|
||||
|
|
@ -1231,6 +1232,9 @@ typedef struct {
|
|||
int32_t sstTrigger;
|
||||
int16_t hashPrefix;
|
||||
int16_t hashSuffix;
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
int32_t tsdbPageSize;
|
||||
int32_t sqlLen;
|
||||
char* sql;
|
||||
|
|
@ -1261,6 +1265,8 @@ typedef struct {
|
|||
int32_t minRows;
|
||||
int32_t walRetentionPeriod;
|
||||
int32_t walRetentionSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
int32_t sqlLen;
|
||||
char* sql;
|
||||
int8_t withArbitrator;
|
||||
|
|
@ -1341,6 +1347,20 @@ typedef struct {
|
|||
int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
|
||||
int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
char db[TSDB_DB_FNAME_LEN];
|
||||
} SS3MigrateDbReq;
|
||||
|
||||
int32_t tSerializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
|
||||
int32_t tDeserializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int32_t timestamp;
|
||||
} SVS3MigrateDbReq;
|
||||
|
||||
int32_t tSerializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
|
||||
int32_t tDeserializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int32_t timestampSec;
|
||||
int32_t ttlDropMaxCount;
|
||||
|
|
@ -1377,6 +1397,9 @@ typedef struct {
|
|||
int8_t replications;
|
||||
int8_t strict;
|
||||
int8_t cacheLast;
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
int32_t tsdbPageSize;
|
||||
int32_t walRetentionPeriod;
|
||||
int32_t walRollPeriod;
|
||||
|
|
@ -1778,7 +1801,10 @@ typedef struct {
|
|||
int16_t hashPrefix;
|
||||
int16_t hashSuffix;
|
||||
int32_t tsdbPageSize;
|
||||
int64_t reserved[8];
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
int64_t reserved[6];
|
||||
int8_t learnerReplica;
|
||||
int8_t learnerSelfIndex;
|
||||
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
|
||||
|
|
@ -1866,13 +1892,15 @@ typedef struct {
|
|||
int8_t walLevel;
|
||||
int8_t strict;
|
||||
int8_t cacheLast;
|
||||
int64_t reserved[8];
|
||||
int64_t reserved[7];
|
||||
// 1st modification
|
||||
int16_t sttTrigger;
|
||||
int32_t minRows;
|
||||
// 2nd modification
|
||||
int32_t walRetentionPeriod;
|
||||
int32_t walRetentionSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
} SAlterVnodeConfigReq;
|
||||
|
||||
int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq);
|
||||
|
|
|
|||
|
|
@ -220,6 +220,8 @@
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
|
||||
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
|
||||
|
||||
|
|
@ -272,6 +274,7 @@
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
|
||||
|
|
|
|||
|
|
@ -84,295 +84,299 @@
|
|||
#define TK_USE 66
|
||||
#define TK_FLUSH 67
|
||||
#define TK_TRIM 68
|
||||
#define TK_COMPACT 69
|
||||
#define TK_IF 70
|
||||
#define TK_NOT 71
|
||||
#define TK_EXISTS 72
|
||||
#define TK_BUFFER 73
|
||||
#define TK_CACHEMODEL 74
|
||||
#define TK_CACHESIZE 75
|
||||
#define TK_COMP 76
|
||||
#define TK_DURATION 77
|
||||
#define TK_NK_VARIABLE 78
|
||||
#define TK_MAXROWS 79
|
||||
#define TK_MINROWS 80
|
||||
#define TK_KEEP 81
|
||||
#define TK_PAGES 82
|
||||
#define TK_PAGESIZE 83
|
||||
#define TK_TSDB_PAGESIZE 84
|
||||
#define TK_PRECISION 85
|
||||
#define TK_REPLICA 86
|
||||
#define TK_VGROUPS 87
|
||||
#define TK_SINGLE_STABLE 88
|
||||
#define TK_RETENTIONS 89
|
||||
#define TK_SCHEMALESS 90
|
||||
#define TK_WAL_LEVEL 91
|
||||
#define TK_WAL_FSYNC_PERIOD 92
|
||||
#define TK_WAL_RETENTION_PERIOD 93
|
||||
#define TK_WAL_RETENTION_SIZE 94
|
||||
#define TK_WAL_ROLL_PERIOD 95
|
||||
#define TK_WAL_SEGMENT_SIZE 96
|
||||
#define TK_STT_TRIGGER 97
|
||||
#define TK_TABLE_PREFIX 98
|
||||
#define TK_TABLE_SUFFIX 99
|
||||
#define TK_KEEP_TIME_OFFSET 100
|
||||
#define TK_NK_COLON 101
|
||||
#define TK_BWLIMIT 102
|
||||
#define TK_START 103
|
||||
#define TK_TIMESTAMP 104
|
||||
#define TK_END 105
|
||||
#define TK_TABLE 106
|
||||
#define TK_NK_LP 107
|
||||
#define TK_NK_RP 108
|
||||
#define TK_STABLE 109
|
||||
#define TK_COLUMN 110
|
||||
#define TK_MODIFY 111
|
||||
#define TK_RENAME 112
|
||||
#define TK_TAG 113
|
||||
#define TK_SET 114
|
||||
#define TK_NK_EQ 115
|
||||
#define TK_USING 116
|
||||
#define TK_TAGS 117
|
||||
#define TK_BOOL 118
|
||||
#define TK_TINYINT 119
|
||||
#define TK_SMALLINT 120
|
||||
#define TK_INT 121
|
||||
#define TK_INTEGER 122
|
||||
#define TK_BIGINT 123
|
||||
#define TK_FLOAT 124
|
||||
#define TK_DOUBLE 125
|
||||
#define TK_BINARY 126
|
||||
#define TK_NCHAR 127
|
||||
#define TK_UNSIGNED 128
|
||||
#define TK_JSON 129
|
||||
#define TK_VARCHAR 130
|
||||
#define TK_MEDIUMBLOB 131
|
||||
#define TK_BLOB 132
|
||||
#define TK_VARBINARY 133
|
||||
#define TK_GEOMETRY 134
|
||||
#define TK_DECIMAL 135
|
||||
#define TK_COMMENT 136
|
||||
#define TK_MAX_DELAY 137
|
||||
#define TK_WATERMARK 138
|
||||
#define TK_ROLLUP 139
|
||||
#define TK_TTL 140
|
||||
#define TK_SMA 141
|
||||
#define TK_DELETE_MARK 142
|
||||
#define TK_FIRST 143
|
||||
#define TK_LAST 144
|
||||
#define TK_SHOW 145
|
||||
#define TK_PRIVILEGES 146
|
||||
#define TK_DATABASES 147
|
||||
#define TK_TABLES 148
|
||||
#define TK_STABLES 149
|
||||
#define TK_MNODES 150
|
||||
#define TK_QNODES 151
|
||||
#define TK_ARBGROUPS 152
|
||||
#define TK_FUNCTIONS 153
|
||||
#define TK_INDEXES 154
|
||||
#define TK_ACCOUNTS 155
|
||||
#define TK_APPS 156
|
||||
#define TK_CONNECTIONS 157
|
||||
#define TK_LICENCES 158
|
||||
#define TK_GRANTS 159
|
||||
#define TK_FULL 160
|
||||
#define TK_LOGS 161
|
||||
#define TK_MACHINES 162
|
||||
#define TK_QUERIES 163
|
||||
#define TK_SCORES 164
|
||||
#define TK_TOPICS 165
|
||||
#define TK_VARIABLES 166
|
||||
#define TK_BNODES 167
|
||||
#define TK_SNODES 168
|
||||
#define TK_TRANSACTIONS 169
|
||||
#define TK_DISTRIBUTED 170
|
||||
#define TK_CONSUMERS 171
|
||||
#define TK_SUBSCRIPTIONS 172
|
||||
#define TK_VNODES 173
|
||||
#define TK_ALIVE 174
|
||||
#define TK_VIEWS 175
|
||||
#define TK_VIEW 176
|
||||
#define TK_COMPACTS 177
|
||||
#define TK_NORMAL 178
|
||||
#define TK_CHILD 179
|
||||
#define TK_LIKE 180
|
||||
#define TK_TBNAME 181
|
||||
#define TK_QTAGS 182
|
||||
#define TK_AS 183
|
||||
#define TK_SYSTEM 184
|
||||
#define TK_INDEX 185
|
||||
#define TK_FUNCTION 186
|
||||
#define TK_INTERVAL 187
|
||||
#define TK_COUNT 188
|
||||
#define TK_LAST_ROW 189
|
||||
#define TK_META 190
|
||||
#define TK_ONLY 191
|
||||
#define TK_TOPIC 192
|
||||
#define TK_CONSUMER 193
|
||||
#define TK_GROUP 194
|
||||
#define TK_DESC 195
|
||||
#define TK_DESCRIBE 196
|
||||
#define TK_RESET 197
|
||||
#define TK_QUERY 198
|
||||
#define TK_CACHE 199
|
||||
#define TK_EXPLAIN 200
|
||||
#define TK_ANALYZE 201
|
||||
#define TK_VERBOSE 202
|
||||
#define TK_NK_BOOL 203
|
||||
#define TK_RATIO 204
|
||||
#define TK_NK_FLOAT 205
|
||||
#define TK_OUTPUTTYPE 206
|
||||
#define TK_AGGREGATE 207
|
||||
#define TK_BUFSIZE 208
|
||||
#define TK_LANGUAGE 209
|
||||
#define TK_REPLACE 210
|
||||
#define TK_STREAM 211
|
||||
#define TK_INTO 212
|
||||
#define TK_PAUSE 213
|
||||
#define TK_RESUME 214
|
||||
#define TK_PRIMARY 215
|
||||
#define TK_KEY 216
|
||||
#define TK_TRIGGER 217
|
||||
#define TK_AT_ONCE 218
|
||||
#define TK_WINDOW_CLOSE 219
|
||||
#define TK_IGNORE 220
|
||||
#define TK_EXPIRED 221
|
||||
#define TK_FILL_HISTORY 222
|
||||
#define TK_UPDATE 223
|
||||
#define TK_SUBTABLE 224
|
||||
#define TK_UNTREATED 225
|
||||
#define TK_KILL 226
|
||||
#define TK_CONNECTION 227
|
||||
#define TK_TRANSACTION 228
|
||||
#define TK_BALANCE 229
|
||||
#define TK_VGROUP 230
|
||||
#define TK_LEADER 231
|
||||
#define TK_MERGE 232
|
||||
#define TK_REDISTRIBUTE 233
|
||||
#define TK_SPLIT 234
|
||||
#define TK_DELETE 235
|
||||
#define TK_INSERT 236
|
||||
#define TK_NK_BIN 237
|
||||
#define TK_NK_HEX 238
|
||||
#define TK_NULL 239
|
||||
#define TK_NK_QUESTION 240
|
||||
#define TK_NK_ALIAS 241
|
||||
#define TK_NK_ARROW 242
|
||||
#define TK_ROWTS 243
|
||||
#define TK_QSTART 244
|
||||
#define TK_QEND 245
|
||||
#define TK_QDURATION 246
|
||||
#define TK_WSTART 247
|
||||
#define TK_WEND 248
|
||||
#define TK_WDURATION 249
|
||||
#define TK_IROWTS 250
|
||||
#define TK_ISFILLED 251
|
||||
#define TK_CAST 252
|
||||
#define TK_NOW 253
|
||||
#define TK_TODAY 254
|
||||
#define TK_TIMEZONE 255
|
||||
#define TK_CLIENT_VERSION 256
|
||||
#define TK_SERVER_VERSION 257
|
||||
#define TK_SERVER_STATUS 258
|
||||
#define TK_CURRENT_USER 259
|
||||
#define TK_CASE 260
|
||||
#define TK_WHEN 261
|
||||
#define TK_THEN 262
|
||||
#define TK_ELSE 263
|
||||
#define TK_BETWEEN 264
|
||||
#define TK_IS 265
|
||||
#define TK_NK_LT 266
|
||||
#define TK_NK_GT 267
|
||||
#define TK_NK_LE 268
|
||||
#define TK_NK_GE 269
|
||||
#define TK_NK_NE 270
|
||||
#define TK_MATCH 271
|
||||
#define TK_NMATCH 272
|
||||
#define TK_CONTAINS 273
|
||||
#define TK_IN 274
|
||||
#define TK_JOIN 275
|
||||
#define TK_INNER 276
|
||||
#define TK_SELECT 277
|
||||
#define TK_NK_HINT 278
|
||||
#define TK_DISTINCT 279
|
||||
#define TK_WHERE 280
|
||||
#define TK_PARTITION 281
|
||||
#define TK_BY 282
|
||||
#define TK_SESSION 283
|
||||
#define TK_STATE_WINDOW 284
|
||||
#define TK_EVENT_WINDOW 285
|
||||
#define TK_COUNT_WINDOW 286
|
||||
#define TK_SLIDING 287
|
||||
#define TK_FILL 288
|
||||
#define TK_VALUE 289
|
||||
#define TK_VALUE_F 290
|
||||
#define TK_NONE 291
|
||||
#define TK_PREV 292
|
||||
#define TK_NULL_F 293
|
||||
#define TK_LINEAR 294
|
||||
#define TK_NEXT 295
|
||||
#define TK_HAVING 296
|
||||
#define TK_RANGE 297
|
||||
#define TK_EVERY 298
|
||||
#define TK_ORDER 299
|
||||
#define TK_SLIMIT 300
|
||||
#define TK_SOFFSET 301
|
||||
#define TK_LIMIT 302
|
||||
#define TK_OFFSET 303
|
||||
#define TK_ASC 304
|
||||
#define TK_NULLS 305
|
||||
#define TK_ABORT 306
|
||||
#define TK_AFTER 307
|
||||
#define TK_ATTACH 308
|
||||
#define TK_BEFORE 309
|
||||
#define TK_BEGIN 310
|
||||
#define TK_BITAND 311
|
||||
#define TK_BITNOT 312
|
||||
#define TK_BITOR 313
|
||||
#define TK_BLOCKS 314
|
||||
#define TK_CHANGE 315
|
||||
#define TK_COMMA 316
|
||||
#define TK_CONCAT 317
|
||||
#define TK_CONFLICT 318
|
||||
#define TK_COPY 319
|
||||
#define TK_DEFERRED 320
|
||||
#define TK_DELIMITERS 321
|
||||
#define TK_DETACH 322
|
||||
#define TK_DIVIDE 323
|
||||
#define TK_DOT 324
|
||||
#define TK_EACH 325
|
||||
#define TK_FAIL 326
|
||||
#define TK_FILE 327
|
||||
#define TK_FOR 328
|
||||
#define TK_GLOB 329
|
||||
#define TK_ID 330
|
||||
#define TK_IMMEDIATE 331
|
||||
#define TK_IMPORT 332
|
||||
#define TK_INITIALLY 333
|
||||
#define TK_INSTEAD 334
|
||||
#define TK_ISNULL 335
|
||||
#define TK_MODULES 336
|
||||
#define TK_NK_BITNOT 337
|
||||
#define TK_NK_SEMI 338
|
||||
#define TK_NOTNULL 339
|
||||
#define TK_OF 340
|
||||
#define TK_PLUS 341
|
||||
#define TK_PRIVILEGE 342
|
||||
#define TK_RAISE 343
|
||||
#define TK_RESTRICT 344
|
||||
#define TK_ROW 345
|
||||
#define TK_SEMI 346
|
||||
#define TK_STAR 347
|
||||
#define TK_STATEMENT 348
|
||||
#define TK_STRICT 349
|
||||
#define TK_STRING 350
|
||||
#define TK_TIMES 351
|
||||
#define TK_VALUES 352
|
||||
#define TK_VARIABLE 353
|
||||
#define TK_WAL 354
|
||||
#define TK_ENCODE 355
|
||||
#define TK_COMPRESS 356
|
||||
#define TK_LEVEL 357
|
||||
#define TK_S3MIGRATE 69
|
||||
#define TK_COMPACT 70
|
||||
#define TK_IF 71
|
||||
#define TK_NOT 72
|
||||
#define TK_EXISTS 73
|
||||
#define TK_BUFFER 74
|
||||
#define TK_CACHEMODEL 75
|
||||
#define TK_CACHESIZE 76
|
||||
#define TK_COMP 77
|
||||
#define TK_DURATION 78
|
||||
#define TK_NK_VARIABLE 79
|
||||
#define TK_MAXROWS 80
|
||||
#define TK_MINROWS 81
|
||||
#define TK_KEEP 82
|
||||
#define TK_PAGES 83
|
||||
#define TK_PAGESIZE 84
|
||||
#define TK_TSDB_PAGESIZE 85
|
||||
#define TK_PRECISION 86
|
||||
#define TK_REPLICA 87
|
||||
#define TK_VGROUPS 88
|
||||
#define TK_SINGLE_STABLE 89
|
||||
#define TK_RETENTIONS 90
|
||||
#define TK_SCHEMALESS 91
|
||||
#define TK_WAL_LEVEL 92
|
||||
#define TK_WAL_FSYNC_PERIOD 93
|
||||
#define TK_WAL_RETENTION_PERIOD 94
|
||||
#define TK_WAL_RETENTION_SIZE 95
|
||||
#define TK_WAL_ROLL_PERIOD 96
|
||||
#define TK_WAL_SEGMENT_SIZE 97
|
||||
#define TK_STT_TRIGGER 98
|
||||
#define TK_TABLE_PREFIX 99
|
||||
#define TK_TABLE_SUFFIX 100
|
||||
#define TK_S3_CHUNKSIZE 101
|
||||
#define TK_S3_KEEPLOCAL 102
|
||||
#define TK_S3_COMPACT 103
|
||||
#define TK_KEEP_TIME_OFFSET 104
|
||||
#define TK_NK_COLON 105
|
||||
#define TK_BWLIMIT 106
|
||||
#define TK_START 107
|
||||
#define TK_TIMESTAMP 108
|
||||
#define TK_END 109
|
||||
#define TK_TABLE 110
|
||||
#define TK_NK_LP 111
|
||||
#define TK_NK_RP 112
|
||||
#define TK_STABLE 113
|
||||
#define TK_COLUMN 114
|
||||
#define TK_MODIFY 115
|
||||
#define TK_RENAME 116
|
||||
#define TK_TAG 117
|
||||
#define TK_SET 118
|
||||
#define TK_NK_EQ 119
|
||||
#define TK_USING 120
|
||||
#define TK_TAGS 121
|
||||
#define TK_BOOL 122
|
||||
#define TK_TINYINT 123
|
||||
#define TK_SMALLINT 124
|
||||
#define TK_INT 125
|
||||
#define TK_INTEGER 126
|
||||
#define TK_BIGINT 127
|
||||
#define TK_FLOAT 128
|
||||
#define TK_DOUBLE 129
|
||||
#define TK_BINARY 130
|
||||
#define TK_NCHAR 131
|
||||
#define TK_UNSIGNED 132
|
||||
#define TK_JSON 133
|
||||
#define TK_VARCHAR 134
|
||||
#define TK_MEDIUMBLOB 135
|
||||
#define TK_BLOB 136
|
||||
#define TK_VARBINARY 137
|
||||
#define TK_GEOMETRY 138
|
||||
#define TK_DECIMAL 139
|
||||
#define TK_COMMENT 140
|
||||
#define TK_MAX_DELAY 141
|
||||
#define TK_WATERMARK 142
|
||||
#define TK_ROLLUP 143
|
||||
#define TK_TTL 144
|
||||
#define TK_SMA 145
|
||||
#define TK_DELETE_MARK 146
|
||||
#define TK_FIRST 147
|
||||
#define TK_LAST 148
|
||||
#define TK_SHOW 149
|
||||
#define TK_PRIVILEGES 150
|
||||
#define TK_DATABASES 151
|
||||
#define TK_TABLES 152
|
||||
#define TK_STABLES 153
|
||||
#define TK_MNODES 154
|
||||
#define TK_QNODES 155
|
||||
#define TK_ARBGROUPS 156
|
||||
#define TK_FUNCTIONS 157
|
||||
#define TK_INDEXES 158
|
||||
#define TK_ACCOUNTS 159
|
||||
#define TK_APPS 160
|
||||
#define TK_CONNECTIONS 161
|
||||
#define TK_LICENCES 162
|
||||
#define TK_GRANTS 163
|
||||
#define TK_FULL 164
|
||||
#define TK_LOGS 165
|
||||
#define TK_MACHINES 166
|
||||
#define TK_QUERIES 167
|
||||
#define TK_SCORES 168
|
||||
#define TK_TOPICS 169
|
||||
#define TK_VARIABLES 170
|
||||
#define TK_BNODES 171
|
||||
#define TK_SNODES 172
|
||||
#define TK_TRANSACTIONS 173
|
||||
#define TK_DISTRIBUTED 174
|
||||
#define TK_CONSUMERS 175
|
||||
#define TK_SUBSCRIPTIONS 176
|
||||
#define TK_VNODES 177
|
||||
#define TK_ALIVE 178
|
||||
#define TK_VIEWS 179
|
||||
#define TK_VIEW 180
|
||||
#define TK_COMPACTS 181
|
||||
#define TK_NORMAL 182
|
||||
#define TK_CHILD 183
|
||||
#define TK_LIKE 184
|
||||
#define TK_TBNAME 185
|
||||
#define TK_QTAGS 186
|
||||
#define TK_AS 187
|
||||
#define TK_SYSTEM 188
|
||||
#define TK_INDEX 189
|
||||
#define TK_FUNCTION 190
|
||||
#define TK_INTERVAL 191
|
||||
#define TK_COUNT 192
|
||||
#define TK_LAST_ROW 193
|
||||
#define TK_META 194
|
||||
#define TK_ONLY 195
|
||||
#define TK_TOPIC 196
|
||||
#define TK_CONSUMER 197
|
||||
#define TK_GROUP 198
|
||||
#define TK_DESC 199
|
||||
#define TK_DESCRIBE 200
|
||||
#define TK_RESET 201
|
||||
#define TK_QUERY 202
|
||||
#define TK_CACHE 203
|
||||
#define TK_EXPLAIN 204
|
||||
#define TK_ANALYZE 205
|
||||
#define TK_VERBOSE 206
|
||||
#define TK_NK_BOOL 207
|
||||
#define TK_RATIO 208
|
||||
#define TK_NK_FLOAT 209
|
||||
#define TK_OUTPUTTYPE 210
|
||||
#define TK_AGGREGATE 211
|
||||
#define TK_BUFSIZE 212
|
||||
#define TK_LANGUAGE 213
|
||||
#define TK_REPLACE 214
|
||||
#define TK_STREAM 215
|
||||
#define TK_INTO 216
|
||||
#define TK_PAUSE 217
|
||||
#define TK_RESUME 218
|
||||
#define TK_PRIMARY 219
|
||||
#define TK_KEY 220
|
||||
#define TK_TRIGGER 221
|
||||
#define TK_AT_ONCE 222
|
||||
#define TK_WINDOW_CLOSE 223
|
||||
#define TK_IGNORE 224
|
||||
#define TK_EXPIRED 225
|
||||
#define TK_FILL_HISTORY 226
|
||||
#define TK_UPDATE 227
|
||||
#define TK_SUBTABLE 228
|
||||
#define TK_UNTREATED 229
|
||||
#define TK_KILL 230
|
||||
#define TK_CONNECTION 231
|
||||
#define TK_TRANSACTION 232
|
||||
#define TK_BALANCE 233
|
||||
#define TK_VGROUP 234
|
||||
#define TK_LEADER 235
|
||||
#define TK_MERGE 236
|
||||
#define TK_REDISTRIBUTE 237
|
||||
#define TK_SPLIT 238
|
||||
#define TK_DELETE 239
|
||||
#define TK_INSERT 240
|
||||
#define TK_NK_BIN 241
|
||||
#define TK_NK_HEX 242
|
||||
#define TK_NULL 243
|
||||
#define TK_NK_QUESTION 244
|
||||
#define TK_NK_ALIAS 245
|
||||
#define TK_NK_ARROW 246
|
||||
#define TK_ROWTS 247
|
||||
#define TK_QSTART 248
|
||||
#define TK_QEND 249
|
||||
#define TK_QDURATION 250
|
||||
#define TK_WSTART 251
|
||||
#define TK_WEND 252
|
||||
#define TK_WDURATION 253
|
||||
#define TK_IROWTS 254
|
||||
#define TK_ISFILLED 255
|
||||
#define TK_CAST 256
|
||||
#define TK_NOW 257
|
||||
#define TK_TODAY 258
|
||||
#define TK_TIMEZONE 259
|
||||
#define TK_CLIENT_VERSION 260
|
||||
#define TK_SERVER_VERSION 261
|
||||
#define TK_SERVER_STATUS 262
|
||||
#define TK_CURRENT_USER 263
|
||||
#define TK_CASE 264
|
||||
#define TK_WHEN 265
|
||||
#define TK_THEN 266
|
||||
#define TK_ELSE 267
|
||||
#define TK_BETWEEN 268
|
||||
#define TK_IS 269
|
||||
#define TK_NK_LT 270
|
||||
#define TK_NK_GT 271
|
||||
#define TK_NK_LE 272
|
||||
#define TK_NK_GE 273
|
||||
#define TK_NK_NE 274
|
||||
#define TK_MATCH 275
|
||||
#define TK_NMATCH 276
|
||||
#define TK_CONTAINS 277
|
||||
#define TK_IN 278
|
||||
#define TK_JOIN 279
|
||||
#define TK_INNER 280
|
||||
#define TK_SELECT 281
|
||||
#define TK_NK_HINT 282
|
||||
#define TK_DISTINCT 283
|
||||
#define TK_WHERE 284
|
||||
#define TK_PARTITION 285
|
||||
#define TK_BY 286
|
||||
#define TK_SESSION 287
|
||||
#define TK_STATE_WINDOW 288
|
||||
#define TK_EVENT_WINDOW 289
|
||||
#define TK_COUNT_WINDOW 290
|
||||
#define TK_SLIDING 291
|
||||
#define TK_FILL 292
|
||||
#define TK_VALUE 293
|
||||
#define TK_VALUE_F 294
|
||||
#define TK_NONE 295
|
||||
#define TK_PREV 296
|
||||
#define TK_NULL_F 297
|
||||
#define TK_LINEAR 298
|
||||
#define TK_NEXT 299
|
||||
#define TK_HAVING 300
|
||||
#define TK_RANGE 301
|
||||
#define TK_EVERY 302
|
||||
#define TK_ORDER 303
|
||||
#define TK_SLIMIT 304
|
||||
#define TK_SOFFSET 305
|
||||
#define TK_LIMIT 306
|
||||
#define TK_OFFSET 307
|
||||
#define TK_ASC 308
|
||||
#define TK_NULLS 309
|
||||
#define TK_ABORT 310
|
||||
#define TK_AFTER 311
|
||||
#define TK_ATTACH 312
|
||||
#define TK_BEFORE 313
|
||||
#define TK_BEGIN 314
|
||||
#define TK_BITAND 315
|
||||
#define TK_BITNOT 316
|
||||
#define TK_BITOR 317
|
||||
#define TK_BLOCKS 318
|
||||
#define TK_CHANGE 319
|
||||
#define TK_COMMA 320
|
||||
#define TK_CONCAT 321
|
||||
#define TK_CONFLICT 322
|
||||
#define TK_COPY 323
|
||||
#define TK_DEFERRED 324
|
||||
#define TK_DELIMITERS 325
|
||||
#define TK_DETACH 326
|
||||
#define TK_DIVIDE 327
|
||||
#define TK_DOT 328
|
||||
#define TK_EACH 329
|
||||
#define TK_FAIL 330
|
||||
#define TK_FILE 331
|
||||
#define TK_FOR 332
|
||||
#define TK_GLOB 333
|
||||
#define TK_ID 334
|
||||
#define TK_IMMEDIATE 335
|
||||
#define TK_IMPORT 336
|
||||
#define TK_INITIALLY 337
|
||||
#define TK_INSTEAD 338
|
||||
#define TK_ISNULL 339
|
||||
#define TK_MODULES 340
|
||||
#define TK_NK_BITNOT 341
|
||||
#define TK_NK_SEMI 342
|
||||
#define TK_NOTNULL 343
|
||||
#define TK_OF 344
|
||||
#define TK_PLUS 345
|
||||
#define TK_PRIVILEGE 346
|
||||
#define TK_RAISE 347
|
||||
#define TK_RESTRICT 348
|
||||
#define TK_ROW 349
|
||||
#define TK_SEMI 350
|
||||
#define TK_STAR 351
|
||||
#define TK_STATEMENT 352
|
||||
#define TK_STRICT 353
|
||||
#define TK_STRING 354
|
||||
#define TK_TIMES 355
|
||||
#define TK_VALUES 356
|
||||
#define TK_VARIABLE 357
|
||||
#define TK_WAL 358
|
||||
#define TK_ENCODE 359
|
||||
#define TK_COMPRESS 360
|
||||
#define TK_LEVEL 361
|
||||
|
||||
#define TK_NK_SPACE 600
|
||||
#define TK_NK_COMMENT 601
|
||||
|
|
|
|||
|
|
@ -101,6 +101,10 @@ typedef struct SDatabaseOptions {
|
|||
int32_t sstTrigger;
|
||||
int32_t tablePrefix;
|
||||
int32_t tableSuffix;
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
SValueNode* s3KeepLocalStr;
|
||||
int8_t s3Compact;
|
||||
int8_t withArbitrator;
|
||||
} SDatabaseOptions;
|
||||
|
||||
|
|
@ -139,6 +143,11 @@ typedef struct STrimDatabaseStmt {
|
|||
int32_t maxSpeed;
|
||||
} STrimDatabaseStmt;
|
||||
|
||||
typedef struct SS3MigrateDatabaseStmt {
|
||||
ENodeType type;
|
||||
char dbName[TSDB_DB_NAME_LEN];
|
||||
} SS3MigrateDatabaseStmt;
|
||||
|
||||
typedef struct SCompactDatabaseStmt {
|
||||
ENodeType type;
|
||||
char dbName[TSDB_DB_NAME_LEN];
|
||||
|
|
|
|||
|
|
@ -51,11 +51,7 @@ typedef enum {
|
|||
CFG_DTYPE_TIMEZONE
|
||||
} ECfgDataType;
|
||||
|
||||
typedef enum {
|
||||
CFG_SCOPE_SERVER,
|
||||
CFG_SCOPE_CLIENT,
|
||||
CFG_SCOPE_BOTH
|
||||
} ECfgScopeType;
|
||||
typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType;
|
||||
|
||||
typedef enum {
|
||||
CFG_DYN_NONE = 0,
|
||||
|
|
@ -138,6 +134,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
|
|||
void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen);
|
||||
|
||||
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
|
||||
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump);
|
||||
|
||||
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl);
|
||||
|
||||
|
|
|
|||
|
|
@ -197,8 +197,8 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788
|
||||
// ACCOUNT is a 32 bit positive integer
|
||||
// this is the length of its string representation, including the terminator zero
|
||||
#define TSDB_ACCT_ID_LEN 11
|
||||
#define TSDB_NODE_ID_LEN 11
|
||||
#define TSDB_ACCT_ID_LEN 11
|
||||
#define TSDB_NODE_ID_LEN 11
|
||||
#define TSDB_VGROUP_ID_LEN 11
|
||||
|
||||
#define TSDB_MAX_COLUMNS 4096
|
||||
|
|
@ -293,8 +293,8 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_DNODE_CONFIG_LEN 128
|
||||
#define TSDB_DNODE_VALUE_LEN 256
|
||||
|
||||
#define TSDB_CLUSTER_VALUE_LEN 1000
|
||||
#define TSDB_GRANT_LOG_COL_LEN 15600
|
||||
#define TSDB_CLUSTER_VALUE_LEN 1000
|
||||
#define TSDB_GRANT_LOG_COL_LEN 15600
|
||||
|
||||
#define TSDB_ACTIVE_KEY_LEN 109
|
||||
#define TSDB_CONN_ACTIVE_KEY_LEN 255
|
||||
|
|
@ -416,6 +416,16 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
|
||||
#define TSDB_DEFAULT_HASH_SUFFIX 0
|
||||
|
||||
#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024)
|
||||
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
|
||||
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
|
||||
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
|
||||
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
|
||||
#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440)
|
||||
#define TSDB_MIN_S3_COMPACT 0
|
||||
#define TSDB_MAX_S3_COMPACT 1
|
||||
#define TSDB_DEFAULT_S3_COMPACT 0
|
||||
|
||||
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
|
||||
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600
|
||||
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600
|
||||
|
|
@ -530,12 +540,12 @@ enum {
|
|||
SND_WORKER_TYPE__UNIQUE,
|
||||
};
|
||||
|
||||
#define DEFAULT_HANDLE 0
|
||||
#define MNODE_HANDLE 1
|
||||
#define QNODE_HANDLE -1
|
||||
#define SNODE_HANDLE -2
|
||||
#define VNODE_HANDLE -3
|
||||
#define CLIENT_HANDLE -5
|
||||
#define DEFAULT_HANDLE 0
|
||||
#define MNODE_HANDLE 1
|
||||
#define QNODE_HANDLE -1
|
||||
#define SNODE_HANDLE -2
|
||||
#define VNODE_HANDLE -3
|
||||
#define CLIENT_HANDLE -5
|
||||
|
||||
#define TSDB_CONFIG_OPTION_LEN 32
|
||||
#define TSDB_CONFIG_VALUE_LEN 64
|
||||
|
|
@ -555,13 +565,13 @@ enum {
|
|||
// sort page size by default
|
||||
#define DEFAULT_PAGESIZE 4096
|
||||
|
||||
#define VNODE_TIMEOUT_SEC 60
|
||||
#define MNODE_TIMEOUT_SEC 60
|
||||
#define VNODE_TIMEOUT_SEC 60
|
||||
#define MNODE_TIMEOUT_SEC 60
|
||||
|
||||
#define MONITOR_TABLENAME_LEN 200
|
||||
#define MONITOR_TAG_NAME_LEN 100
|
||||
#define MONITOR_TAG_VALUE_LEN 300
|
||||
#define MONITOR_METRIC_NAME_LEN 100
|
||||
#define MONITOR_TABLENAME_LEN 200
|
||||
#define MONITOR_TAG_NAME_LEN 100
|
||||
#define MONITOR_TAG_VALUE_LEN 300
|
||||
#define MONITOR_METRIC_NAME_LEN 100
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -16,14 +16,15 @@ ENDIF ()
|
|||
IF (TD_STORAGE)
|
||||
ADD_DEFINITIONS(-D_STORAGE)
|
||||
TARGET_LINK_LIBRARIES(common PRIVATE storage)
|
||||
ENDIF ()
|
||||
|
||||
IF (TD_ENTERPRISE)
|
||||
IF(${BUILD_WITH_S3})
|
||||
add_definitions(-DUSE_S3)
|
||||
ELSEIF(${BUILD_WITH_COS})
|
||||
add_definitions(-DUSE_COS)
|
||||
ENDIF()
|
||||
|
||||
ENDIF ()
|
||||
ENDIF()
|
||||
|
||||
target_include_directories(
|
||||
common
|
||||
|
|
|
|||
|
|
@ -53,6 +53,100 @@ int32_t s3Init() { return 0; /*s3Begin();*/ }
|
|||
void s3CleanUp() { /*s3End();*/
|
||||
}
|
||||
|
||||
static int32_t s3ListBucket(char const *bucketname);
|
||||
|
||||
int32_t s3CheckCfg() {
|
||||
int32_t code = 0;
|
||||
|
||||
code = s3Begin();
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "failed to initialize s3.\n");
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
// test put
|
||||
char testdata[17] = "0123456789abcdef";
|
||||
const char *objectname[] = {"s3test.txt"};
|
||||
char path[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP);
|
||||
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]);
|
||||
} else {
|
||||
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]);
|
||||
}
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
|
||||
if (!fp) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
fprintf(stderr, "failed to open test file: %s.\n", path);
|
||||
// uError("ERROR: %s Failed to open %s", __func__, path);
|
||||
goto _exit;
|
||||
}
|
||||
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
fprintf(stderr, "failed to write test file: %s.\n", path);
|
||||
goto _exit;
|
||||
}
|
||||
if (taosFsyncFile(fp) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
fprintf(stderr, "failed to fsync test file: %s.\n", path);
|
||||
goto _exit;
|
||||
}
|
||||
taosCloseFile(&fp);
|
||||
|
||||
fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
|
||||
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
|
||||
goto _exit;
|
||||
}
|
||||
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
|
||||
|
||||
// list buckets
|
||||
fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName);
|
||||
code = s3ListBucket(tsS3BucketName);
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
|
||||
goto _exit;
|
||||
}
|
||||
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
|
||||
|
||||
// test range get
|
||||
uint8_t *pBlock = NULL;
|
||||
int c_offset = 10;
|
||||
int c_len = 6;
|
||||
|
||||
fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
|
||||
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
|
||||
goto _exit;
|
||||
}
|
||||
char buf[7] = {0};
|
||||
memcpy(buf, pBlock, c_len);
|
||||
taosMemoryFree(pBlock);
|
||||
fprintf(stderr, "object content: %s\n", buf);
|
||||
fprintf(stderr, "get object %s: success.\n\n", objectname[0]);
|
||||
|
||||
// delete test object
|
||||
fprintf(stderr, "start to delete object: %s.\n", objectname[0]);
|
||||
code = s3DeleteObjects(objectname, 1);
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
|
||||
goto _exit;
|
||||
}
|
||||
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
|
||||
|
||||
s3End();
|
||||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
|
||||
static int should_retry() {
|
||||
/*
|
||||
if (retriesG--) {
|
||||
|
|
@ -85,7 +179,7 @@ typedef struct {
|
|||
} TS3GetData;
|
||||
|
||||
typedef struct {
|
||||
char err_msg[128];
|
||||
char err_msg[512];
|
||||
S3Status status;
|
||||
uint64_t content_length;
|
||||
char *buf;
|
||||
|
|
@ -137,6 +231,30 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
|
|||
}
|
||||
}
|
||||
|
||||
static SArray *getListByPrefix(const char *prefix);
|
||||
static void s3FreeObjectKey(void *pItem);
|
||||
|
||||
static int32_t s3ListBucket(char const *bucketname) {
|
||||
int32_t code = 0;
|
||||
|
||||
SArray *objectArray = getListByPrefix("s3");
|
||||
if (objectArray == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char **object_name = TARRAY_DATA(objectArray);
|
||||
int size = TARRAY_SIZE(objectArray);
|
||||
|
||||
fprintf(stderr, "objects:\n");
|
||||
for (int i = 0; i < size; ++i) {
|
||||
fprintf(stderr, "%s\n", object_name[i]);
|
||||
}
|
||||
|
||||
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
typedef struct growbuffer {
|
||||
// The total number of bytes, and the start byte
|
||||
int size;
|
||||
|
|
@ -471,10 +589,11 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
|
|||
} while (S3_status_is_retryable(data->status) && should_retry());
|
||||
|
||||
if (data->status != S3StatusOK) {
|
||||
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
|
||||
// s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
|
||||
s3PrintError(NULL, __LINE__, __func__, data->status, data->err_msg);
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
} else if (data->contentLength) {
|
||||
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
|
||||
uError("%s Failed to put remaining %llu bytes", __func__, (unsigned long long)data->contentLength);
|
||||
code = TAOS_SYSTEM_ERROR(EIO);
|
||||
}
|
||||
|
||||
|
|
@ -830,6 +949,66 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
|
||||
int32_t code = 0;
|
||||
int32_t lmtime = 0;
|
||||
const char *filename = 0;
|
||||
uint64_t contentLength = 0;
|
||||
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
|
||||
const char *contentDispositionFilename = 0, *contentEncoding = 0;
|
||||
int64_t expires = -1;
|
||||
S3CannedAcl cannedAcl = S3CannedAclPrivate;
|
||||
int metaPropertiesCount = 0;
|
||||
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
|
||||
char useServerSideEncryption = 0;
|
||||
put_object_callback_data data = {0};
|
||||
|
||||
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
|
||||
return code;
|
||||
}
|
||||
|
||||
contentLength = size;
|
||||
|
||||
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
uError("ERROR: %s Failed to open file %s: ", __func__, file);
|
||||
return code;
|
||||
}
|
||||
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
|
||||
taosCloseFile(&data.infileFD);
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
return code;
|
||||
}
|
||||
|
||||
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
|
||||
contentLength;
|
||||
|
||||
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||
0, awsRegionG};
|
||||
|
||||
S3PutProperties putProperties = {contentType, md5,
|
||||
cacheControl, contentDispositionFilename,
|
||||
contentEncoding, expires,
|
||||
cannedAcl, metaPropertiesCount,
|
||||
metaProperties, useServerSideEncryption};
|
||||
|
||||
if (contentLength <= MULTIPART_CHUNK_SIZE) {
|
||||
code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
|
||||
} else {
|
||||
code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
|
||||
}
|
||||
|
||||
if (data.infileFD) {
|
||||
taosCloseFile(&data.infileFD);
|
||||
} else if (data.gb) {
|
||||
growbuffer_destroy(data.gb);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
typedef struct list_bucket_callback_data {
|
||||
char err_msg[512];
|
||||
S3Status status;
|
||||
|
|
@ -888,7 +1067,7 @@ static SArray *getListByPrefix(const char *prefix) {
|
|||
|
||||
const char *marker = 0, *delimiter = 0;
|
||||
int maxkeys = 0, allDetails = 0;
|
||||
list_bucket_callback_data data;
|
||||
list_bucket_callback_data data = {0};
|
||||
data.objectArray = taosArrayInit(32, sizeof(void *));
|
||||
if (!data.objectArray) {
|
||||
uError("%s: %s", __func__, "out of memoty");
|
||||
|
|
@ -918,14 +1097,17 @@ static SArray *getListByPrefix(const char *prefix) {
|
|||
return data.objectArray;
|
||||
}
|
||||
} else {
|
||||
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
|
||||
uError("failed to list with prefix %s: %s", prefix, S3_get_status_name(data.status));
|
||||
// s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
|
||||
}
|
||||
|
||||
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void s3DeleteObjects(const char *object_name[], int nobject) {
|
||||
int32_t s3DeleteObjects(const char *object_name[], int nobject) {
|
||||
int32_t code = 0;
|
||||
|
||||
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
|
||||
0, awsRegionG};
|
||||
S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
|
||||
|
|
@ -938,8 +1120,11 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
|
|||
|
||||
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
|
||||
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
|
||||
code = -1;
|
||||
}
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
void s3DeleteObjectsByPrefix(const char *prefix) {
|
||||
|
|
@ -991,12 +1176,12 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
|
|||
} while (S3_status_is_retryable(cbd.status) && should_retry());
|
||||
|
||||
if (cbd.status != S3StatusOK) {
|
||||
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
|
||||
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
|
||||
return TAOS_SYSTEM_ERROR(EIO);
|
||||
}
|
||||
|
||||
if (check && cbd.buf_pos != size) {
|
||||
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
|
||||
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
|
||||
return TAOS_SYSTEM_ERROR(EIO);
|
||||
}
|
||||
|
||||
|
|
@ -1233,7 +1418,7 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
|
|||
cos_pool_destroy(p);
|
||||
}
|
||||
|
||||
void s3DeleteObjects(const char *object_name[], int nobject) {
|
||||
int32_t s3DeleteObjects(const char *object_name[], int nobject) {
|
||||
cos_pool_t *p = NULL;
|
||||
int is_cname = 0;
|
||||
cos_string_t bucket;
|
||||
|
|
@ -1267,6 +1452,8 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
|
|||
} else {
|
||||
cos_warn_log("delete objects failed\n");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool s3Exists(const char *object_name) {
|
||||
|
|
@ -1535,8 +1722,9 @@ int32_t s3Init() { return 0; }
|
|||
void s3CleanUp() {}
|
||||
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
|
||||
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
|
||||
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }
|
||||
void s3DeleteObjectsByPrefix(const char *prefix) {}
|
||||
void s3DeleteObjects(const char *object_name[], int nobject) {}
|
||||
int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; }
|
||||
bool s3Exists(const char *object_name) { return false; }
|
||||
bool s3Get(const char *object_name, const char *path) { return false; }
|
||||
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
|
||||
// clang-format off
|
||||
static const SSysDbTableSchema dnodesSchema[] = {
|
||||
|
|
@ -117,6 +117,9 @@ static const SSysDbTableSchema userDBSchema[] = {
|
|||
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
|
||||
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
|
||||
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
|
||||
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ int32_t tsMetaCacheMaxSize = -1; // MB
|
|||
int32_t tsSlowLogThreshold = 3; // seconds
|
||||
int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL;
|
||||
int32_t tsTimeSeriesThreshold = 50;
|
||||
bool tsMultiResultFunctionStarReturnTags = false;
|
||||
|
||||
/*
|
||||
* denote if the server needs to compress response message at the application layer to client, including query rsp,
|
||||
|
|
@ -265,7 +266,9 @@ float tsSinkDataRate = 2.0;
|
|||
int32_t tsStreamNodeCheckInterval = 16;
|
||||
int32_t tsTtlUnit = 86400;
|
||||
int32_t tsTtlPushIntervalSec = 10;
|
||||
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
|
||||
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
|
||||
int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups
|
||||
bool tsS3MigrateEnabled = 1;
|
||||
int32_t tsGrantHBInterval = 60;
|
||||
int32_t tsUptimeInterval = 300; // seconds
|
||||
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
|
||||
|
|
@ -291,7 +294,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
|
|||
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
|
||||
int32_t tsS3BlockCacheSize = 16; // number of blocks
|
||||
int32_t tsS3PageCacheSize = 4096; // number of pages
|
||||
int32_t tsS3UploadDelaySec = 60 * 60 * 24;
|
||||
int32_t tsS3UploadDelaySec = 60;
|
||||
|
||||
bool tsExperimental = true;
|
||||
|
||||
|
|
@ -351,7 +354,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
|
|||
}
|
||||
if (tsS3BucketName[0] != '<') {
|
||||
#if defined(USE_COS) || defined(USE_S3)
|
||||
if (tsDiskCfgNum > 1) tsS3Enabled = true;
|
||||
#ifdef TD_ENTERPRISE
|
||||
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
|
||||
#endif
|
||||
tsS3StreamEnabled = true;
|
||||
#endif
|
||||
}
|
||||
|
|
@ -546,6 +551,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
|
||||
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
|
||||
|
||||
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -674,8 +681,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16,
|
||||
(TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2),
|
||||
CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
|
||||
if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
|
||||
|
|
@ -700,7 +707,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
|
||||
|
|
@ -745,6 +753,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
|
||||
0)
|
||||
return -1;
|
||||
if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
|
|
@ -763,8 +775,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
|
||||
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
|
||||
return -1;
|
||||
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
|
||||
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
|
|
@ -795,6 +806,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
|
||||
/*
|
||||
if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) {
|
||||
|
|
@ -804,10 +816,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
|
||||
0)
|
||||
return -1;
|
||||
*/
|
||||
if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
|
||||
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
|
||||
|
|
@ -1107,6 +1120,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
|
||||
|
||||
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
|
||||
|
||||
tsMultiResultFunctionStarReturnTags = cfgGetItem(pCfg, "multiResultFunctionStarReturnTags")->bval;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1247,8 +1262,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
|
||||
tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64;
|
||||
|
||||
tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
|
||||
tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
|
||||
// tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
|
||||
// tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
|
||||
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
|
||||
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
|
||||
|
||||
|
|
@ -1526,6 +1541,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
|
|||
{"ttlBatchDropNum", &tsTtlBatchDropNum},
|
||||
{"ttlFlushThreshold", &tsTtlFlushThreshold},
|
||||
{"ttlPushInterval", &tsTtlPushIntervalSec},
|
||||
{"s3MigrateIntervalSec", &tsS3MigrateIntervalSec},
|
||||
{"s3MigrateEnabled", &tsS3MigrateEnabled},
|
||||
//{"s3BlockSize", &tsS3BlockSize},
|
||||
{"s3BlockCacheSize", &tsS3BlockCacheSize},
|
||||
{"s3PageCacheSize", &tsS3PageCacheSize},
|
||||
|
|
@ -1748,7 +1765,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
|
|||
{"shellActivityTimer", &tsShellActivityTimer},
|
||||
{"slowLogThreshold", &tsSlowLogThreshold},
|
||||
{"useAdapter", &tsUseAdapter},
|
||||
{"experimental", &tsExperimental}};
|
||||
{"experimental", &tsExperimental},
|
||||
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags} };
|
||||
|
||||
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
|
||||
taosCfgSetOption(options, tListLen(options), pItem, false);
|
||||
|
|
|
|||
|
|
@ -1873,7 +1873,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
char *tb = taosHashIterate(pRsp->readTbs, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1888,7 +1888,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
tb = taosHashIterate(pRsp->writeTbs, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1903,7 +1903,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
tb = taosHashIterate(pRsp->alterTbs, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1918,7 +1918,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
tb = taosHashIterate(pRsp->readViews, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1933,7 +1933,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
tb = taosHashIterate(pRsp->writeViews, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1948,7 +1948,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
tb = taosHashIterate(pRsp->alterViews, NULL);
|
||||
while (tb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(tb, &keyLen);
|
||||
void *key = taosHashGetKey(tb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -1963,7 +1963,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
|
||||
while (useDb != NULL) {
|
||||
size_t keyLen = 0;
|
||||
void * key = taosHashGetKey(useDb, &keyLen);
|
||||
void *key = taosHashGetKey(useDb, &keyLen);
|
||||
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
|
||||
if (tEncodeCStr(pEncoder, key) < 0) return -1;
|
||||
|
||||
|
|
@ -3044,6 +3044,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
|
|||
}
|
||||
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
|
||||
|
||||
ENCODESQL();
|
||||
|
||||
|
|
@ -3116,6 +3119,15 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
|
|||
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
|
||||
}
|
||||
|
||||
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
|
||||
}
|
||||
|
||||
DECODESQL();
|
||||
|
||||
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
|
||||
|
|
@ -3161,6 +3173,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
|
|||
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
|
||||
|
||||
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
|
||||
|
||||
ENCODESQL();
|
||||
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
|
@ -3211,6 +3227,13 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
|
|||
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
|
||||
}
|
||||
|
||||
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
|
||||
}
|
||||
|
||||
DECODESQL();
|
||||
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
|
|
@ -3907,6 +3930,57 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
@ -4004,6 +4078,9 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
|
|||
if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1;
|
||||
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
|
||||
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
|
||||
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
|
||||
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
|
||||
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -4081,6 +4158,15 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
|
|||
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
|
||||
}
|
||||
|
||||
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (!tDecodeIsEnd(decoder)) {
|
||||
if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1;
|
||||
if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1;
|
||||
if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -5126,7 +5212,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
|
|||
if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1;
|
||||
if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
for (int32_t i = 0; i < 6; ++i) {
|
||||
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
|
||||
}
|
||||
if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1;
|
||||
|
|
@ -5137,6 +5223,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
|
|||
}
|
||||
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
||||
|
|
@ -5212,7 +5301,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
|
|||
if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1;
|
||||
if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1;
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
for (int32_t i = 0; i < 6; ++i) {
|
||||
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
|
||||
}
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
|
|
@ -5231,6 +5320,15 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
|
|||
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
|
||||
}
|
||||
|
||||
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
|
@ -5486,7 +5584,7 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
|
|||
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
for (int32_t i = 0; i < 7; ++i) {
|
||||
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
|
||||
}
|
||||
|
||||
|
|
@ -5497,6 +5595,10 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
|
|||
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
|
||||
|
||||
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -5522,7 +5624,7 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
|
|||
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
|
||||
for (int32_t i = 0; i < 8; ++i) {
|
||||
for (int32_t i = 0; i < 7; ++i) {
|
||||
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
|
||||
}
|
||||
|
||||
|
|
@ -5548,6 +5650,13 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
|
|||
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
|
||||
}
|
||||
|
||||
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
|
@ -9005,7 +9114,7 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
|
|||
|
||||
for (int32_t i = 0; i < pRsp->blockNum; i++) {
|
||||
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
|
||||
void * data = taosArrayGetP(pRsp->blockData, i);
|
||||
void *data = taosArrayGetP(pRsp->blockData, i);
|
||||
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
|
||||
if (pRsp->withSchema) {
|
||||
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
|
||||
|
|
@ -9044,7 +9153,7 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < pRsp->blockNum; i++) {
|
||||
void * data;
|
||||
void *data;
|
||||
uint64_t bLen;
|
||||
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
|
||||
taosArrayPush(pRsp->blockData, &data);
|
||||
|
|
@ -9104,7 +9213,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
|
|||
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
|
||||
if (pRsp->createTableNum) {
|
||||
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
|
||||
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i);
|
||||
void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
|
||||
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
|
||||
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
|
||||
}
|
||||
|
|
@ -9123,7 +9232,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion
|
|||
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
|
||||
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
|
||||
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
|
||||
void * pCreate = NULL;
|
||||
void *pCreate = NULL;
|
||||
uint64_t len;
|
||||
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
|
||||
int32_t l = (int32_t)len;
|
||||
|
|
@ -9435,7 +9544,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
|
|||
taosArrayDestroy(pTbData->aCol);
|
||||
} else {
|
||||
int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
|
||||
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
|
||||
SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
|
||||
|
||||
for (int32_t i = 0; i < nRow; ++i) {
|
||||
tRowDestroy(rows[i]);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,14 @@ target_include_directories(
|
|||
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
|
||||
)
|
||||
|
||||
IF (TD_ENTERPRISE)
|
||||
IF(${BUILD_WITH_S3})
|
||||
add_definitions(-DUSE_S3)
|
||||
ELSEIF(${BUILD_WITH_COS})
|
||||
add_definitions(-DUSE_COS)
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
|
||||
ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc)
|
||||
SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc")
|
||||
|
|
|
|||
|
|
@ -27,15 +27,15 @@
|
|||
#include "cus_name.h"
|
||||
#else
|
||||
#ifndef CUS_NAME
|
||||
#define CUS_NAME "TDengine"
|
||||
#define CUS_NAME "TDengine"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_PROMPT
|
||||
#define CUS_PROMPT "taos"
|
||||
#define CUS_PROMPT "taos"
|
||||
#endif
|
||||
|
||||
#ifndef CUS_EMAIL
|
||||
#define CUS_EMAIL "<support@taosdata.com>"
|
||||
#define CUS_EMAIL "<support@taosdata.com>"
|
||||
#endif
|
||||
#endif
|
||||
// clang-format off
|
||||
|
|
@ -58,6 +58,7 @@ static struct {
|
|||
bool dumpSdb;
|
||||
bool generateGrant;
|
||||
bool memDbg;
|
||||
bool checkS3;
|
||||
bool printAuth;
|
||||
bool printVersion;
|
||||
bool printHelp;
|
||||
|
|
@ -169,7 +170,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
|
|||
return -1;
|
||||
}
|
||||
} else if (strcmp(argv[i], "-a") == 0) {
|
||||
if(i < argc - 1) {
|
||||
if (i < argc - 1) {
|
||||
if (strlen(argv[++i]) >= PATH_MAX) {
|
||||
printf("apollo url overflow");
|
||||
return -1;
|
||||
|
|
@ -182,7 +183,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
|
|||
} else if (strcmp(argv[i], "-s") == 0) {
|
||||
global.dumpSdb = true;
|
||||
} else if (strcmp(argv[i], "-E") == 0) {
|
||||
if(i < argc - 1) {
|
||||
if (i < argc - 1) {
|
||||
if (strlen(argv[++i]) >= PATH_MAX) {
|
||||
printf("env file path overflow");
|
||||
return -1;
|
||||
|
|
@ -207,6 +208,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
|
|||
cmdEnvIndex++;
|
||||
} else if (strcmp(argv[i], "-dm") == 0) {
|
||||
global.memDbg = true;
|
||||
} else if (strcmp(argv[i], "--checks3") == 0) {
|
||||
global.checkS3 = true;
|
||||
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 ||
|
||||
strcmp(argv[i], "-?") == 0) {
|
||||
global.printHelp = true;
|
||||
|
|
@ -267,8 +270,21 @@ static void dmDumpCfg() {
|
|||
cfgDumpCfg(pCfg, 0, true);
|
||||
}
|
||||
|
||||
static int32_t dmCheckS3() {
|
||||
int32_t code = 0;
|
||||
SConfig *pCfg = taosGetCfg();
|
||||
cfgDumpCfgS3(pCfg, 0, true);
|
||||
#if defined(USE_S3)
|
||||
extern int32_t s3CheckCfg();
|
||||
|
||||
code = s3CheckCfg();
|
||||
#endif
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t dmInitLog() {
|
||||
return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
|
||||
return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
|
||||
0);
|
||||
}
|
||||
|
||||
static void taosCleanupArgs() {
|
||||
|
|
@ -355,6 +371,15 @@ int mainWindows(int argc, char **argv) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (global.checkS3) {
|
||||
int32_t code = dmCheckS3();
|
||||
taosCleanupCfg();
|
||||
taosCloseLog();
|
||||
taosCleanupArgs();
|
||||
taosConvDestroy();
|
||||
return code;
|
||||
}
|
||||
|
||||
if (global.dumpConfig) {
|
||||
dmDumpCfg();
|
||||
taosCleanupCfg();
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_S3MIGRATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
@ -208,6 +209,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
|
|||
SVnodeObj **ppVnode = pIter;
|
||||
if (ppVnode == NULL || *ppVnode == NULL) continue;
|
||||
|
||||
SVnodeObj *pVnode = *ppVnode;
|
||||
SVnodeObj *pVnode = *ppVnode;
|
||||
if (!pVnode->failed) {
|
||||
SVnodeLoadLite vload = {0};
|
||||
if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
|
||||
|
|
@ -159,6 +159,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
|
|||
pCfg->hashSuffix = pCreate->hashSuffix;
|
||||
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
|
||||
|
||||
pCfg->s3ChunkSize = pCreate->s3ChunkSize;
|
||||
pCfg->s3KeepLocal = pCreate->s3KeepLocal;
|
||||
pCfg->s3Compact = pCreate->s3Compact;
|
||||
|
||||
pCfg->standby = 0;
|
||||
pCfg->syncCfg.replicaNum = 0;
|
||||
pCfg->syncCfg.totalReplicaNum = 0;
|
||||
|
|
@ -238,17 +242,18 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
dInfo(
|
||||
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
|
||||
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
|
||||
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d"
|
||||
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d s3ChunkSize:%d s3KeepLocal:%d s3Compact:%d tsma:%d "
|
||||
"precision:%d compression:%d minRows:%d maxRows:%d"
|
||||
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
|
||||
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
|
||||
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
|
||||
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
|
||||
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
|
||||
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
|
||||
req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod,
|
||||
req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
|
||||
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica,
|
||||
req.learnerSelfIndex, req.strict, req.changeVersion);
|
||||
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
|
||||
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
|
||||
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
|
||||
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion);
|
||||
|
||||
for (int32_t i = 0; i < req.replica; ++i) {
|
||||
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
|
||||
|
|
@ -347,7 +352,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
_OVER:
|
||||
if (code != 0) {
|
||||
vnodeClose(pImpl);
|
||||
vnodeDestroy(0, path, pMgmt->pTfs);
|
||||
vnodeDestroy(0, path, pMgmt->pTfs, 0);
|
||||
} else {
|
||||
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId,
|
||||
TMSG_INFO(pMsg->msgType));
|
||||
|
|
@ -358,7 +363,7 @@ _OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
//alter replica doesn't use this, but restore dnode still use this
|
||||
// alter replica doesn't use this, but restore dnode still use this
|
||||
int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
SAlterVnodeTypeReq req = {0};
|
||||
if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
|
||||
|
|
@ -400,8 +405,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name);
|
||||
|
||||
int32_t vgId = req.vgId;
|
||||
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d",
|
||||
vgId, req.replica, req.selfIndex, req.strict, req.changeVersion);
|
||||
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", vgId, req.replica,
|
||||
req.selfIndex, req.strict, req.changeVersion);
|
||||
for (int32_t i = 0; i < req.replica; ++i) {
|
||||
SReplica *pReplica = &req.replicas[i];
|
||||
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id);
|
||||
|
|
@ -484,12 +489,12 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if(req.learnerReplicas == 0){
|
||||
if (req.learnerReplicas == 0) {
|
||||
req.learnerSelfIndex = -1;
|
||||
}
|
||||
|
||||
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request",
|
||||
req.vgId, TMSG_INFO(pMsg->msgType));
|
||||
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", req.vgId,
|
||||
TMSG_INFO(pMsg->msgType));
|
||||
|
||||
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
|
||||
if (pVnode == NULL) {
|
||||
|
|
@ -501,7 +506,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
|
||||
ESyncRole role = vnodeGetRole(pVnode->pImpl);
|
||||
dInfo("vgId:%d, checking node role:%d", req.vgId, role);
|
||||
if(role == TAOS_SYNC_ROLE_VOTER){
|
||||
if (role == TAOS_SYNC_ROLE_VOTER) {
|
||||
dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role);
|
||||
terrno = TSDB_CODE_VND_ALREADY_IS_VOTER;
|
||||
vmReleaseVnode(pMgmt, pVnode);
|
||||
|
|
@ -509,7 +514,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
}
|
||||
|
||||
dInfo("vgId:%d, checking node catch up", req.vgId);
|
||||
if(vnodeIsCatchUp(pVnode->pImpl) != 1){
|
||||
if (vnodeIsCatchUp(pVnode->pImpl) != 1) {
|
||||
terrno = TSDB_CODE_VND_NOT_CATCH_UP;
|
||||
vmReleaseVnode(pMgmt, pVnode);
|
||||
return -1;
|
||||
|
|
@ -519,8 +524,8 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
|
||||
vmReleaseVnode(pMgmt, pVnode);
|
||||
|
||||
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request",
|
||||
req.vgId, TMSG_INFO(pMsg->msgType));
|
||||
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", req.vgId,
|
||||
TMSG_INFO(pMsg->msgType));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -933,6 +938,7 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "vmInt.h"
|
||||
#include "libs/function/tudf.h"
|
||||
#include "tfs.h"
|
||||
#include "vnd.h"
|
||||
#include "libs/function/tudf.h"
|
||||
|
||||
int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
|
||||
int32_t diskId = -1;
|
||||
|
|
@ -234,6 +234,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
|
|||
dInfo("vgId:%d, commit data finished", pVnode->vgId);
|
||||
}
|
||||
|
||||
int32_t nodeId = vnodeNodeId(pVnode->pImpl);
|
||||
vnodeClose(pVnode->pImpl);
|
||||
pVnode->pImpl = NULL;
|
||||
|
||||
|
|
@ -250,7 +251,7 @@ _closed:
|
|||
if (pVnode->dropped) {
|
||||
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
|
||||
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
|
||||
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs);
|
||||
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs, nodeId);
|
||||
}
|
||||
|
||||
vmFreeVnodeObj(&pVnode);
|
||||
|
|
|
|||
|
|
@ -4,15 +4,13 @@ target_link_libraries(
|
|||
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
|
||||
)
|
||||
|
||||
IF (TD_STORAGE)
|
||||
|
||||
IF (TD_ENTERPRISE)
|
||||
IF(${BUILD_WITH_S3})
|
||||
add_definitions(-DUSE_S3)
|
||||
add_definitions(-DUSE_S3)
|
||||
ELSEIF(${BUILD_WITH_COS})
|
||||
add_definitions(-DUSE_COS)
|
||||
add_definitions(-DUSE_COS)
|
||||
ENDIF()
|
||||
|
||||
ENDIF ()
|
||||
ENDIF()
|
||||
|
||||
IF (DEFINED GRANT_CFG_INCLUDE_DIR)
|
||||
add_definitions(-DGRANTS_CFG)
|
||||
|
|
|
|||
|
|
@ -379,6 +379,9 @@ typedef struct {
|
|||
int32_t walRollPeriod;
|
||||
int64_t walRetentionSize;
|
||||
int64_t walSegmentSize;
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
int8_t withArbitrator;
|
||||
} SDbCfg;
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@
|
|||
#include "tjson.h"
|
||||
|
||||
#define DB_VER_NUMBER 1
|
||||
#define DB_RESERVE_SIZE 41
|
||||
#define DB_RESERVE_SIZE 32
|
||||
|
||||
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
|
||||
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
|
||||
|
|
@ -49,6 +49,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq);
|
|||
static int32_t mndProcessDropDbReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessUseDbReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessTrimDbReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq);
|
||||
static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity);
|
||||
static void mndCancelGetNextDb(SMnode *pMnode, void *pIter);
|
||||
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq);
|
||||
|
|
@ -76,6 +77,7 @@ int32_t mndInitDb(SMnode *pMnode) {
|
|||
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB, mndProcessS3MigrateDbReq);
|
||||
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
|
||||
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb);
|
||||
|
|
@ -140,6 +142,9 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
|
|||
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
|
||||
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
|
||||
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
|
||||
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
|
||||
|
||||
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
|
||||
|
|
@ -232,11 +237,26 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
|
|||
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
|
||||
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
|
||||
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
|
||||
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
|
||||
|
||||
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
|
||||
taosInitRWLatch(&pDb->lock);
|
||||
|
||||
if (pDb->cfg.s3ChunkSize == 0) {
|
||||
pDb->cfg.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
|
||||
mInfo("db:%s, s3ChunkSize set from %d to default %d", pDb->name, pDb->cfg.s3ChunkSize, TSDB_DEFAULT_S3_CHUNK_SIZE);
|
||||
}
|
||||
|
||||
if (pDb->cfg.s3KeepLocal == 0) {
|
||||
pDb->cfg.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
|
||||
mInfo("db:%s, s3KeepLocal set from %d to default %d", pDb->name, pDb->cfg.s3KeepLocal, TSDB_DEFAULT_S3_KEEP_LOCAL);
|
||||
}
|
||||
|
||||
if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) {
|
||||
mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize,
|
||||
TSDB_DEFAULT_TSDB_PAGESIZE);
|
||||
|
|
@ -322,6 +342,9 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
|
|||
pOld->cfg.minRows = pNew->cfg.minRows;
|
||||
pOld->cfg.maxRows = pNew->cfg.maxRows;
|
||||
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
|
||||
pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize;
|
||||
pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal;
|
||||
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
|
||||
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
|
||||
pOld->compactStartTime = pNew->compactStartTime;
|
||||
taosWUnLockLatch(&pOld->lock);
|
||||
|
|
@ -422,6 +445,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
|
|||
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
|
||||
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
|
||||
|
||||
if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
|
||||
if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
|
||||
if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
|
||||
|
||||
terrno = 0;
|
||||
return terrno;
|
||||
}
|
||||
|
|
@ -471,6 +498,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
|
|||
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
|
||||
return -1;
|
||||
}
|
||||
if (pNewCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pNewCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
|
||||
if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
|
||||
if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
|
||||
|
||||
terrno = 0;
|
||||
return terrno;
|
||||
|
|
@ -507,6 +537,9 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
|
|||
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
|
||||
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
|
||||
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
|
||||
if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
|
||||
}
|
||||
|
||||
|
|
@ -679,6 +712,9 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
|
|||
.sstTrigger = pCreate->sstTrigger,
|
||||
.hashPrefix = pCreate->hashPrefix,
|
||||
.hashSuffix = pCreate->hashSuffix,
|
||||
.s3ChunkSize = pCreate->s3ChunkSize,
|
||||
.s3KeepLocal = pCreate->s3KeepLocal,
|
||||
.s3Compact = pCreate->s3Compact,
|
||||
.tsdbPageSize = pCreate->tsdbPageSize,
|
||||
.withArbitrator = pCreate->withArbitrator,
|
||||
};
|
||||
|
|
@ -945,6 +981,18 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
|
|||
terrno = 0;
|
||||
}
|
||||
|
||||
if (pAlter->s3KeepLocal > TSDB_MIN_S3_KEEP_LOCAL && pAlter->s3KeepLocal != pDb->cfg.s3KeepLocal) {
|
||||
pDb->cfg.s3KeepLocal = pAlter->s3KeepLocal;
|
||||
pDb->vgVersion++;
|
||||
terrno = 0;
|
||||
}
|
||||
|
||||
if (pAlter->s3Compact > TSDB_MIN_S3_COMPACT && pAlter->s3Compact != pDb->cfg.s3Compact) {
|
||||
pDb->cfg.s3Compact = pAlter->s3Compact;
|
||||
pDb->vgVersion++;
|
||||
terrno = 0;
|
||||
}
|
||||
|
||||
if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) {
|
||||
pDb->cfg.withArbitrator = pAlter->withArbitrator;
|
||||
pDb->vgVersion++;
|
||||
|
|
@ -1151,6 +1199,9 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
|
|||
cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
|
||||
cfgRsp->schemaless = pDb->cfg.schemaless;
|
||||
cfgRsp->sstTrigger = pDb->cfg.sstTrigger;
|
||||
cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize;
|
||||
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
|
||||
cfgRsp->s3Compact = pDb->cfg.s3Compact;
|
||||
cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
|
||||
}
|
||||
|
||||
|
|
@ -1218,7 +1269,7 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
|
|||
if (pIter == NULL) break;
|
||||
|
||||
if (pArbGroup->dbUid == pDb->uid) {
|
||||
if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) {
|
||||
if (mndSetDropArbGroupPrepareLogs(pTrans, pArbGroup) != 0) {
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pArbGroup);
|
||||
return -1;
|
||||
|
|
@ -1246,7 +1297,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
|
|||
if (pIter == NULL) break;
|
||||
|
||||
if (pArbGroup->dbUid == pDb->uid) {
|
||||
if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) {
|
||||
if (mndSetDropArbGroupCommitLogs(pTrans, pArbGroup) != 0) {
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pArbGroup);
|
||||
return -1;
|
||||
|
|
@ -1798,6 +1849,77 @@ _OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t mndS3MigrateDb(SMnode *pMnode, SDbObj *pDb) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SVgObj *pVgroup = NULL;
|
||||
void *pIter = NULL;
|
||||
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
|
||||
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
|
||||
int32_t contLen = reqLen + sizeof(SMsgHead);
|
||||
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
if (pVgroup->dbUid != pDb->uid) continue;
|
||||
|
||||
SMsgHead *pHead = rpcMallocCont(contLen);
|
||||
if (pHead == NULL) {
|
||||
sdbCancelFetch(pSdb, pVgroup);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
pHead->contLen = htonl(contLen);
|
||||
pHead->vgId = htonl(pVgroup->vgId);
|
||||
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), contLen, &s3migrateReq);
|
||||
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
|
||||
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
|
||||
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
|
||||
if (code != 0) {
|
||||
mError("vgId:%d, failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
|
||||
} else {
|
||||
mInfo("vgId:%d, send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
|
||||
}
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
SDbObj *pDb = NULL;
|
||||
SS3MigrateDbReq s3migrateReq = {0};
|
||||
|
||||
if (tDeserializeSS3MigrateDbReq(pReq->pCont, pReq->contLen, &s3migrateReq) != 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
mInfo("db:%s, start to s3migrate", s3migrateReq.db);
|
||||
|
||||
pDb = mndAcquireDb(pMnode, s3migrateReq.db);
|
||||
if (pDb == NULL) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
code = mndS3MigrateDb(pMnode, pDb);
|
||||
|
||||
_OVER:
|
||||
if (code != 0) {
|
||||
mError("db:%s, failed to process s3migrate db req since %s", s3migrateReq.db, terrstr());
|
||||
}
|
||||
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
return code;
|
||||
}
|
||||
|
||||
const char *mndGetDbStr(const char *src) {
|
||||
char *pos = strstr(src, TS_PATH_DELIMITER);
|
||||
if (pos != NULL) ++pos;
|
||||
|
|
@ -2096,6 +2218,18 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
|
|||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false);
|
||||
|
||||
char keeplocalVstr[128] = {0};
|
||||
len = sprintf(&keeplocalVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.s3KeepLocal);
|
||||
varDataSetLen(keeplocalVstr, len);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataSetVal(pColInfo, rows, (const char *)keeplocalVstr, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,6 +134,14 @@ static void mndPullupTtl(SMnode *pMnode) {
|
|||
}
|
||||
|
||||
static void mndPullupTrimDb(SMnode *pMnode) {
|
||||
mTrace("pullup s3migrate");
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_S3MIGRATE_DB_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
}
|
||||
|
||||
static void mndPullupS3MigrateDb(SMnode *pMnode) {
|
||||
mTrace("pullup trim");
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
|
|
@ -319,6 +327,7 @@ static int32_t minCronTime() {
|
|||
int32_t min = INT32_MAX;
|
||||
min = TMIN(min, tsTtlPushIntervalSec);
|
||||
min = TMIN(min, tsTrimVDbIntervalSec);
|
||||
min = TMIN(min, tsS3MigrateIntervalSec);
|
||||
min = TMIN(min, tsTransPullupInterval);
|
||||
min = TMIN(min, tsCompactPullupInterval);
|
||||
min = TMIN(min, tsMqRebalanceInterval);
|
||||
|
|
@ -342,6 +351,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
|
|||
mndPullupTrimDb(pMnode);
|
||||
}
|
||||
|
||||
if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) {
|
||||
mndPullupS3MigrateDb(pMnode);
|
||||
}
|
||||
|
||||
if (sec % tsTransPullupInterval == 0) {
|
||||
mndPullupTrans(pMnode);
|
||||
}
|
||||
|
|
@ -806,6 +819,7 @@ _OVER:
|
|||
pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
|
||||
pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
|
||||
pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
|
||||
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
|
||||
pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
|
||||
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
|
||||
pMnode->stopped, state.restored, syncStr(state.state));
|
||||
|
|
@ -950,7 +964,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
|
|||
if (pObj->id == pMnode->selfDnodeId) {
|
||||
pClusterInfo->first_ep_dnode_id = pObj->id;
|
||||
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
|
||||
//pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
|
||||
// pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
|
||||
pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
|
||||
// pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
|
||||
tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
|
|||
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
|
||||
static int32_t mndProcessTtlTimer(SRpcMsg *pReq);
|
||||
static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq);
|
||||
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq);
|
||||
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pReq);
|
||||
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessAlterStbReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessDropStbReq(SRpcMsg *pReq);
|
||||
|
|
@ -82,6 +84,8 @@ int32_t mndInitStb(SMnode *pMnode) {
|
|||
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
|
||||
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
|
||||
|
||||
|
|
@ -1044,6 +1048,43 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SVgObj *pVgroup = NULL;
|
||||
void *pIter = NULL;
|
||||
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
|
||||
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
|
||||
int32_t contLen = reqLen + sizeof(SMsgHead);
|
||||
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
SMsgHead *pHead = rpcMallocCont(contLen);
|
||||
if (pHead == NULL) {
|
||||
sdbCancelFetch(pSdb, pVgroup);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
pHead->contLen = htonl(contLen);
|
||||
pHead->vgId = htonl(pVgroup->vgId);
|
||||
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq);
|
||||
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
|
||||
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
|
||||
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
|
||||
if (code != 0) {
|
||||
mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
|
||||
} else {
|
||||
mInfo("vgId:%d, timer send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
|
||||
}
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
|
||||
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
|
||||
if (strcmp(pStb->pTags[tag].name, tagName) == 0) {
|
||||
|
|
@ -2695,6 +2736,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
|
|||
|
||||
static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; }
|
||||
static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; }
|
||||
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pRsp) { return 0; }
|
||||
|
||||
static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
|
|
|
|||
|
|
@ -32,10 +32,10 @@
|
|||
#define VGROUP_VER_NUMBER 1
|
||||
#define VGROUP_RESERVE_SIZE 64
|
||||
|
||||
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
|
||||
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
|
||||
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
|
||||
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
|
||||
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
|
||||
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
|
||||
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
|
||||
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
|
||||
|
||||
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
|
||||
static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
|
||||
|
|
@ -276,6 +276,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
|
|||
createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
|
||||
createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
|
||||
createReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
|
||||
createReq.s3ChunkSize = pDb->cfg.s3ChunkSize;
|
||||
createReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
|
||||
createReq.s3Compact = pDb->cfg.s3Compact;
|
||||
createReq.minRows = pDb->cfg.minRows;
|
||||
createReq.maxRows = pDb->cfg.maxRows;
|
||||
createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
|
||||
|
|
@ -399,6 +402,8 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV
|
|||
alterReq.minRows = pDb->cfg.minRows;
|
||||
alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
|
||||
alterReq.walRetentionSize = pDb->cfg.walRetentionSize;
|
||||
alterReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
|
||||
alterReq.s3Compact = pDb->cfg.s3Compact;
|
||||
|
||||
mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId);
|
||||
int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq);
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
|
|||
int32_t diskPrimary, STfs *pTfs);
|
||||
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
|
||||
int32_t diskPrimary, STfs *pTfs);
|
||||
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs);
|
||||
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId);
|
||||
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force);
|
||||
void vnodePreClose(SVnode *pVnode);
|
||||
void vnodePostClose(SVnode *pVnode);
|
||||
|
|
@ -319,6 +319,9 @@ struct SVnodeCfg {
|
|||
int16_t hashPrefix;
|
||||
int16_t hashSuffix;
|
||||
int32_t tsdbPageSize;
|
||||
int32_t s3ChunkSize;
|
||||
int32_t s3KeepLocal;
|
||||
int8_t s3Compact;
|
||||
};
|
||||
|
||||
#define TABLE_ROLLUP_ON ((int8_t)0x1)
|
||||
|
|
|
|||
|
|
@ -636,6 +636,7 @@ typedef struct {
|
|||
STsdb * pTsdb;
|
||||
const char *objName;
|
||||
uint8_t s3File;
|
||||
int32_t lcn;
|
||||
int32_t fid;
|
||||
int64_t cid;
|
||||
int64_t blkno;
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
|
|||
bool vnodeShouldRollback(SVnode* pVnode);
|
||||
|
||||
// vnodeSync.c
|
||||
int64_t vnodeClusterId(SVnode* pVnode);
|
||||
int32_t vnodeNodeId(SVnode* pVnode);
|
||||
int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion);
|
||||
int32_t vnodeSyncStart(SVnode* pVnode);
|
||||
void vnodeSyncPreClose(SVnode* pVnode);
|
||||
|
|
|
|||
|
|
@ -239,6 +239,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t
|
|||
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
|
||||
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
|
||||
int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
|
||||
int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
|
||||
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
|
||||
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
|
||||
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
|
||||
|
|
|
|||
|
|
@ -130,21 +130,8 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
|
|||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
extern int8_t tsS3Enabled;
|
||||
|
||||
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
|
||||
committer->ctx->skipTsRow = false;
|
||||
if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) {
|
||||
committer->ctx->skipTsRow = true;
|
||||
}
|
||||
*/
|
||||
int64_t ts = TSDBROW_TS(&row->row);
|
||||
|
||||
if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) {
|
||||
ts = committer->ctx->maxKey + 1;
|
||||
}
|
||||
|
||||
if (ts > committer->ctx->maxKey) {
|
||||
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
|
||||
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
|
||||
|
|
@ -408,31 +395,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
|
|||
// reset nextKey
|
||||
committer->ctx->nextKey = TSKEY_MAX;
|
||||
|
||||
committer->ctx->skipTsRow = false;
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
extern int32_t tsS3UploadDelaySec;
|
||||
long s3Size(const char *object_name);
|
||||
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
|
||||
if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) {
|
||||
STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA];
|
||||
if (fobj && fobj->f->did.level == nlevel - 1) {
|
||||
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
|
||||
if (taosCheckExistFile(fobj->fname)) {
|
||||
int32_t mtime = 0;
|
||||
taosStatFile(fobj->fname, NULL, &mtime, NULL);
|
||||
if (mtime < committer->ctx->now - tsS3UploadDelaySec) {
|
||||
committer->ctx->skipTsRow = true;
|
||||
}
|
||||
} else /*if (s3Size(object_name) > 0) */ {
|
||||
committer->ctx->skipTsRow = true;
|
||||
}
|
||||
}
|
||||
// new fset can be written with ts data
|
||||
}
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
|
||||
|
|
|
|||
|
|
@ -29,15 +29,16 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
|
|||
|
||||
reader[0]->config[0] = config[0];
|
||||
|
||||
int32_t lcn = config->file.lcn;
|
||||
if (fname) {
|
||||
if (fname) {
|
||||
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
|
||||
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
} else {
|
||||
char fname1[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(config->tsdb, &config->file, fname1);
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
|
|
@ -113,8 +114,8 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
|
|||
static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
|
||||
|
||||
static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
ASSERT(writer->ctx->offset <= writer->file.size);
|
||||
ASSERT(writer->config->fid == writer->file.fid);
|
||||
|
||||
|
|
@ -151,7 +152,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
|
|||
}
|
||||
|
||||
tsdbTFileName(writer->config->tsdb, &writer->file, fname);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
|
|||
if (fname) {
|
||||
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
|
||||
if (fname[i]) {
|
||||
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i]);
|
||||
int32_t lcn = config->files[i].file.lcn;
|
||||
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
}
|
||||
|
|
@ -113,7 +114,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
|
|||
if (config->files[i].exist) {
|
||||
char fname1[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(config->tsdb, &config->files[i].file, fname1);
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i]);
|
||||
int32_t lcn = config->files[i].file.lcn;
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
}
|
||||
|
|
@ -327,15 +329,15 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
|
|||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
ASSERT(br.offset == buffer0->size);
|
||||
|
||||
bool loadExtra = false;
|
||||
int extraColIdx = -1;
|
||||
for (int i = 0; i < ncid; i++) {
|
||||
if (tBlockDataGetColData(bData, cids[i]) == NULL) {
|
||||
loadExtra = true;
|
||||
extraColIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!loadExtra) {
|
||||
if (extraColIdx < 0) {
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
|
|
@ -345,10 +347,78 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
|
|||
buffer0, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// calc szHint
|
||||
int64_t szHint = 0;
|
||||
int extraCols = 1;
|
||||
for (int i = extraColIdx + 1; i < ncid; ++i) {
|
||||
if (tBlockDataGetColData(bData, cids[i]) == NULL) {
|
||||
++extraCols;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (extraCols >= 2) {
|
||||
br = BUFFER_READER_INITIALIZER(0, buffer0);
|
||||
|
||||
SBlockCol blockCol = {.cid = 0};
|
||||
for (int32_t i = extraColIdx; i < ncid; ++i) {
|
||||
int16_t extraColCid = cids[i];
|
||||
|
||||
while (extraColCid > blockCol.cid) {
|
||||
if (br.offset >= buffer0->size) {
|
||||
blockCol.cid = INT16_MAX;
|
||||
break;
|
||||
}
|
||||
|
||||
code = tGetBlockCol(&br, &blockCol);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
if (extraColCid == blockCol.cid || blockCol.cid == INT16_MAX) {
|
||||
extraColIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (blockCol.cid > 0 && blockCol.cid < INT16_MAX /*&& blockCol->flag == HAS_VALUE*/) {
|
||||
int64_t offset = blockCol.offset;
|
||||
SBlockCol lastNonNoneBlockCol = {.cid = 0};
|
||||
|
||||
for (int32_t i = extraColIdx; i < ncid; ++i) {
|
||||
int16_t extraColCid = cids[i];
|
||||
|
||||
while (extraColCid > blockCol.cid) {
|
||||
if (br.offset >= buffer0->size) {
|
||||
blockCol.cid = INT16_MAX;
|
||||
break;
|
||||
}
|
||||
|
||||
code = tGetBlockCol(&br, &blockCol);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
if (extraColCid == blockCol.cid) {
|
||||
lastNonNoneBlockCol = blockCol;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (blockCol.cid == INT16_MAX) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastNonNoneBlockCol.cid > 0) {
|
||||
szHint = lastNonNoneBlockCol.offset + lastNonNoneBlockCol.szBitmap + lastNonNoneBlockCol.szOffset +
|
||||
lastNonNoneBlockCol.szValue - offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// load each column
|
||||
SBlockCol blockCol = {
|
||||
.cid = 0,
|
||||
};
|
||||
bool firstRead = true;
|
||||
br = BUFFER_READER_INITIALIZER(0, buffer0);
|
||||
for (int32_t i = 0; i < ncid; i++) {
|
||||
int16_t cid = cids[i];
|
||||
|
|
@ -386,11 +456,13 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
|
|||
} else if (cid == blockCol.cid) {
|
||||
// load from file
|
||||
tBufferClear(buffer1);
|
||||
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA],
|
||||
record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
|
||||
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0);
|
||||
code = tsdbReadFileToBuffer(
|
||||
reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
|
||||
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
firstRead = false;
|
||||
|
||||
// decode the buffer
|
||||
SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
|
||||
code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist);
|
||||
|
|
@ -659,6 +731,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
|
|||
.fid = writer->config->fid,
|
||||
.cid = writer->config->cid,
|
||||
.size = 0,
|
||||
.lcn = writer->config->lcn == -1 ? 0 : -1,
|
||||
.minVer = VERSION_MAX,
|
||||
.maxVer = VERSION_MIN,
|
||||
};
|
||||
|
|
@ -1648,8 +1721,9 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
|
|||
flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
|
||||
}
|
||||
|
||||
int32_t lcn = writer->files[ftype].lcn;
|
||||
tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
if (writer->files[ftype].size == 0) {
|
||||
|
|
@ -1815,8 +1889,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
|
|||
|
||||
int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
|
||||
|
||||
int32_t lcn = writer->files[ftype].lcn;
|
||||
tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ typedef struct SDataFileWriterConfig {
|
|||
int64_t cid;
|
||||
SDiskID did;
|
||||
int64_t compactVersion;
|
||||
int32_t lcn;
|
||||
struct {
|
||||
bool exist;
|
||||
STFile file;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ typedef struct SFDataPtr {
|
|||
int64_t size;
|
||||
} SFDataPtr;
|
||||
|
||||
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD);
|
||||
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
|
||||
extern void tsdbCloseFile(STsdbFD **ppFD);
|
||||
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
|
||||
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
|
||||
#define BLOCK_COMMIT_FACTOR 3
|
||||
|
||||
extern void remove_file(const char *fname, bool last_level);
|
||||
extern void remove_file(const char *fname);
|
||||
|
||||
#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
|
||||
#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
|
||||
|
|
@ -355,17 +355,32 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
|
|||
|
||||
// check file existence
|
||||
if (!taosCheckExistFile(fobj->fname)) {
|
||||
if (tsS3Enabled) {
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
long s3_size = s3Size(object_name);
|
||||
if (s3_size > 0) {
|
||||
return 0;
|
||||
bool found = false;
|
||||
|
||||
if (tsS3Enabled && fobj->f->lcn > 1) {
|
||||
char fname1[TSDB_FILENAME_LEN];
|
||||
tsdbTFileLastChunkName(fs->tsdb, fobj->f, fname1);
|
||||
if (!taosCheckExistFile(fname1)) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fname1);
|
||||
return code;
|
||||
}
|
||||
|
||||
found = true;
|
||||
/*
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
long s3_size = s3Size(object_name);
|
||||
if (s3_size > 0) {
|
||||
return 0;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname);
|
||||
return code;
|
||||
if (!found) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname);
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
{ // TODO: check file size
|
||||
|
|
@ -530,9 +545,9 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
|
|||
if (taosIsDir(file->aname)) continue;
|
||||
|
||||
if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL &&
|
||||
strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3)) {
|
||||
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
|
||||
remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1);
|
||||
strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) &&
|
||||
strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) {
|
||||
remove_file(file->aname);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -900,57 +915,28 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
|
|||
continue;
|
||||
}
|
||||
|
||||
bool skipMerge = false;
|
||||
// bool skipMerge = false;
|
||||
int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
|
||||
if (numFile >= sttTrigger && (!fset->mergeScheduled)) {
|
||||
// launch merge
|
||||
{
|
||||
extern int8_t tsS3Enabled;
|
||||
extern int32_t tsS3UploadDelaySec;
|
||||
long s3Size(const char *object_name);
|
||||
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
|
||||
if (tsS3Enabled && nlevel > 1) {
|
||||
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
|
||||
if (fobj && fobj->f->did.level == nlevel - 1) {
|
||||
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
code = tsdbTFileSetOpenChannel(fset);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
if (taosCheckExistFile(fobj->fname)) {
|
||||
int32_t now = taosGetTimestampSec();
|
||||
int32_t mtime = 0;
|
||||
taosStatFile(fobj->fname, NULL, &mtime, NULL);
|
||||
if (mtime < now - tsS3UploadDelaySec) {
|
||||
skipMerge = true;
|
||||
}
|
||||
} else /* if (s3Size(object_name) > 0) */ {
|
||||
skipMerge = true;
|
||||
}
|
||||
}
|
||||
// new fset can be written with ts data
|
||||
}
|
||||
SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
|
||||
if (arg == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
if (!skipMerge) {
|
||||
code = tsdbTFileSetOpenChannel(fset);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
arg->tsdb = fs->tsdb;
|
||||
arg->fid = fset->fid;
|
||||
|
||||
SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
|
||||
if (arg == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
arg->tsdb = fs->tsdb;
|
||||
arg->fid = fset->fid;
|
||||
|
||||
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree,
|
||||
arg, NULL);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
fset->mergeScheduled = true;
|
||||
}
|
||||
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
|
||||
NULL);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
fset->mergeScheduled = true;
|
||||
}
|
||||
|
||||
if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR && !skipMerge) {
|
||||
if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
|
||||
tsdbFSSetBlockCommit(fset, true);
|
||||
} else {
|
||||
tsdbFSSetBlockCommit(fset, false);
|
||||
|
|
|
|||
|
|
@ -386,7 +386,11 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
|
|||
fobj2->f[0] = fobj1->f[0];
|
||||
}
|
||||
} else {
|
||||
tsdbTFileObjRemove(fobj2);
|
||||
if (fobj1->f->cid != fobj2->f->cid) {
|
||||
tsdbTFileObjRemove(fobj2);
|
||||
} else {
|
||||
tsdbTFileObjRemoveUpdateLC(fobj2);
|
||||
}
|
||||
code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]);
|
||||
if (code) return code;
|
||||
}
|
||||
|
|
@ -585,7 +589,7 @@ int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) {
|
||||
int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
|
||||
if (ppArr && ppArr[0]) {
|
||||
TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear);
|
||||
taosMemoryFree(ppArr[0]);
|
||||
|
|
@ -663,4 +667,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
|
|||
int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
|
||||
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
|
||||
return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -153,6 +153,7 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) {
|
|||
.compactVersion = config->compactVersion,
|
||||
.skmTb = writer[0]->skmTb,
|
||||
.skmRow = writer[0]->skmRow,
|
||||
.lcn = config->lcn,
|
||||
.buffers = writer[0]->buffers,
|
||||
};
|
||||
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ typedef struct {
|
|||
int64_t cid;
|
||||
SDiskID did;
|
||||
int32_t level;
|
||||
int32_t lcn;
|
||||
struct {
|
||||
bool exist;
|
||||
STFile file;
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include "tsdbFile2.h"
|
||||
#include "cos.h"
|
||||
#include "vnd.h"
|
||||
|
||||
// to_json
|
||||
static int32_t head_to_json(const STFile *file, cJSON *json);
|
||||
|
|
@ -42,24 +43,9 @@ static const struct {
|
|||
[TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
|
||||
};
|
||||
|
||||
void remove_file(const char *fname, bool last_level) {
|
||||
int32_t code = taosRemoveFile(fname);
|
||||
if (code) {
|
||||
if (tsS3Enabled && last_level) {
|
||||
const char *object_name = taosDirEntryBaseName((char *)fname);
|
||||
long s3_size = tsS3Enabled ? s3Size(object_name) : 0;
|
||||
if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) {
|
||||
s3DeleteObjects(&object_name, 1);
|
||||
tsdbInfo("file:%s is removed from s3", fname);
|
||||
} else {
|
||||
tsdbError("file:%s remove failed", fname);
|
||||
}
|
||||
} else {
|
||||
tsdbError("file:%s remove failed", fname);
|
||||
}
|
||||
} else {
|
||||
tsdbInfo("file:%s is removed", fname);
|
||||
}
|
||||
void remove_file(const char *fname) {
|
||||
taosRemoveFile(fname);
|
||||
tsdbInfo("file:%s is removed", fname);
|
||||
}
|
||||
|
||||
static int32_t tfile_to_json(const STFile *file, cJSON *json) {
|
||||
|
|
@ -73,6 +59,11 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) {
|
|||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
/* lcn - last chunk number */
|
||||
if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
/* fid */
|
||||
if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
|
@ -121,6 +112,14 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) {
|
|||
return TSDB_CODE_FILE_CORRUPTED;
|
||||
}
|
||||
|
||||
/* lcn */
|
||||
item = cJSON_GetObjectItem(json, "lcn");
|
||||
if (cJSON_IsNumber(item)) {
|
||||
file->lcn = item->valuedouble;
|
||||
} else {
|
||||
// return TSDB_CODE_FILE_CORRUPTED;
|
||||
}
|
||||
|
||||
/* fid */
|
||||
item = cJSON_GetObjectItem(json, "fid");
|
||||
if (cJSON_IsNumber(item)) {
|
||||
|
|
@ -236,7 +235,8 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
|
|||
fobj[0]->state = TSDB_FSTATE_LIVE;
|
||||
fobj[0]->ref = 1;
|
||||
tsdbTFileName(pTsdb, f, fobj[0]->fname);
|
||||
fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
|
||||
// fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
|
||||
fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -258,7 +258,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
|
|||
tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
|
||||
if (nRef == 0) {
|
||||
if (fobj->state == TSDB_FSTATE_DEAD) {
|
||||
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
|
||||
remove_file(fobj->fname);
|
||||
}
|
||||
taosMemoryFree(fobj);
|
||||
}
|
||||
|
|
@ -266,6 +266,61 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
|
||||
if (fobj->f->type != TSDB_FTYPE_DATA) {
|
||||
remove_file(fobj->fname);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!remove_all) {
|
||||
if (fobj->f->lcn < 1) {
|
||||
remove_file(fobj->fname);
|
||||
return;
|
||||
} else {
|
||||
// remove local last chunk file
|
||||
char lc_path[TSDB_FILENAME_LEN];
|
||||
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
|
||||
|
||||
char *dot = strrchr(lc_path, '.');
|
||||
if (!dot) {
|
||||
tsdbError("unexpected path: %s", lc_path);
|
||||
return;
|
||||
}
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
|
||||
|
||||
remove_file(lc_path);
|
||||
}
|
||||
} else {
|
||||
// delete by data file prefix
|
||||
char lc_path[TSDB_FILENAME_LEN];
|
||||
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
|
||||
|
||||
char *object_name = taosDirEntryBaseName(lc_path);
|
||||
int32_t node_id = fobj->nlevel;
|
||||
char object_name_prefix[TSDB_FILENAME_LEN];
|
||||
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
|
||||
|
||||
char *dot = strrchr(object_name_prefix, '.');
|
||||
if (!dot) {
|
||||
tsdbError("unexpected path: %s", object_name_prefix);
|
||||
return;
|
||||
}
|
||||
*(dot + 1) = 0;
|
||||
|
||||
s3DeleteObjectsByPrefix(object_name_prefix);
|
||||
|
||||
// remove local last chunk file
|
||||
dot = strrchr(lc_path, '.');
|
||||
if (!dot) {
|
||||
tsdbError("unexpected path: %s", lc_path);
|
||||
return;
|
||||
}
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
|
||||
|
||||
remove_file(lc_path);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tsdbTFileObjRemove(STFileObj *fobj) {
|
||||
taosThreadMutexLock(&fobj->mutex);
|
||||
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
|
||||
|
|
@ -274,7 +329,21 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
|
|||
taosThreadMutexUnlock(&fobj->mutex);
|
||||
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
|
||||
if (nRef == 0) {
|
||||
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
|
||||
tsdbTFileObjRemoveLC(fobj, true);
|
||||
taosMemoryFree(fobj);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
|
||||
taosThreadMutexLock(&fobj->mutex);
|
||||
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
|
||||
fobj->state = TSDB_FSTATE_DEAD;
|
||||
int32_t nRef = --fobj->ref;
|
||||
taosThreadMutexUnlock(&fobj->mutex);
|
||||
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
|
||||
if (nRef == 0) {
|
||||
tsdbTFileObjRemoveLC(fobj, false);
|
||||
taosMemoryFree(fobj);
|
||||
}
|
||||
return 0;
|
||||
|
|
@ -310,13 +379,45 @@ int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) {
|
||||
SVnode *pVnode = pTsdb->pVnode;
|
||||
STfs *pTfs = pVnode->pTfs;
|
||||
|
||||
if (pTfs) {
|
||||
snprintf(fname, //
|
||||
TSDB_FILENAME_LEN, //
|
||||
"%s%s%s%sv%df%dver%" PRId64 ".%d.%s", //
|
||||
tfsGetDiskPath(pTfs, f->did), //
|
||||
TD_DIRSEP, //
|
||||
pTsdb->path, //
|
||||
TD_DIRSEP, //
|
||||
TD_VID(pVnode), //
|
||||
f->fid, //
|
||||
f->cid, //
|
||||
f->lcn, //
|
||||
g_tfile_info[f->type].suffix);
|
||||
} else {
|
||||
snprintf(fname, //
|
||||
TSDB_FILENAME_LEN, //
|
||||
"%s%sv%df%dver%" PRId64 ".%d.%s", //
|
||||
pTsdb->path, //
|
||||
TD_DIRSEP, //
|
||||
TD_VID(pVnode), //
|
||||
f->fid, //
|
||||
f->cid, //
|
||||
f->lcn, //
|
||||
g_tfile_info[f->type].suffix);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
|
||||
if (f1->type != f2->type) return false;
|
||||
if (f1->did.level != f2->did.level) return false;
|
||||
if (f1->did.id != f2->did.id) return false;
|
||||
if (f1->fid != f2->fid) return false;
|
||||
if (f1->cid != f2->cid) return false;
|
||||
if (f1->s3flag != f2->s3flag) return false;
|
||||
if (f1->lcn != f2->lcn) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ enum {
|
|||
int32_t tsdbTFileToJson(const STFile *f, cJSON *json);
|
||||
int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f);
|
||||
int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]);
|
||||
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]);
|
||||
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2);
|
||||
bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2);
|
||||
|
||||
|
|
@ -53,12 +54,13 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj);
|
|||
int32_t tsdbTFileObjRef(STFileObj *fobj);
|
||||
int32_t tsdbTFileObjUnref(STFileObj *fobj);
|
||||
int32_t tsdbTFileObjRemove(STFileObj *fobj);
|
||||
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj);
|
||||
int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2);
|
||||
|
||||
struct STFile {
|
||||
tsdb_ftype_t type;
|
||||
SDiskID did; // disk id
|
||||
int32_t s3flag;
|
||||
int32_t lcn; // last chunk number
|
||||
int32_t fid; // file id
|
||||
int64_t cid; // commit id
|
||||
int64_t size;
|
||||
|
|
|
|||
|
|
@ -544,7 +544,7 @@ int32_t tsdbMerge(void *arg) {
|
|||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
if (merger->fset == NULL) return 0;
|
||||
|
||||
/*
|
||||
bool skipMerge = false;
|
||||
{
|
||||
extern int8_t tsS3Enabled;
|
||||
|
|
@ -565,7 +565,8 @@ int32_t tsdbMerge(void *arg) {
|
|||
if (mtime < now - tsS3UploadDelaySec) {
|
||||
skipMerge = true;
|
||||
}
|
||||
} else /* if (s3Size(object_name) > 0) */ {
|
||||
} else // if (s3Size(object_name) > 0)
|
||||
{
|
||||
skipMerge = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -577,7 +578,7 @@ int32_t tsdbMerge(void *arg) {
|
|||
code = 0;
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
*/
|
||||
// do merge
|
||||
tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
|
||||
code = tsdbDoMerge(merger);
|
||||
|
|
|
|||
|
|
@ -15,16 +15,48 @@
|
|||
|
||||
#include "cos.h"
|
||||
#include "tsdb.h"
|
||||
#include "vnd.h"
|
||||
|
||||
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
|
||||
int32_t code = 0;
|
||||
const char *path = pFD->path;
|
||||
int32_t szPage = pFD->szPage;
|
||||
int32_t flag = pFD->flag;
|
||||
int64_t lc_size = 0;
|
||||
|
||||
pFD->pFD = taosOpenFile(path, flag);
|
||||
if (pFD->pFD == NULL) {
|
||||
int errsv = errno;
|
||||
if (tsS3Enabled && pFD->lcn > 1 && !strncmp(path + strlen(path) - 5, ".data", 5)) {
|
||||
char lc_path[TSDB_FILENAME_LEN];
|
||||
tstrncpy(lc_path, path, TSDB_FQDN_LEN);
|
||||
|
||||
char *dot = strrchr(lc_path, '.');
|
||||
if (!dot) {
|
||||
tsdbError("unexpected path: %s", lc_path);
|
||||
code = TAOS_SYSTEM_ERROR(ENOENT);
|
||||
goto _exit;
|
||||
}
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn);
|
||||
|
||||
pFD->pFD = taosOpenFile(lc_path, flag);
|
||||
if (pFD->pFD == NULL) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
// taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
// taosCloseFile(&pFD->pFD);
|
||||
// taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
} else {
|
||||
tsdbInfo("no file: %s", path);
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
// taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
/*
|
||||
const char *object_name = taosDirEntryBaseName((char *)path);
|
||||
long s3_size = 0;
|
||||
if (tsS3Enabled) {
|
||||
|
|
@ -43,7 +75,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
|
|||
s3Get(object_name, path);
|
||||
|
||||
pFD->pFD = taosOpenFile(path, flag);
|
||||
|
||||
if (pFD->pFD == NULL) {
|
||||
code = TAOS_SYSTEM_ERROR(ENOENT);
|
||||
// taosMemoryFree(pFD);
|
||||
|
|
@ -57,12 +88,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
|
|||
pFD->objName = object_name;
|
||||
// pFD->szFile = s3_size;
|
||||
#endif
|
||||
} else {
|
||||
tsdbInfo("no file: %s", path);
|
||||
code = TAOS_SYSTEM_ERROR(errsv);
|
||||
// taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
pFD->pBuf = taosMemoryCalloc(1, szPage);
|
||||
|
|
@ -73,26 +99,33 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
|
|||
goto _exit;
|
||||
}
|
||||
|
||||
if (lc_size > 0) {
|
||||
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
|
||||
pFD->szFile = lc_size + chunksize * (pFD->lcn - 1);
|
||||
}
|
||||
|
||||
// not check file size when reading data files.
|
||||
if (flag != TD_FILE_READ && !pFD->s3File) {
|
||||
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
|
||||
if (flag != TD_FILE_READ /* && !pFD->s3File*/) {
|
||||
if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
// taosMemoryFree(pFD->pBuf);
|
||||
// taosCloseFile(&pFD->pFD);
|
||||
// taosMemoryFree(pFD);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
ASSERT(pFD->szFile % szPage == 0);
|
||||
pFD->szFile = pFD->szFile / szPage;
|
||||
}
|
||||
|
||||
ASSERT(pFD->szFile % szPage == 0);
|
||||
pFD->szFile = pFD->szFile / szPage;
|
||||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
|
||||
// =============== PAGE-WISE FILE ===============
|
||||
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD) {
|
||||
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) {
|
||||
int32_t code = 0;
|
||||
STsdbFD *pFD = NULL;
|
||||
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
|
||||
|
|
@ -111,6 +144,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
|
|||
pFD->flag = flag;
|
||||
pFD->szPage = szPage;
|
||||
pFD->pgno = 0;
|
||||
pFD->lcn = lcn;
|
||||
pFD->pTsdb = pTsdb;
|
||||
|
||||
*ppFD = pFD;
|
||||
|
|
@ -123,9 +157,9 @@ void tsdbCloseFile(STsdbFD **ppFD) {
|
|||
STsdbFD *pFD = *ppFD;
|
||||
if (pFD) {
|
||||
taosMemoryFree(pFD->pBuf);
|
||||
if (!pFD->s3File) {
|
||||
taosCloseFile(&pFD->pFD);
|
||||
}
|
||||
// if (!pFD->s3File) {
|
||||
taosCloseFile(&pFD->pFD);
|
||||
//}
|
||||
taosMemoryFree(pFD);
|
||||
*ppFD = NULL;
|
||||
}
|
||||
|
|
@ -141,12 +175,18 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
|
|||
}
|
||||
}
|
||||
|
||||
if (pFD->s3File) {
|
||||
tsdbWarn("%s file: %s", __func__, pFD->path);
|
||||
return code;
|
||||
}
|
||||
if (pFD->pgno > 0) {
|
||||
int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET);
|
||||
int64_t offset = PAGE_OFFSET(pFD->pgno, pFD->szPage);
|
||||
if (pFD->lcn > 1) {
|
||||
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
|
||||
|
||||
offset -= chunkoffset;
|
||||
}
|
||||
ASSERT(offset >= 0);
|
||||
|
||||
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
goto _exit;
|
||||
|
|
@ -182,7 +222,15 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
|
|||
}
|
||||
|
||||
int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
|
||||
if (pFD->lcn > 1) {
|
||||
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
|
||||
|
||||
offset -= chunkoffset;
|
||||
}
|
||||
ASSERT(offset >= 0);
|
||||
/*
|
||||
if (pFD->s3File) {
|
||||
LRUHandle *handle = NULL;
|
||||
|
||||
|
|
@ -203,24 +251,25 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
|
|||
|
||||
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
|
||||
} else {
|
||||
// seek
|
||||
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
// read
|
||||
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
goto _exit;
|
||||
} else if (n < pFD->szPage) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
goto _exit;
|
||||
}
|
||||
*/
|
||||
// seek
|
||||
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
// read
|
||||
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
goto _exit;
|
||||
} else if (n < pFD->szPage) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
goto _exit;
|
||||
}
|
||||
//}
|
||||
|
||||
// check
|
||||
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
|
|
@ -294,6 +343,74 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
|
||||
int32_t code = 0;
|
||||
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
int64_t cOffset = offset % chunksize;
|
||||
int64_t n = 0;
|
||||
|
||||
char *object_name = taosDirEntryBaseName(pFD->path);
|
||||
char object_name_prefix[TSDB_FILENAME_LEN];
|
||||
int32_t node_id = vnodeNodeId(pFD->pTsdb->pVnode);
|
||||
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
|
||||
|
||||
char *dot = strrchr(object_name_prefix, '.');
|
||||
if (!dot) {
|
||||
tsdbError("unexpected path: %s", object_name_prefix);
|
||||
code = TAOS_SYSTEM_ERROR(ENOENT);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
char *buf = taosMemoryCalloc(1, size);
|
||||
|
||||
for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) {
|
||||
int64_t nRead = TMIN(chunksize - cOffset, size - n);
|
||||
|
||||
if (chunkno >= pFD->lcn) {
|
||||
// read last chunk
|
||||
int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET);
|
||||
if (ret < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(buf);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
ret = taosReadFile(pFD->pFD, buf + n, nRead);
|
||||
if (ret < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(buf);
|
||||
goto _exit;
|
||||
} else if (ret < nRead) {
|
||||
code = TSDB_CODE_FILE_CORRUPTED;
|
||||
taosMemoryFree(buf);
|
||||
goto _exit;
|
||||
}
|
||||
} else {
|
||||
uint8_t *pBlock = NULL;
|
||||
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
|
||||
|
||||
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(buf);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
memcpy(buf + n, pBlock, nRead);
|
||||
taosMemoryFree(pBlock);
|
||||
}
|
||||
|
||||
n += nRead;
|
||||
cOffset = 0;
|
||||
}
|
||||
|
||||
*ppBlock = buf;
|
||||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
|
||||
int32_t code = 0;
|
||||
int64_t n = 0;
|
||||
|
|
@ -356,15 +473,22 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
|
|||
}
|
||||
|
||||
int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
|
||||
/*
|
||||
code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
*/
|
||||
code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _exit;
|
||||
}
|
||||
// 3, Store Pages in Cache
|
||||
int nPage = pgnoEnd - pgno + 1;
|
||||
for (int i = 0; i < nPage; ++i) {
|
||||
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
|
||||
if (pFD->szFile != pgno) { // DONOT cache last volatile page
|
||||
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
|
||||
}
|
||||
|
||||
if (szHint > 0 && n >= size) {
|
||||
++pgno;
|
||||
|
|
@ -404,7 +528,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
|
|||
}
|
||||
}
|
||||
|
||||
if (pFD->s3File && tsS3BlockSize < 0) {
|
||||
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
|
||||
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
|
||||
} else {
|
||||
return tsdbReadFileImp(pFD, offset, pBuf, size);
|
||||
|
|
@ -428,11 +552,12 @@ int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer
|
|||
|
||||
int32_t tsdbFsyncFile(STsdbFD *pFD) {
|
||||
int32_t code = 0;
|
||||
|
||||
/*
|
||||
if (pFD->s3File) {
|
||||
tsdbWarn("%s file: %s", __func__, pFD->path);
|
||||
return code;
|
||||
}
|
||||
*/
|
||||
code = tsdbWriteFilePage(pFD);
|
||||
if (code) goto _exit;
|
||||
|
||||
|
|
@ -464,23 +589,23 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
|
|||
|
||||
// head
|
||||
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// data
|
||||
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// sma
|
||||
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// stt
|
||||
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
|
||||
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt]);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt], 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
|
|
@ -658,7 +783,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
|
|||
pDelFReader->fDel = *pFile;
|
||||
|
||||
tsdbDelFileName(pTsdb, pFile, fname);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH);
|
||||
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0);
|
||||
if (code) {
|
||||
taosMemoryFree(pDelFReader);
|
||||
goto _exit;
|
||||
|
|
|
|||
|
|
@ -38,24 +38,42 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
|
|||
return TARRAY2_APPEND(rtner->fopArr, op);
|
||||
}
|
||||
|
||||
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) {
|
||||
int32_t code = 0, lino = 0;
|
||||
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
TdFilePtr fdFrom = NULL, fdTo = NULL;
|
||||
char fname_from[TSDB_FILENAME_LEN];
|
||||
char fname_to[TSDB_FILENAME_LEN];
|
||||
|
||||
STFileOp op = {
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from);
|
||||
tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
fdFrom = taosOpenFile(fname_from, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
|
||||
s3DeleteObjects(&object_name, 1);
|
||||
tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);
|
||||
|
||||
fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
|
||||
if (fdTo == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);
|
||||
int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
taosCloseFile(&fdFrom);
|
||||
taosCloseFile(&fdTo);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
if (fdFrom) taosCloseFile(&fdFrom);
|
||||
if (fdTo) taosCloseFile(&fdTo);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
@ -97,38 +115,11 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
TdFilePtr fdFrom = NULL;
|
||||
// TdFilePtr fdTo = NULL;
|
||||
|
||||
tsdbTFileName(rtner->tsdb, to, fname);
|
||||
|
||||
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
code = s3PutObjectFromFile2(from->fname, object_name, 1);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
taosCloseFile(&fdFrom);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
taosCloseFile(&fdFrom);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
STFileOp op = {0};
|
||||
int32_t lcn = fobj->f->lcn;
|
||||
|
||||
// remove old
|
||||
op = (STFileOp){
|
||||
|
|
@ -153,6 +144,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
|
|||
.maxVer = fobj->f->maxVer,
|
||||
.cid = fobj->f->cid,
|
||||
.size = fobj->f->size,
|
||||
.lcn = lcn,
|
||||
.stt[0] =
|
||||
{
|
||||
.level = fobj->f->stt[0].level,
|
||||
|
|
@ -164,59 +156,14 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
|
|||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// do copy the file
|
||||
code = tsdbDoCopyFile(rtner, fobj, &op.nf);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
if (lcn < 1) {
|
||||
code = tsdbDoCopyFile(rtner, fobj, &op.nf);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
} else {
|
||||
code = tsdbDoCopyFileLC(rtner, fobj, &op.nf);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
STFileOp op = {0};
|
||||
|
||||
// remove old
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// create new
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_CREATE,
|
||||
.fid = fobj->f->fid,
|
||||
.nf =
|
||||
{
|
||||
.type = fobj->f->type,
|
||||
.did = did[0],
|
||||
.fid = fobj->f->fid,
|
||||
.minVer = fobj->f->minVer,
|
||||
.maxVer = fobj->f->maxVer,
|
||||
.cid = fobj->f->cid,
|
||||
.size = fobj->f->size,
|
||||
.stt[0] =
|
||||
{
|
||||
.level = fobj->f->stt[0].level,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
op.nf.s3flag = true;
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// do copy the file
|
||||
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
|
|
@ -327,38 +274,14 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
|
|||
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) {
|
||||
if (fobj == NULL) continue;
|
||||
|
||||
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
|
||||
|
||||
if (fobj->f->did.level == did.level) {
|
||||
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 &&
|
||||
taosCheckExistFile(fobj->fname)) {
|
||||
int32_t mtime = 0;
|
||||
taosStatFile(fobj->fname, NULL, &mtime, NULL);
|
||||
if (mtime < rtner->now - tsS3UploadDelaySec) {
|
||||
tsdbInfo("file:%s size: %" PRId64 " do migrate s3", fobj->fname, fobj->f->size);
|
||||
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
*/
|
||||
continue;
|
||||
}
|
||||
/*
|
||||
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
|
||||
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
} else {
|
||||
|
||||
if (tsS3Enabled) {
|
||||
int64_t fsize = 0;
|
||||
if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(terrno);
|
||||
tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode),
|
||||
__func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
s3EvictCache(fobj->fname, fsize * 2);
|
||||
}
|
||||
*/
|
||||
if (fobj->f->did.level > did.level) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -367,7 +290,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
|
|||
|
||||
code = tsdbDoMigrateFileObj(rtner, fobj, &did);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
//}
|
||||
}
|
||||
|
||||
// stt
|
||||
|
|
@ -471,3 +393,414 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
|
|||
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3KeepLocal, int64_t nowSec) {
|
||||
int32_t localFid;
|
||||
TSKEY key;
|
||||
|
||||
if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) {
|
||||
nowSec = nowSec * 1000;
|
||||
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) {
|
||||
nowSec = nowSec * 1000000l;
|
||||
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
|
||||
nowSec = nowSec * 1000000000l;
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];
|
||||
|
||||
key = nowSec - s3KeepLocal * tsTickPerMin[pKeepCfg->precision];
|
||||
localFid = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
|
||||
|
||||
if (fid >= localFid) {
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
TdFilePtr fdFrom = NULL;
|
||||
// TdFilePtr fdTo = NULL;
|
||||
|
||||
tsdbTFileName(rtner->tsdb, to, fname);
|
||||
|
||||
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
code = s3PutObjectFromFile2(from->fname, object_name, 1);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
taosCloseFile(&fdFrom);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
taosCloseFile(&fdFrom);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
STFileOp op = {0};
|
||||
TdFilePtr fdFrom = NULL, fdTo = NULL;
|
||||
int32_t lcn = fobj->f->lcn + (size - 1) / chunksize;
|
||||
|
||||
// remove old
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// create new
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_CREATE,
|
||||
.fid = fobj->f->fid,
|
||||
.nf =
|
||||
{
|
||||
.type = fobj->f->type,
|
||||
.did = fobj->f->did,
|
||||
.fid = fobj->f->fid,
|
||||
.minVer = fobj->f->minVer,
|
||||
.maxVer = fobj->f->maxVer,
|
||||
.cid = fobj->f->cid,
|
||||
.size = fobj->f->size,
|
||||
.lcn = lcn,
|
||||
.stt[0] =
|
||||
{
|
||||
.level = fobj->f->stt[0].level,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(rtner->tsdb, &op.nf, fname);
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
char object_name_prefix[TSDB_FILENAME_LEN];
|
||||
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
|
||||
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
|
||||
|
||||
char *dot = strrchr(object_name_prefix, '.');
|
||||
if (!dot) {
|
||||
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
|
||||
return -1;
|
||||
}
|
||||
|
||||
char *dot2 = strchr(object_name, '.');
|
||||
if (!dot) {
|
||||
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
|
||||
return -1;
|
||||
}
|
||||
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
|
||||
|
||||
// do copy the file
|
||||
for (int32_t cn = fobj->f->lcn; cn < lcn; ++cn) {
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
|
||||
int64_t c_offset = chunksize * (cn - fobj->f->lcn);
|
||||
|
||||
code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
// copy last chunk
|
||||
int64_t lc_offset = chunksize * (lcn - fobj->f->lcn);
|
||||
int64_t lc_size = size - lc_offset;
|
||||
|
||||
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
|
||||
|
||||
fdFrom = taosOpenFile(fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
|
||||
|
||||
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
|
||||
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
|
||||
if (fdTo == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
taosCloseFile(&fdFrom);
|
||||
taosCloseFile(&fdTo);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
if (fdFrom) taosCloseFile(&fdFrom);
|
||||
if (fdTo) taosCloseFile(&fdTo);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
STFileOp op = {0};
|
||||
int32_t lcn = (size - 1) / chunksize + 1;
|
||||
|
||||
// remove old
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_REMOVE,
|
||||
.fid = fobj->f->fid,
|
||||
.of = fobj->f[0],
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// create new
|
||||
op = (STFileOp){
|
||||
.optype = TSDB_FOP_CREATE,
|
||||
.fid = fobj->f->fid,
|
||||
.nf =
|
||||
{
|
||||
.type = fobj->f->type,
|
||||
.did = fobj->f->did,
|
||||
.fid = fobj->f->fid,
|
||||
.minVer = fobj->f->minVer,
|
||||
.maxVer = fobj->f->maxVer,
|
||||
.cid = fobj->f->cid,
|
||||
.size = fobj->f->size,
|
||||
.lcn = lcn,
|
||||
.stt[0] =
|
||||
{
|
||||
.level = fobj->f->stt[0].level,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
code = TARRAY2_APPEND(rtner->fopArr, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
char fname[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(rtner->tsdb, &op.nf, fname);
|
||||
char *object_name = taosDirEntryBaseName(fname);
|
||||
char object_name_prefix[TSDB_FILENAME_LEN];
|
||||
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
|
||||
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
|
||||
|
||||
char *dot = strrchr(object_name_prefix, '.');
|
||||
if (!dot) {
|
||||
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// do copy the file
|
||||
for (int32_t cn = 1; cn < lcn; ++cn) {
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
|
||||
int64_t c_offset = chunksize * (cn - 1);
|
||||
|
||||
code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
// copy last chunk
|
||||
TdFilePtr fdFrom = NULL, fdTo = NULL;
|
||||
int64_t lc_offset = (int64_t)(lcn - 1) * chunksize;
|
||||
int64_t lc_size = size - lc_offset;
|
||||
|
||||
dot = strchr(object_name, '.');
|
||||
if (!dot) {
|
||||
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
|
||||
return -1;
|
||||
}
|
||||
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn);
|
||||
|
||||
fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ);
|
||||
if (fdFrom == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size);
|
||||
|
||||
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
|
||||
if (fdTo == NULL) code = terrno;
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
|
||||
if (n < 0) {
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
taosCloseFile(&fdFrom);
|
||||
taosCloseFile(&fdTo);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
taosCloseFile(&fdFrom);
|
||||
taosCloseFile(&fdTo);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
|
||||
if (!fobj) return code;
|
||||
|
||||
int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
|
||||
if (expLevel < 0) return code; // expired
|
||||
|
||||
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
|
||||
int32_t s3KeepLocal = pCfg->s3KeepLocal;
|
||||
int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now);
|
||||
if (s3ExpLevel < 1) return code; // keep on local storage
|
||||
|
||||
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
|
||||
int32_t lcn = fobj->f->lcn;
|
||||
|
||||
if (lcn < 1 && taosCheckExistFile(fobj->fname)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
taosStatFile(fobj->fname, &size, &mtime, NULL);
|
||||
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
|
||||
if (pCfg->s3Compact && lcn < 0) {
|
||||
extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync);
|
||||
|
||||
STimeWindow win = {0};
|
||||
tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey);
|
||||
|
||||
tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
|
||||
tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1);
|
||||
tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
|
||||
return code;
|
||||
}
|
||||
|
||||
code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
} else {
|
||||
if (lcn <= 1) {
|
||||
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
|
||||
return code;
|
||||
}
|
||||
char fname1[TSDB_FILENAME_LEN];
|
||||
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
|
||||
|
||||
if (taosCheckExistFile(fname1)) {
|
||||
int32_t mtime = 0;
|
||||
int64_t size = 0;
|
||||
taosStatFile(fname1, &size, &mtime, NULL);
|
||||
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
|
||||
code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
} else {
|
||||
tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino);
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tsdbDoS3MigrateAsync(void *arg) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
SRTNer rtner[1] = {0};
|
||||
|
||||
code = tsdbDoRetentionBegin(arg, rtner);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
STFileSet *fset;
|
||||
TARRAY2_FOREACH(rtner->fsetArr, fset) {
|
||||
if (fset->fid != ((SRtnArg *)arg)->fid) continue;
|
||||
|
||||
code = tsdbDoS3MigrateOnFileSet(rtner, fset);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
code = tsdbDoRetentionEnd(rtner);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
_exit:
|
||||
if (code) {
|
||||
if (TARRAY2_DATA(rtner->fopArr)) {
|
||||
TARRAY2_DESTROY(rtner->fopArr, NULL);
|
||||
}
|
||||
TFileSetArray **fsetArr = &rtner->fsetArr;
|
||||
if (fsetArr[0]) {
|
||||
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
|
||||
}
|
||||
|
||||
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
|
||||
int32_t code = 0;
|
||||
|
||||
if (!tsS3Enabled) {
|
||||
return code;
|
||||
}
|
||||
|
||||
taosThreadMutexLock(&tsdb->mutex);
|
||||
|
||||
if (tsdb->bgTaskDisabled) {
|
||||
taosThreadMutexUnlock(&tsdb->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
STFileSet *fset;
|
||||
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
|
||||
code = tsdbTFileSetOpenChannel(fset);
|
||||
if (code) {
|
||||
taosThreadMutexUnlock(&tsdb->mutex);
|
||||
return code;
|
||||
}
|
||||
|
||||
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
|
||||
if (arg == NULL) {
|
||||
taosThreadMutexUnlock(&tsdb->mutex);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
arg->tsdb = tsdb;
|
||||
arg->now = now;
|
||||
arg->fid = fset->fid;
|
||||
|
||||
if (sync) {
|
||||
code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
|
||||
tsdbFreeRtnArg, arg, NULL);
|
||||
} else {
|
||||
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
|
||||
tsdbFreeRtnArg, arg, NULL);
|
||||
}
|
||||
if (code) {
|
||||
tsdbFreeRtnArg(arg);
|
||||
taosThreadMutexUnlock(&tsdb->mutex);
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
taosThreadMutexUnlock(&tsdb->mutex);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,12 +50,12 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
|
|||
|
||||
// open file
|
||||
if (fname) {
|
||||
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
|
||||
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
} else {
|
||||
char fname1[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(config->tsdb, config->file, fname1);
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
|
||||
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
}
|
||||
|
||||
|
|
@ -724,7 +724,7 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
|
|||
char fname[TSDB_FILENAME_LEN];
|
||||
|
||||
tsdbTFileName(writer->config->tsdb, writer->file, fname);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
|
||||
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
|
|||
char fname[TSDB_FILENAME_LEN];
|
||||
tsdbTFileName(tsdb, &file, fname);
|
||||
|
||||
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd);
|
||||
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// convert
|
||||
|
|
@ -272,7 +272,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
|
|||
code = tsdbTFileObjInit(tsdb, &file, &fobj);
|
||||
TSDB_CHECK_CODE(code, lino, _exit1);
|
||||
|
||||
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd);
|
||||
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit1);
|
||||
|
||||
for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) {
|
||||
|
|
@ -427,7 +427,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
|
|||
}
|
||||
|
||||
char fname[TSDB_FILENAME_LEN] = {0};
|
||||
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd);
|
||||
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
|
|||
.hashEnd = 0,
|
||||
.hashMethod = 0,
|
||||
.sttTrigger = TSDB_DEFAULT_SST_TRIGGER,
|
||||
.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE,
|
||||
.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL,
|
||||
.s3Compact = TSDB_DEFAULT_S3_COMPACT,
|
||||
.tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE};
|
||||
|
||||
int vnodeCheckCfg(const SVnodeCfg *pCfg) {
|
||||
|
|
@ -106,6 +109,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
|
|||
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
|
||||
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
|
||||
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
|
||||
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1;
|
||||
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1;
|
||||
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1;
|
||||
if (pCfg->tsdbCfg.retentions[0].keep > 0) {
|
||||
int32_t nRetention = 1;
|
||||
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
|
||||
|
|
@ -154,9 +160,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
|
|||
SJson *nodeInfo = tjsonCreateArray();
|
||||
if (nodeInfo == NULL) return -1;
|
||||
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
|
||||
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d",
|
||||
pCfg->vgId, pCfg->syncCfg.replicaNum,
|
||||
pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
|
||||
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
|
||||
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
|
||||
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
|
||||
SJson *info = tjsonCreateObject();
|
||||
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
|
||||
|
|
@ -317,6 +322,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
|
|||
pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024;
|
||||
}
|
||||
|
||||
tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code);
|
||||
if (code < 0) {
|
||||
pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
}
|
||||
tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code);
|
||||
if (code < 0) {
|
||||
pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
}
|
||||
tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code);
|
||||
if (code < 0) {
|
||||
pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -305,14 +305,14 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s
|
|||
return dstVgId;
|
||||
}
|
||||
|
||||
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs) {
|
||||
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) {
|
||||
vInfo("path:%s is removed while destroy vnode", path);
|
||||
tfsRmdir(pTfs, path);
|
||||
|
||||
int32_t nlevel = tfsGetLevel(pTfs);
|
||||
if (vgId > 0 && nlevel > 1 && tsS3Enabled) {
|
||||
// int32_t nlevel = tfsGetLevel(pTfs);
|
||||
if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) {
|
||||
char vnode_prefix[TSDB_FILENAME_LEN];
|
||||
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "v%df", vgId);
|
||||
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId);
|
||||
s3DeleteObjectsByPrefix(vnode_prefix);
|
||||
}
|
||||
}
|
||||
|
|
@ -480,11 +480,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
|
|||
vnodeRollback(pVnode);
|
||||
}
|
||||
|
||||
snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
|
||||
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
|
||||
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId);
|
||||
snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%" PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
|
||||
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
|
||||
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId);
|
||||
|
||||
if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){
|
||||
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
|
||||
taos_counter_t *counter = NULL;
|
||||
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
|
||||
if(counter == NULL){
|
||||
|
|
@ -499,11 +499,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
|
|||
if(taos_collector_registry_register_metric(counter) == 1){
|
||||
taos_counter_destroy(counter);
|
||||
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
|
||||
vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter);
|
||||
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);
|
||||
}
|
||||
}
|
||||
pVnode->monitor.insertCounter = counter;
|
||||
vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter);
|
||||
vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter);
|
||||
}
|
||||
|
||||
return pVnode;
|
||||
|
|
|
|||
|
|
@ -23,4 +23,12 @@ int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
|
|||
if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
|
||||
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pR
|
|||
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
|
||||
SRpcMsg *pOriginalMsg);
|
||||
static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
|
|
@ -565,6 +566,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
|
|||
case TDMT_VND_TRIM:
|
||||
if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
|
||||
break;
|
||||
case TDMT_VND_S3MIGRATE:
|
||||
if (vnodeProcessS3MigrateReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
|
||||
break;
|
||||
case TDMT_VND_CREATE_SMA:
|
||||
if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
|
||||
break;
|
||||
|
|
@ -875,6 +879,26 @@ _exit:
|
|||
return code;
|
||||
}
|
||||
|
||||
extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
|
||||
|
||||
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
|
||||
int32_t code = 0;
|
||||
SVS3MigrateDbReq s3migrateReq = {0};
|
||||
|
||||
// decode
|
||||
if (tDeserializeSVS3MigrateDbReq(pReq, len, &s3migrateReq) != 0) {
|
||||
code = TSDB_CODE_INVALID_MSG;
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
|
||||
|
||||
code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
|
||||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
|
||||
SVDropTtlTableReq ttlReq = {0};
|
||||
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
|
||||
|
|
@ -1894,11 +1918,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
|
|||
}
|
||||
|
||||
vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
|
||||
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d fsync:%d level:%d "
|
||||
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d s3KeepLocal:%d "
|
||||
"s3Compact:%d fsync:%d level:%d "
|
||||
"walRetentionPeriod:%d walRetentionSize:%d",
|
||||
TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
|
||||
req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
|
||||
req.keepTimeOffset, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize);
|
||||
req.keepTimeOffset, req.s3KeepLocal, req.s3Compact, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod,
|
||||
req.walRetentionSize);
|
||||
|
||||
if (pVnode->config.cacheLastSize != req.cacheLastSize) {
|
||||
pVnode->config.cacheLastSize = req.cacheLastSize;
|
||||
|
|
@ -1982,6 +2008,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
|
|||
pVnode->config.tsdbCfg.minRows = req.minRows;
|
||||
}
|
||||
|
||||
if (req.s3KeepLocal != -1 && req.s3KeepLocal != pVnode->config.s3KeepLocal) {
|
||||
pVnode->config.s3KeepLocal = req.s3KeepLocal;
|
||||
}
|
||||
if (req.s3Compact != -1 && req.s3Compact != pVnode->config.s3Compact) {
|
||||
pVnode->config.s3Compact = req.s3Compact;
|
||||
}
|
||||
|
||||
if (walChanged) {
|
||||
walAlter(pVnode->pWal, &pVnode->config.walCfg);
|
||||
}
|
||||
|
|
@ -2264,4 +2297,5 @@ _OVER:
|
|||
int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
|
||||
return 0;
|
||||
}
|
||||
int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -820,6 +820,16 @@ bool vnodeIsLeader(SVnode *pVnode) {
|
|||
return true;
|
||||
}
|
||||
|
||||
int64_t vnodeClusterId(SVnode *pVnode) {
|
||||
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
|
||||
return syncCfg->nodeInfo[syncCfg->myIndex].clusterId;
|
||||
}
|
||||
|
||||
int32_t vnodeNodeId(SVnode *pVnode) {
|
||||
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
|
||||
return syncCfg->nodeInfo[syncCfg->myIndex].nodeId;
|
||||
}
|
||||
|
||||
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
|
||||
int code = 0;
|
||||
pSnap->lastApplyIndex = pVnode->state.committed;
|
||||
|
|
|
|||
|
|
@ -360,12 +360,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
|
|||
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
|
||||
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
|
||||
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
|
||||
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d",
|
||||
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
|
||||
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
|
||||
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
|
||||
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
|
||||
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
|
||||
pCfg->keepTimeOffset);
|
||||
pCfg->keepTimeOffset, pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
|
||||
|
||||
if (retentions) {
|
||||
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ const char* nodesNodeName(ENodeType type) {
|
|||
return "FlushDatabaseStmt";
|
||||
case QUERY_NODE_TRIM_DATABASE_STMT:
|
||||
return "TrimDatabaseStmt";
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
|
||||
return "S3MigrateDatabaseStmt";
|
||||
case QUERY_NODE_CREATE_TABLE_STMT:
|
||||
return "CreateTableStmt";
|
||||
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
|
||||
|
|
@ -825,7 +827,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
|
|||
|
||||
static const char* jkProjectLogicPlanProjections = "Projections";
|
||||
static const char* jkProjectLogicPlanIgnoreGroupId = "IgnoreGroupId";
|
||||
static const char* jkProjectLogicPlanInputIgnoreGroup= "InputIgnoreGroup";
|
||||
static const char* jkProjectLogicPlanInputIgnoreGroup = "InputIgnoreGroup";
|
||||
|
||||
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
|
||||
|
|
@ -1341,7 +1343,6 @@ static int32_t jsonToLogicDynQueryCtrlNode(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static const char* jkSubplanIdQueryId = "QueryId";
|
||||
static const char* jkSubplanIdGroupId = "GroupId";
|
||||
static const char* jkSubplanIdSubplanId = "SubplanId";
|
||||
|
|
@ -1848,8 +1849,6 @@ static int32_t jsonToFuncType(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
|
||||
|
||||
|
|
@ -1863,7 +1862,7 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddTArray(pJson, jkLastRowScanPhysiPlanFuncTypes, funcTypeToJson, pNode->pFuncTypes);
|
||||
}
|
||||
|
||||
|
|
@ -2257,7 +2256,6 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
|
||||
SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj;
|
||||
|
||||
|
|
@ -2292,7 +2290,6 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static const char* jkAggPhysiPlanExprs = "Exprs";
|
||||
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
|
||||
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
|
||||
|
|
@ -2506,7 +2503,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
|
|||
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
|
||||
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
|
@ -3189,7 +3186,6 @@ static const char* jkGroupCachePhysiPlanGroupByUid = "GroupByUid";
|
|||
static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup";
|
||||
static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch";
|
||||
|
||||
|
||||
static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj;
|
||||
|
||||
|
|
@ -3323,8 +3319,6 @@ static int32_t jsonToPhysiDynQueryCtrlNode(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static const char* jkQueryNodeAddrId = "Id";
|
||||
static const char* jkQueryNodeAddrInUse = "InUse";
|
||||
static const char* jkQueryNodeAddrNumOfEps = "NumOfEps";
|
||||
|
|
@ -4851,6 +4845,10 @@ static const char* jkDatabaseOptionsNumOfVgroups = "NumOfVgroups";
|
|||
static const char* jkDatabaseOptionsSingleStable = "SingleStable";
|
||||
static const char* jkDatabaseOptionsRetentions = "Retentions";
|
||||
static const char* jkDatabaseOptionsSchemaless = "Schemaless";
|
||||
static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize";
|
||||
static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode";
|
||||
static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal";
|
||||
static const char* jkDatabaseOptionsS3Compact = "S3Compact";
|
||||
|
||||
static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
|
||||
const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj;
|
||||
|
|
@ -4910,6 +4908,18 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3ChunkSize, pNode->s3ChunkSize);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkDatabaseOptionsS3KeepLocalNode, nodeToJson, pNode->s3KeepLocalStr);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3KeepLocal, pNode->s3KeepLocal);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -4972,6 +4982,18 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3ChunkSize, &pNode->s3ChunkSize);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeObject(pJson, jkDatabaseOptionsS3KeepLocalNode, (SNode**)&pNode->s3KeepLocalStr);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3KeepLocal, &pNode->s3KeepLocal);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -5580,6 +5602,24 @@ static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
static const char* jkS3MigrateDatabaseStmtDbName = "DbName";
|
||||
|
||||
static int32_t s3migrateDatabaseStmtToJson(const void* pObj, SJson* pJson) {
|
||||
const SS3MigrateDatabaseStmt* pNode = (const SS3MigrateDatabaseStmt*)pObj;
|
||||
|
||||
int32_t code = tjsonAddStringToObject(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t jsonToS3MigrateDatabaseStmt(const SJson* pJson, void* pObj) {
|
||||
SS3MigrateDatabaseStmt* pNode = (SS3MigrateDatabaseStmt*)pObj;
|
||||
|
||||
int32_t code = tjsonGetStringValue(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
static const char* jkCreateTableStmtDbName = "DbName";
|
||||
static const char* jkCreateTableStmtTableName = "TableName";
|
||||
static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists";
|
||||
|
|
@ -6249,10 +6289,6 @@ static int32_t jsonToRestoreVnodeStmt(const SJson* pJson, void* pObj) {
|
|||
return jsonToRestoreComponentNodeStmt(pJson, pObj);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static const char* jkCreateTopicStmtTopicName = "TopicName";
|
||||
static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
|
||||
static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
|
||||
|
|
@ -6908,7 +6944,6 @@ static int32_t showCreateViewStmtToJson(const void* pObj, SJson* pJson) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
|
||||
SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj;
|
||||
|
||||
|
|
@ -6920,7 +6955,6 @@ static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static const char* jkShowTableDistributedStmtDbName = "DbName";
|
||||
static const char* jkShowTableDistributedStmtTableName = "TableName";
|
||||
|
||||
|
|
@ -7168,6 +7202,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
|
|||
return alterDatabaseStmtToJson(pObj, pJson);
|
||||
case QUERY_NODE_TRIM_DATABASE_STMT:
|
||||
return trimDatabaseStmtToJson(pObj, pJson);
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
|
||||
return s3migrateDatabaseStmtToJson(pObj, pJson);
|
||||
case QUERY_NODE_CREATE_TABLE_STMT:
|
||||
return createTableStmtToJson(pObj, pJson);
|
||||
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
|
||||
|
|
@ -7509,6 +7545,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
|
|||
return jsonToAlterDatabaseStmt(pJson, pObj);
|
||||
case QUERY_NODE_TRIM_DATABASE_STMT:
|
||||
return jsonToTrimDatabaseStmt(pJson, pObj);
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
|
||||
return jsonToS3MigrateDatabaseStmt(pJson, pObj);
|
||||
case QUERY_NODE_CREATE_TABLE_STMT:
|
||||
return jsonToCreateTableStmt(pJson, pObj);
|
||||
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
|
||||
|
|
|
|||
|
|
@ -326,6 +326,8 @@ SNode* nodesMakeNode(ENodeType type) {
|
|||
return makeNode(type, sizeof(SFlushDatabaseStmt));
|
||||
case QUERY_NODE_TRIM_DATABASE_STMT:
|
||||
return makeNode(type, sizeof(STrimDatabaseStmt));
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
|
||||
return makeNode(type, sizeof(SS3MigrateDatabaseStmt));
|
||||
case QUERY_NODE_CREATE_TABLE_STMT:
|
||||
return makeNode(type, sizeof(SCreateTableStmt));
|
||||
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
|
||||
|
|
@ -809,6 +811,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_DATABASE_OPTIONS: {
|
||||
SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode;
|
||||
nodesDestroyNode((SNode*)pOptions->pDaysPerFile);
|
||||
nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr);
|
||||
nodesDestroyList(pOptions->pKeep);
|
||||
nodesDestroyList(pOptions->pRetentions);
|
||||
break;
|
||||
|
|
@ -950,6 +953,8 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field
|
||||
case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field
|
||||
break;
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT: // no pointer field
|
||||
break;
|
||||
case QUERY_NODE_CREATE_TABLE_STMT: {
|
||||
SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode;
|
||||
nodesDestroyList(pStmt->pCols);
|
||||
|
|
|
|||
|
|
@ -64,6 +64,9 @@ typedef enum EDatabaseOptionType {
|
|||
DB_OPTION_STT_TRIGGER,
|
||||
DB_OPTION_TABLE_PREFIX,
|
||||
DB_OPTION_TABLE_SUFFIX,
|
||||
DB_OPTION_S3_CHUNKSIZE,
|
||||
DB_OPTION_S3_KEEPLOCAL,
|
||||
DB_OPTION_S3_COMPACT,
|
||||
DB_OPTION_KEEP_TIME_OFFSET
|
||||
} EDatabaseOptionType;
|
||||
|
||||
|
|
@ -98,7 +101,7 @@ typedef struct STokenPair {
|
|||
|
||||
typedef struct SShowTablesOption {
|
||||
EShowKind kind;
|
||||
SToken dbName;
|
||||
SToken dbName;
|
||||
} SShowTablesOption;
|
||||
|
||||
extern SToken nil_token;
|
||||
|
|
@ -119,37 +122,37 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
|
|||
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
|
||||
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
|
||||
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
|
||||
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
|
||||
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
|
||||
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
|
||||
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
|
||||
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
|
||||
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
|
||||
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
|
||||
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
|
||||
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
|
||||
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
|
||||
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
|
||||
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
|
||||
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
|
||||
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
|
||||
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
|
||||
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
|
||||
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
|
||||
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
|
||||
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
|
||||
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
|
||||
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
|
||||
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
|
||||
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
|
||||
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
|
||||
SNode* pFill);
|
||||
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
|
||||
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
|
||||
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
|
||||
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
|
||||
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
|
||||
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
|
||||
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
|
||||
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
|
||||
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
|
||||
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
|
||||
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
|
||||
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
|
||||
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
|
||||
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
|
||||
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
|
||||
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
|
||||
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
|
||||
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
|
||||
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
|
||||
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
|
||||
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
|
||||
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
|
||||
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
|
||||
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
|
||||
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
|
||||
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
|
||||
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
|
||||
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
|
||||
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
|
||||
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
|
||||
SNode* pFill);
|
||||
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
|
||||
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
|
||||
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
|
||||
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
|
||||
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
|
||||
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
|
||||
|
||||
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
|
||||
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
|
||||
|
|
@ -162,7 +165,8 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
|
|||
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
|
||||
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
|
||||
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
|
||||
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
|
||||
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
|
||||
SNodeList* pHint);
|
||||
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
|
||||
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
|
||||
|
||||
|
|
@ -178,6 +182,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo
|
|||
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
|
||||
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
|
||||
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
|
||||
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
|
||||
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd);
|
||||
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
|
||||
SNode* createAlterTableOptions(SAstCreateContext* pCxt);
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ cmd ::= USE db_name(A).
|
|||
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
|
||||
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
|
||||
cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
|
||||
cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); }
|
||||
cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); }
|
||||
|
||||
%type not_exists_opt { bool }
|
||||
|
|
@ -260,6 +261,10 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
|
|||
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
|
||||
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
|
||||
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
|
||||
db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); }
|
||||
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
|
||||
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
|
||||
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }
|
||||
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
|
||||
|
||||
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
|
||||
|
|
@ -291,6 +296,9 @@ alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C).
|
|||
t.n = (C.z + C.n) - B.z;
|
||||
A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t;
|
||||
}
|
||||
alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
|
||||
alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
|
||||
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
|
||||
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
|
||||
|
||||
%type integer_list { SNodeList* }
|
||||
|
|
|
|||
|
|
@ -228,7 +228,6 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
|
|||
return true;
|
||||
}
|
||||
|
||||
|
||||
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
|
||||
trimEscape(pStreamName);
|
||||
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
|
||||
|
|
@ -918,7 +917,6 @@ SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewNam
|
|||
return (SNode*)pView;
|
||||
}
|
||||
|
||||
|
||||
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
|
||||
|
|
@ -1263,6 +1261,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
|
|||
pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
|
||||
pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX;
|
||||
pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX;
|
||||
pOptions->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
|
||||
pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
|
||||
pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT;
|
||||
pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
|
||||
return (SNode*)pOptions;
|
||||
}
|
||||
|
|
@ -1299,6 +1300,9 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
|
|||
pOptions->sstTrigger = -1;
|
||||
pOptions->tablePrefix = -1;
|
||||
pOptions->tableSuffix = -1;
|
||||
pOptions->s3ChunkSize = -1;
|
||||
pOptions->s3KeepLocal = -1;
|
||||
pOptions->s3Compact = -1;
|
||||
pOptions->withArbitrator = -1;
|
||||
return (SNode*)pOptions;
|
||||
}
|
||||
|
|
@ -1415,6 +1419,21 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
|
|||
nodesDestroyNode((SNode*)pNode);
|
||||
break;
|
||||
}
|
||||
case DB_OPTION_S3_CHUNKSIZE:
|
||||
pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
|
||||
break;
|
||||
case DB_OPTION_S3_KEEPLOCAL: {
|
||||
SToken* pToken = pVal;
|
||||
if (TK_NK_INTEGER == pToken->type) {
|
||||
pDbOptions->s3KeepLocal = taosStr2Int32(pToken->z, NULL, 10) * 1440;
|
||||
} else {
|
||||
pDbOptions->s3KeepLocalStr = (SValueNode*)createDurationValueNode(pCxt, pToken);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case DB_OPTION_S3_COMPACT:
|
||||
pDbOptions->s3Compact = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
|
||||
break;
|
||||
case DB_OPTION_KEEP_TIME_OFFSET: {
|
||||
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
|
||||
break;
|
||||
|
|
@ -1501,6 +1520,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t
|
|||
return (SNode*)pStmt;
|
||||
}
|
||||
|
||||
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
if (!checkDbName(pCxt, pDbName, false)) {
|
||||
return NULL;
|
||||
}
|
||||
SS3MigrateDatabaseStmt* pStmt = (SS3MigrateDatabaseStmt*)nodesMakeNode(QUERY_NODE_S3MIGRATE_DATABASE_STMT);
|
||||
CHECK_OUT_OF_MEM(pStmt);
|
||||
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
|
||||
return (SNode*)pStmt;
|
||||
}
|
||||
|
||||
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
if (!checkDbName(pCxt, pDbName, false)) {
|
||||
|
|
@ -1864,7 +1894,8 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD
|
|||
return (SNode*)pStmt;
|
||||
}
|
||||
|
||||
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType) {
|
||||
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName,
|
||||
EOperatorType tableCondType) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
SNode* pDbName = NULL;
|
||||
if (option.dbName.type == TK_NK_NIL) {
|
||||
|
|
@ -1940,7 +1971,6 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode*
|
|||
return (SNode*)pStmt;
|
||||
}
|
||||
|
||||
|
||||
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) {
|
||||
CHECK_PARSER_STATUS(pCxt);
|
||||
SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT);
|
||||
|
|
|
|||
|
|
@ -305,6 +305,10 @@ static SKeyword keywordTable[] = {
|
|||
{"_WSTART", TK_WSTART},
|
||||
{"ALIVE", TK_ALIVE},
|
||||
{"VARBINARY", TK_VARBINARY},
|
||||
{"S3_CHUNKSIZE", TK_S3_CHUNKSIZE},
|
||||
{"S3_KEEPLOCAL", TK_S3_KEEPLOCAL},
|
||||
{"S3_COMPACT", TK_S3_COMPACT},
|
||||
{"S3MIGRATE", TK_S3MIGRATE},
|
||||
{"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET},
|
||||
{"ENCODE", TK_ENCODE},
|
||||
{"COMPRESS", TK_COMPRESS},
|
||||
|
|
|
|||
|
|
@ -2481,6 +2481,20 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF
|
|||
|
||||
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
|
||||
SNode* pParam = NULL;
|
||||
if (strcmp((*pFunc)->functionName, "tbname") == 0 && (*pFunc)->pParameterList != NULL) {
|
||||
pParam = nodesListGetNode((*pFunc)->pParameterList, 0);
|
||||
if (pParam && nodeType(pParam) == QUERY_NODE_VALUE) {
|
||||
if (pCxt && pCxt->pCurrStmt && pCxt->pCurrStmt->type == QUERY_NODE_SELECT_STMT &&
|
||||
((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
|
||||
nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) == QUERY_NODE_REAL_TABLE) {
|
||||
SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pCxt->pCurrStmt)->pFromTable;
|
||||
if (strcmp(((SValueNode*)pParam)->literal, pRealTable->table.tableName) == 0) {
|
||||
NODES_DESTORY_LIST((*pFunc)->pParameterList);
|
||||
(*pFunc)->pParameterList = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
FOREACH(pParam, (*pFunc)->pParameterList) {
|
||||
if (isMultiResFunc(pParam)) {
|
||||
pCxt->errCode = TSDB_CODE_FUNC_FUNTION_PARA_NUM;
|
||||
|
|
@ -2727,11 +2741,13 @@ static bool hasTbnameFunction(SNodeList* pPartitionByList) {
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool fromSubtable(SNode* table) {
|
||||
static bool fromSingleTable(SNode* table) {
|
||||
if (NULL == table) return false;
|
||||
if (table->type == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)table)->pMeta &&
|
||||
((SRealTableNode*)table)->pMeta->tableType == TSDB_CHILD_TABLE) {
|
||||
return true;
|
||||
if (table->type == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)table)->pMeta) {
|
||||
int8_t type = ((SRealTableNode*)table)->pMeta->tableType;
|
||||
if (type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE || type == TSDB_SYSTEM_TABLE) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -2830,7 +2846,7 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
|
|||
}
|
||||
SNode* pPartKey = NULL;
|
||||
bool partionByTbname = false;
|
||||
if (fromSubtable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) ||
|
||||
if (fromSingleTable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) ||
|
||||
hasTbnameFunction(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList)) {
|
||||
partionByTbname = true;
|
||||
}
|
||||
|
|
@ -2839,7 +2855,9 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
|
|||
return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode);
|
||||
}
|
||||
}
|
||||
if (partionByTbname && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) {
|
||||
if (partionByTbname &&
|
||||
((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) ||
|
||||
(QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType))) {
|
||||
return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode);
|
||||
}
|
||||
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
|
||||
|
|
@ -3424,9 +3442,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc
|
|||
SNode* pPara = NULL;
|
||||
FOREACH(pPara, pSrcParas) {
|
||||
if (nodesIsStar(pPara)) {
|
||||
code = createAllColumns(pCxt, true, &pExprs);
|
||||
code = createAllColumns(pCxt, !tsMultiResultFunctionStarReturnTags, &pExprs);
|
||||
} else if (nodesIsTableStar(pPara)) {
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pPara, true, &pExprs);
|
||||
code = createTableAllCols(pCxt, (SColumnNode*)pPara, !tsMultiResultFunctionStarReturnTags, &pExprs);
|
||||
} else {
|
||||
code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara));
|
||||
}
|
||||
|
|
@ -3937,7 +3955,8 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
|
|||
}
|
||||
|
||||
static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP;
|
||||
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
|
||||
|
||||
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
|
||||
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
|
||||
|
||||
SValueNode* pInter = (SValueNode*)pInterval->pInterval;
|
||||
|
|
@ -5262,6 +5281,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
|
|||
pReq->hashSuffix = pStmt->pOptions->tableSuffix;
|
||||
pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize;
|
||||
pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset;
|
||||
pReq->s3ChunkSize = pStmt->pOptions->s3ChunkSize;
|
||||
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
|
||||
pReq->s3Compact = pStmt->pOptions->s3Compact;
|
||||
pReq->ignoreExist = pStmt->ignoreExists;
|
||||
pReq->withArbitrator = pStmt->pOptions->withArbitrator;
|
||||
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
|
||||
|
|
@ -5287,6 +5309,22 @@ static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName,
|
|||
return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal);
|
||||
}
|
||||
|
||||
static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
|
||||
if (NULL != pOptions->s3KeepLocalStr) {
|
||||
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->s3KeepLocalStr)) {
|
||||
return pCxt->errCode;
|
||||
}
|
||||
if (TIME_UNIT_MINUTE != pOptions->s3KeepLocalStr->unit && TIME_UNIT_HOUR != pOptions->s3KeepLocalStr->unit &&
|
||||
TIME_UNIT_DAY != pOptions->s3KeepLocalStr->unit) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
|
||||
"Invalid option s3_keeplocal unit: %c, only %c, %c, %c allowed",
|
||||
pOptions->s3KeepLocalStr->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY);
|
||||
}
|
||||
pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr);
|
||||
}
|
||||
return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL);
|
||||
}
|
||||
|
||||
static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
|
||||
if (NULL != pOptions->pDaysPerFile) {
|
||||
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) {
|
||||
|
|
@ -5549,6 +5587,7 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP
|
|||
|
||||
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
|
||||
int32_t daysPerFile = pOptions->daysPerFile;
|
||||
int32_t s3KeepLocal = pOptions->s3KeepLocal;
|
||||
int64_t daysToKeep0 = pOptions->keep[0];
|
||||
if (-1 == daysPerFile && -1 == daysToKeep0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
@ -5565,6 +5604,10 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
|
|||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
|
||||
"Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration");
|
||||
}
|
||||
if (s3KeepLocal > 0 && daysPerFile > s3KeepLocal / 3) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
|
||||
"Invalid parameters, should be s3_keeplocal >= 3 * duration");
|
||||
}
|
||||
|
||||
if ((pOptions->replica == 2) ^ (pOptions->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
|
||||
|
|
@ -5669,9 +5712,19 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = checkDbS3KeepLocalOption(pCxt, pOptions);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = checkOptionsDependency(pCxt, pDbName, pOptions);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code =
|
||||
checkDbRangeOption(pCxt, "s3_chunksize", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, TSDB_MAX_S3_CHUNK_SIZE);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -5877,6 +5930,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
|
|||
pReq->minRows = pStmt->pOptions->minRowsPerBlock;
|
||||
pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod;
|
||||
pReq->walRetentionSize = pStmt->pOptions->walRetentionSize;
|
||||
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
|
||||
pReq->s3Compact = pStmt->pOptions->s3Compact;
|
||||
pReq->withArbitrator = pStmt->pOptions->withArbitrator;
|
||||
return;
|
||||
}
|
||||
|
|
@ -5920,6 +5975,13 @@ static int32_t checkColumnOptions(SNodeList* pList) {
|
|||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
static int32_t translateS3MigrateDatabase(STranslateContext* pCxt, SS3MigrateDatabaseStmt* pStmt) {
|
||||
SS3MigrateDbReq req = {0};
|
||||
SName name = {0};
|
||||
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
|
||||
tNameGetFullDbName(&name, req.db);
|
||||
return buildCmdMsg(pCxt, TDMT_MND_S3MIGRATE_DB, (FSerializeFunc)tSerializeSS3MigrateDbReq, &req);
|
||||
}
|
||||
|
||||
static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) {
|
||||
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SFieldWithOptions));
|
||||
|
|
@ -5950,7 +6012,7 @@ static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calByte
|
|||
FOREACH(pNode, pList) {
|
||||
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
|
||||
SField field = {
|
||||
.type = pCol->dataType.type,
|
||||
.type = pCol->dataType.type,
|
||||
};
|
||||
if (calBytes) {
|
||||
field.bytes = calcTypeBytes(pCol->dataType);
|
||||
|
|
@ -6116,7 +6178,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
|
|||
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
|
||||
}
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code && pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey && colIndex != 1) {
|
||||
if (TSDB_CODE_SUCCESS == code && pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey &&
|
||||
colIndex != 1) {
|
||||
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SECOND_COL_PK);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code && pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey &&
|
||||
|
|
@ -7416,7 +7479,7 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) {
|
|||
static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) {
|
||||
SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId};
|
||||
int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)),
|
||||
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq);
|
||||
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq);
|
||||
tFreeSMCreateQnodeReq(&createReq);
|
||||
return code;
|
||||
}
|
||||
|
|
@ -7440,7 +7503,7 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) {
|
|||
static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) {
|
||||
SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId};
|
||||
int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)),
|
||||
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq);
|
||||
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq);
|
||||
tFreeSDDropQnodeReq(&dropReq);
|
||||
return code;
|
||||
}
|
||||
|
|
@ -7835,7 +7898,7 @@ static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* a
|
|||
static void setColumnDefNodePrimaryKey(SColumnDefNode* pNode, bool isPk) {
|
||||
if (!pNode) return;
|
||||
if (!isPk && !pNode->pOptions) return;
|
||||
if(!pNode->pOptions) {
|
||||
if (!pNode->pOptions) {
|
||||
pNode->pOptions = nodesMakeNode(QUERY_NODE_COLUMN_OPTIONS);
|
||||
}
|
||||
((SColumnOptions*)pNode->pOptions)->bPrimaryKey = isPk;
|
||||
|
|
@ -9346,6 +9409,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
|
|||
case QUERY_NODE_TRIM_DATABASE_STMT:
|
||||
code = translateTrimDatabase(pCxt, (STrimDatabaseStmt*)pNode);
|
||||
break;
|
||||
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
|
||||
code = translateS3MigrateDatabase(pCxt, (SS3MigrateDatabaseStmt*)pNode);
|
||||
break;
|
||||
case QUERY_NODE_CREATE_TABLE_STMT:
|
||||
code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode);
|
||||
break;
|
||||
|
|
@ -10086,7 +10152,7 @@ static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
|
|||
SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot;
|
||||
SSelectStmt* pSelect = NULL;
|
||||
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal,
|
||||
((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
|
||||
((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
|
||||
}
|
||||
|
|
@ -10237,7 +10303,8 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
|
|||
if (pColDef->pOptions) {
|
||||
req.colCmpr.pColCmpr[index].id = index + 1;
|
||||
int8_t valid = setColCompressByOption(
|
||||
pScheam->type, columnEncodeVal(((SColumnOptions*)pColDef->pOptions)->encode), columnCompressVal(((SColumnOptions*)pColDef->pOptions)->compress),
|
||||
pScheam->type, columnEncodeVal(((SColumnOptions*)pColDef->pOptions)->encode),
|
||||
columnCompressVal(((SColumnOptions*)pColDef->pOptions)->compress),
|
||||
columnLevelVal(((SColumnOptions*)pColDef->pOptions)->compressLevel), true, &req.colCmpr.pColCmpr[index].alg);
|
||||
if (!valid) {
|
||||
tdDestroySVCreateTbReq(&req);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -2689,7 +2689,8 @@ static bool isNeedSplitCacheLastFunc(SFunctionNode* pFunc, SScanLogicNode* pScan
|
|||
int32_t funcType = pFunc->funcType;
|
||||
if ((FUNCTION_TYPE_LAST_ROW != funcType || (FUNCTION_TYPE_LAST_ROW == funcType && TSDB_CACHE_MODEL_LAST_VALUE == pScan->cacheLastMode)) &&
|
||||
(FUNCTION_TYPE_LAST != funcType || (FUNCTION_TYPE_LAST == funcType && (TSDB_CACHE_MODEL_LAST_ROW == pScan->cacheLastMode ||
|
||||
QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))))) &&
|
||||
QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) ||
|
||||
COLUMN_TYPE_COLUMN != ((SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0))->colType))) &&
|
||||
FUNCTION_TYPE_SELECT_VALUE != funcType && FUNCTION_TYPE_GROUP_KEY != funcType) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -2709,8 +2710,9 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, int8_t cacheLastModel
|
|||
if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pParam)) {
|
||||
SColumnNode* pCol = (SColumnNode*)pParam;
|
||||
if (pCol->colType != COLUMN_TYPE_COLUMN) {
|
||||
return false;
|
||||
if (pCol->colType != COLUMN_TYPE_COLUMN && TSDB_CACHE_MODEL_LAST_ROW != cacheLastModel) {
|
||||
needSplitFuncCount++;
|
||||
*hasOtherFunc = true;
|
||||
}
|
||||
if (lastColId != pCol->colId) {
|
||||
lastColId = pCol->colId;
|
||||
|
|
@ -2991,6 +2993,13 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
|
|||
}
|
||||
}
|
||||
}
|
||||
FOREACH(pColNode, pScan->pScanPseudoCols) {
|
||||
if (nodesEqualNode(pParamNode, pColNode)) {
|
||||
if (funcType != FUNCTION_TYPE_LAST) {
|
||||
nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3129,6 +3138,10 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg,
|
|||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
code = nodesCollectColumnsFromNode((SNode*)list, NULL, COLLECT_COL_TYPE_TAG, &pScan->pScanPseudoCols);
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
nodesFree(list);
|
||||
bool found = false;
|
||||
FOREACH(pNode, pScan->pScanCols) {
|
||||
|
|
@ -3150,6 +3163,10 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg,
|
|||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
code = createColumnByRewriteExprs(pScan->pScanPseudoCols, &pScan->node.pTargets);
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
return code;
|
||||
}
|
||||
OPTIMIZE_FLAG_CLEAR_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@
|
|||
#include "tgrant.h"
|
||||
#include "tjson.h"
|
||||
#include "tlog.h"
|
||||
#include "tutil.h"
|
||||
#include "tunit.h"
|
||||
#include "tutil.h"
|
||||
|
||||
#define CFG_NAME_PRINT_LEN 24
|
||||
#define CFG_SRC_PRINT_LEN 12
|
||||
|
|
@ -315,19 +315,19 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value,
|
|||
static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) {
|
||||
SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag");
|
||||
if (resetArray) {
|
||||
// reset
|
||||
if (pDebugFlagItem == NULL) return -1;
|
||||
// reset
|
||||
if (pDebugFlagItem == NULL) return -1;
|
||||
|
||||
// logflag names that should 'not' be set by 'debugFlag'
|
||||
// logflag names that should 'not' be set by 'debugFlag'
|
||||
if (pDebugFlagItem->array == NULL) {
|
||||
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
|
||||
if (pDebugFlagItem->array == NULL) {
|
||||
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
|
||||
if (pDebugFlagItem->array == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
taosArrayClear(pDebugFlagItem->array);
|
||||
return 0;
|
||||
}
|
||||
taosArrayClear(pDebugFlagItem->array);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// update
|
||||
|
|
@ -406,8 +406,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
|
|||
case CFG_DTYPE_BOOL: {
|
||||
int32_t ival = (int32_t)atoi(pVal);
|
||||
if (ival != 0 && ival != 1) {
|
||||
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name,
|
||||
cfgDtypeStr(pItem->dtype), ival);
|
||||
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival);
|
||||
terrno = TSDB_CODE_OUT_OF_RANGE;
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -691,6 +690,89 @@ void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
|
|||
*pLen = len;
|
||||
}
|
||||
|
||||
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) {
|
||||
if (dump) {
|
||||
printf(" s3 config");
|
||||
printf("\n");
|
||||
printf("=================================================================");
|
||||
printf("\n");
|
||||
} else {
|
||||
uInfo(" s3 config");
|
||||
uInfo("=================================================================");
|
||||
}
|
||||
|
||||
char src[CFG_SRC_PRINT_LEN + 1] = {0};
|
||||
char name[CFG_NAME_PRINT_LEN + 1] = {0};
|
||||
|
||||
int32_t size = taosArrayGetSize(pCfg->array);
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
|
||||
if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue;
|
||||
if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
|
||||
if (dump && strncmp(pItem->name, "s3", 2) != 0) continue;
|
||||
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
|
||||
for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) {
|
||||
if (src[j] == 0) src[j] = ' ';
|
||||
}
|
||||
|
||||
tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN);
|
||||
for (int32_t j = 0; j < CFG_NAME_PRINT_LEN; ++j) {
|
||||
if (name[j] == 0) name[j] = ' ';
|
||||
}
|
||||
|
||||
switch (pItem->dtype) {
|
||||
case CFG_DTYPE_BOOL:
|
||||
if (dump) {
|
||||
printf("%s %s %u\n", src, name, pItem->bval);
|
||||
} else {
|
||||
uInfo("%s %s %u", src, name, pItem->bval);
|
||||
}
|
||||
|
||||
break;
|
||||
case CFG_DTYPE_INT32:
|
||||
if (dump) {
|
||||
printf("%s %s %d\n", src, name, pItem->i32);
|
||||
} else {
|
||||
uInfo("%s %s %d", src, name, pItem->i32);
|
||||
}
|
||||
break;
|
||||
case CFG_DTYPE_INT64:
|
||||
if (dump) {
|
||||
printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
|
||||
} else {
|
||||
uInfo("%s %s %" PRId64, src, name, pItem->i64);
|
||||
}
|
||||
break;
|
||||
case CFG_DTYPE_DOUBLE:
|
||||
case CFG_DTYPE_FLOAT:
|
||||
if (dump) {
|
||||
printf("%s %s %.2f\n", src, name, pItem->fval);
|
||||
} else {
|
||||
uInfo("%s %s %.2f", src, name, pItem->fval);
|
||||
}
|
||||
break;
|
||||
case CFG_DTYPE_STRING:
|
||||
case CFG_DTYPE_DIR:
|
||||
case CFG_DTYPE_LOCALE:
|
||||
case CFG_DTYPE_CHARSET:
|
||||
case CFG_DTYPE_TIMEZONE:
|
||||
case CFG_DTYPE_NONE:
|
||||
if (dump) {
|
||||
printf("%s %s %s\n", src, name, pItem->str);
|
||||
} else {
|
||||
uInfo("%s %s %s", src, name, pItem->str);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dump) {
|
||||
printf("=================================================================\n");
|
||||
} else {
|
||||
uInfo("=================================================================");
|
||||
}
|
||||
}
|
||||
|
||||
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
|
||||
if (dump) {
|
||||
printf(" global config");
|
||||
|
|
@ -738,7 +820,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
|
|||
break;
|
||||
case CFG_DTYPE_INT64:
|
||||
if (dump) {
|
||||
printf("%s %s %" PRId64"\n", src, name, pItem->i64);
|
||||
printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
|
||||
} else {
|
||||
uInfo("%s %s %" PRId64, src, name, pItem->i64);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,17 +42,17 @@ class TDTestCase:
|
|||
tdSql.query('show create database scd;')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
tdSql.query('show create database scd2;')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd2')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
tdSql.query('show create database scd4')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd4')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
|
||||
self.restartTaosd(1, dbname='scd')
|
||||
|
|
@ -60,17 +60,17 @@ class TDTestCase:
|
|||
tdSql.query('show create database scd;')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
tdSql.query('show create database scd2;')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd2')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
tdSql.query('show create database scd4')
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, 'scd4')
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
|
||||
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
|
||||
|
||||
|
||||
tdSql.execute('drop database scd')
|
||||
|
|
|
|||
|
|
@ -109,6 +109,10 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 2
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 4
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 2
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 4
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py
|
||||
|
|
|
|||
|
|
@ -1655,3 +1655,43 @@
|
|||
fun:start_thread
|
||||
fun:clone
|
||||
}
|
||||
{
|
||||
<insert_a_suppression_name_here>
|
||||
Memcheck:Param
|
||||
write(buf)
|
||||
fun:__libc_write
|
||||
fun:write
|
||||
fun:uv__try_write
|
||||
fun:uv__write
|
||||
fun:uv_write2
|
||||
fun:uvStartSendRespImpl
|
||||
fun:uvStartSendResp
|
||||
fun:uvHandleResp
|
||||
fun:uvPrepareCb
|
||||
fun:uv__run_prepare
|
||||
fun:uv_run
|
||||
fun:transWorkerThread
|
||||
fun:start_thread
|
||||
fun:clone
|
||||
}
|
||||
|
||||
{
|
||||
<memalign suppression>
|
||||
Memcheck:Param
|
||||
write(buf)
|
||||
fun:__libc_write
|
||||
fun:write
|
||||
fun:uv__try_write
|
||||
fun:uv__write
|
||||
fun:uv_write2
|
||||
fun:uvStartSendRespImpl
|
||||
fun:uvStartSendResp
|
||||
fun:uvHandleResp
|
||||
fun:uvWorkerAsyncCb
|
||||
fun:uv__async_io
|
||||
fun:uv__io_poll
|
||||
fun:uv_run
|
||||
fun:transWorkerThread
|
||||
fun:start_thread
|
||||
fun:clone
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ print ============= create database
|
|||
# | REPLICA value [1 | 3]
|
||||
# | WAL_LEVEL value [1 | 2]
|
||||
|
||||
sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1
|
||||
sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 S3_KEEPLOCAL 720 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1
|
||||
sql select * from information_schema.ins_databases
|
||||
print rows: $rows
|
||||
print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
system sh/stop_dnodes.sh
|
||||
system sh/deploy.sh -n dnode1 -i 1
|
||||
system sh/exec.sh -n dnode1 -s start
|
||||
sql connect
|
||||
|
||||
sql alter local "multiResultFunctionStarReturnTags" "0";
|
||||
|
||||
print step1=====================
|
||||
sql drop database if exists test;
|
||||
sql create database test vgroups 4 CACHEMODEL 'both';
|
||||
sql use test;
|
||||
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
|
||||
sql create table t1 using st tags(1,1,1);
|
||||
sql create table t2 using st tags(2,2,2);
|
||||
sql create table t3 using st tags(3,3,3);
|
||||
sql create table t4 using st tags(NULL,4,4);
|
||||
|
||||
sql insert into t1 values(1648791211000,1,1,1);
|
||||
sql insert into t1 values(1648791211001,2,2,2);
|
||||
sql insert into t2 values(1648791211002,3,3,3);
|
||||
sql insert into t2 values(1648791211003,4,4,4);
|
||||
sql insert into t3 values(1648791211004,5,5,5);
|
||||
sql insert into t3 values(1648791211005,6,6,6);
|
||||
sql insert into t4 values(1648791211007,NULL,NULL,NULL);
|
||||
|
||||
sql select last(*),last_row(*) from st;
|
||||
|
||||
if $cols != 8 then
|
||||
print ======cols=$cols
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql alter local "multiResultFunctionStarReturnTags" "1";
|
||||
|
||||
sql select last(*),last_row(*) from st;
|
||||
|
||||
if $cols != 14 then
|
||||
print ======cols=$cols
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last_row(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*),last_row(*) from t1;
|
||||
|
||||
if $cols != 8 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print step2=====================
|
||||
|
||||
sql drop database if exists test1;
|
||||
sql create database test1 vgroups 4 CACHEMODEL 'last_row';
|
||||
sql use test1;
|
||||
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
|
||||
sql create table t1 using st tags(1,1,1);
|
||||
sql create table t2 using st tags(2,2,2);
|
||||
sql create table t3 using st tags(3,3,3);
|
||||
sql create table t4 using st tags(NULL,4,4);
|
||||
|
||||
sql insert into t1 values(1648791211000,1,1,1);
|
||||
sql insert into t1 values(1648791211001,2,2,2);
|
||||
sql insert into t2 values(1648791211002,3,3,3);
|
||||
sql insert into t2 values(1648791211003,4,4,4);
|
||||
sql insert into t3 values(1648791211004,5,5,5);
|
||||
sql insert into t3 values(1648791211005,6,6,6);
|
||||
sql insert into t4 values(1648791211007,NULL,NULL,NULL);
|
||||
|
||||
sql select last(*),last_row(*) from st;
|
||||
|
||||
if $cols != 14 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
return -1
|
||||
|
||||
sql select last_row(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*),last_row(*) from t1;
|
||||
|
||||
if $cols != 8 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print step3=====================
|
||||
|
||||
sql drop database if exists test2;
|
||||
sql create database test2 vgroups 4 CACHEMODEL 'last_value';
|
||||
sql use test2;
|
||||
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
|
||||
sql create table t1 using st tags(1,1,1);
|
||||
sql create table t2 using st tags(2,2,2);
|
||||
sql create table t3 using st tags(3,3,3);
|
||||
sql create table t4 using st tags(NULL,4,4);
|
||||
|
||||
sql insert into t1 values(1648791211000,1,1,1);
|
||||
sql insert into t1 values(1648791211001,2,2,2);
|
||||
sql insert into t2 values(1648791211002,3,3,3);
|
||||
sql insert into t2 values(1648791211003,4,4,4);
|
||||
sql insert into t3 values(1648791211004,5,5,5);
|
||||
sql insert into t3 values(1648791211005,6,6,6);
|
||||
sql insert into t4 values(1648791211007,NULL,NULL,NULL);
|
||||
|
||||
sql select last(*),last_row(*) from st;
|
||||
|
||||
if $cols != 14 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last_row(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*),last_row(*) from t1;
|
||||
|
||||
if $cols != 8 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql drop database if exists test4;
|
||||
sql create database test4 vgroups 4;
|
||||
sql use test4;
|
||||
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
|
||||
sql create table t1 using st tags(1,1,1);
|
||||
sql create table t2 using st tags(2,2,2);
|
||||
sql create table t3 using st tags(3,3,3);
|
||||
sql create table t4 using st tags(NULL,4,4);
|
||||
|
||||
sql insert into t1 values(1648791211000,1,1,1);
|
||||
sql insert into t1 values(1648791211001,2,2,2);
|
||||
sql insert into t2 values(1648791211002,3,3,3);
|
||||
sql insert into t2 values(1648791211003,4,4,4);
|
||||
sql insert into t3 values(1648791211004,5,5,5);
|
||||
sql insert into t3 values(1648791211005,6,6,6);
|
||||
sql insert into t4 values(1648791211007,NULL,NULL,NULL);
|
||||
|
||||
sql select last(*),last_row(*) from st;
|
||||
|
||||
if $cols != 14 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last_row(*) from st;
|
||||
|
||||
if $cols != 7 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select last(*),last_row(*) from t1;
|
||||
|
||||
if $cols != 8 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
|
@ -18,7 +18,7 @@ class TDTestCase:
|
|||
def prepare_datas_of_distribute(self, dbname="testdb"):
|
||||
|
||||
# prepate datas for 20 tables distributed at different vgroups
|
||||
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5")
|
||||
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 s3_keeplocal 3000 vgroups 5")
|
||||
tdSql.execute(f" use {dbname} ")
|
||||
tdSql.execute(
|
||||
f'''create table {dbname}.stb1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import taos
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
|
||||
|
||||
class TDTestCase:
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def td29092(self, dbname="db"):
|
||||
tdSql.execute(f'use {dbname}')
|
||||
tdSql.execute('CREATE STABLE `st` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT);')
|
||||
tdSql.execute('CREATE STABLE `st2` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT);')
|
||||
tdSql.execute('CREATE TABLE `t1` USING `st` (`t1`) TAGS (1);')
|
||||
tdSql.execute('CREATE TABLE `t2` USING `st` (`t1`) TAGS (2);')
|
||||
tdSql.execute('CREATE TABLE `t21` USING `st2` (`t1`) TAGS (21);')
|
||||
tdSql.execute('CREATE TABLE `nt` (`ts` TIMESTAMP, `v1` INT);')
|
||||
|
||||
now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
|
||||
for i in range(3):
|
||||
tdSql.execute(
|
||||
f"insert into {dbname}.t1 values ( { now_time + i * 1000 }, {i} )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into {dbname}.t2 values ( { now_time + i * 1000 }, {i} )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into {dbname}.nt values ( { now_time + i * 1000 }, {i} )"
|
||||
)
|
||||
|
||||
tdLog.debug(f"-------------- step1: normal table test ------------------")
|
||||
tdSql.query("select tbname, count(*) from nt;")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, "nt")
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select nt.tbname, count(*) from nt;")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, "nt")
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from nt group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, "nt")
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select nt.tbname, count(*) from nt group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, "nt")
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select nt.tbname, count(*) from nt group by nt.tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 0, "nt")
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdLog.debug(f"-------------- step2: system table test ------------------")
|
||||
tdSql.query("select tbname, count(*) from information_schema.ins_dnodes")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
|
||||
tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
|
||||
tdSql.query("select tbname, count(*) from information_schema.ins_dnodes group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
|
||||
tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
|
||||
tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes group by ins_dnodes.tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
|
||||
tdLog.debug(f"-------------- step3: subtable test ------------------")
|
||||
tdSql.query("select tbname, count(*) from t1")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select t1.tbname, count(*) from t1")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from t1 group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select t1.tbname, count(*) from t1 group by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select t1.tbname, count(*) from t1 group by t1.tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.error("select t1.tbname, count(*) from t2 group by t1.tbname")
|
||||
tdSql.error("select t1.tbname, count(*) from t1 group by t2.tbname")
|
||||
tdSql.error("select t2.tbname, count(*) from t1 group by t1.tbname")
|
||||
|
||||
tdLog.debug(f"-------------- step4: super table test ------------------")
|
||||
tdSql.query("select tbname, count(*) from st group by tbname")
|
||||
tdSql.checkRows(2)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
tdSql.checkData(1, 1, 3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from st partition by tbname")
|
||||
tdSql.checkRows(2)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
tdSql.checkData(1, 1, 3)
|
||||
|
||||
tdSql.query("select ts, t1 from st where st.tbname=\"t1\"")
|
||||
tdSql.checkRows(3)
|
||||
tdSql.checkData(0, 1, 1)
|
||||
tdSql.checkData(1, 1, 1)
|
||||
tdSql.checkData(2, 1, 1)
|
||||
|
||||
tdSql.query("select tbname, ts from st where tbname=\"t2\"")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
tdSql.query("select tbname, ts from st where tbname=\"t2\" order by tbname")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
tdSql.query("select tbname, ts from st where tbname=\"t2\" order by st.tbname")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from st where tbname=\"t2\" group by tbname order by tbname")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from st group by tbname order by tbname")
|
||||
tdSql.checkRows(2)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
tdSql.checkData(1, 1, 3)
|
||||
|
||||
tdSql.query("select tbname, count(*) from st group by st.tbname order by st.tbname")
|
||||
tdSql.checkRows(2)
|
||||
tdSql.checkData(0, 1, 3)
|
||||
tdSql.checkData(1, 1, 3)
|
||||
|
||||
tdLog.debug(f"-------------- step4: join test ------------------")
|
||||
tdSql.query("select t1.tbname, t2.tbname from t1, t2 where t1.ts=t2.ts and t1.tbname!=t2.tbname")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
tdSql.query("select t1.tbname, t2.tbname from t1, t2 where t1.ts=t2.ts and t1.tbname!=t2.tbname order by t1.tbname")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
tdSql.query("select st.tbname, st2.tbname from st, st2 where st.ts=st2.ts and st.tbname!=st2.tbname order by st.tbname")
|
||||
tdSql.checkRows(0)
|
||||
|
||||
tdSql.execute(f"insert into t21 values ( { now_time + 1000 }, 1 )")
|
||||
tdSql.query("select st.tbname, st2.tbname from st, st2 where st.ts=st2.ts and st.tbname!=st2.tbname order by st.tbname")
|
||||
tdSql.checkRows(2)
|
||||
|
||||
tdSql.query("select t1.tbname, st2.tbname from t1, st2 where t1.ts=st2.ts and t1.tbname!=st2.tbname order by t1.tbname")
|
||||
tdSql.checkRows(1)
|
||||
|
||||
tdSql.query("select nt.ts, st.tbname from nt, st where nt.ts=st.ts order by st.tbname")
|
||||
tdSql.checkRows(6)
|
||||
|
||||
tdSql.query("select nt.ts, t1.tbname from nt, t1 where nt.ts=t1.ts order by t1.tbname")
|
||||
tdSql.checkRows(3)
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
self.td29092()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
|
|
@ -214,6 +214,7 @@ SWords shellCommands[] = {
|
|||
{"insert into <tb_name> using <stb_name> <anyword> values(", 0, 0, NULL},
|
||||
{"insert into <tb_name> file ", 0, 0, NULL},
|
||||
{"trim database <db_name>", 0, 0, NULL},
|
||||
{"s3migrate database <db_name>", 0, 0, NULL},
|
||||
{"use <db_name>", 0, 0, NULL},
|
||||
{"quit", 0, 0, NULL}};
|
||||
|
||||
|
|
@ -283,6 +284,9 @@ char* db_options[] = {"keep ",
|
|||
"wal_level ",
|
||||
"vgroups ",
|
||||
"single_stable ",
|
||||
"s3_chunksize ",
|
||||
"s3_keeplocal ",
|
||||
"s3_compact ",
|
||||
"wal_retention_period ",
|
||||
"wal_roll_period ",
|
||||
"wal_retention_size ",
|
||||
|
|
@ -290,6 +294,7 @@ char* db_options[] = {"keep ",
|
|||
|
||||
char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ",
|
||||
"wal_retention_period ", "wal_retention_size ", "cachesize ",
|
||||
"s3_keeplocal ", "s3_compact ",
|
||||
"wal_fsync_period ", "buffer ", "pages " ,"wal_level "};
|
||||
|
||||
char* data_types[] = {"timestamp", "int",
|
||||
|
|
|
|||
Loading…
Reference in New Issue