Merge pull request #23036 from taosdata/feat/TS-4011-3.0

feat/TS-4011: adjust config 'keep_time_offset' level from dnode to db
This commit is contained in:
wade zhang 2023-09-26 14:25:54 +08:00 committed by GitHub
commit f6134b42d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 5573 additions and 5501 deletions

View File

@ -731,16 +731,6 @@ The charset that takes effect is UTF-8.
| Value Range | 0: not change; 1: change by modification |
| Default Value | 0 |
### keepTimeOffset
| Attribute | Description |
| ------------- | ------------------------- |
| Applicable | Server Only |
| Meaning | Latency of data migration |
| Unit | hour |
| Value Range | 0-23 |
| Default Value | 0 |
### tmqMaxTopicNum
| Attribute | Description |
@ -807,4 +797,4 @@ The charset that takes effect is UTF-8.
| 53 | udf | Yes | Yes | |
| 54 | enableCoreFile | Yes | Yes | |
| 55 | ttlChangeOnWrite | No | Yes | |
| 56 | keepTimeOffset | Yes | Yes | |
| 56 | keepTimeOffset | Yes | Yes(discarded since 3.2.0.0) | |

View File

@ -716,16 +716,6 @@ charset 的有效值是 UTF-8。
| 取值范围 | 0: 不改变1改变 |
| 缺省值 | 0 |
### keepTimeOffset
| 属性 | 说明 |
| -------- | -------------- |
| 适用范围 | 仅服务端适用 |
| 含义 | 迁移操作的延时 |
| 单位 | 小时 |
| 取值范围 | 0-23 |
| 缺省值 | 0 |
### tmqMaxTopicNum
| 属性 | 说明 |
@ -803,7 +793,7 @@ charset 的有效值是 UTF-8。
| 53 | udf | 是 | 是 | |
| 54 | enableCoreFile | 是 | 是 | |
| 55 | ttlChangeOnWrite | 否 | 是 | |
| 56 | keepTimeOffset | 是 | 是 | |
| 56 | keepTimeOffset | 否 | 是从3.2.0.0开始,该配置废弃) | |
## 2.x->3.0 的废弃参数

View File

@ -208,7 +208,6 @@ extern int32_t tsRpcRetryInterval;
extern bool tsDisableStream;
extern int64_t tsStreamBufferSize;
extern bool tsFilterScalarMode;
extern int32_t tsKeepTimeOffset;
extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold;
extern int32_t tsResolveFQDNRetryTime;

View File

@ -1096,6 +1096,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t minRows;
int32_t maxRows;
int32_t walFsyncPeriod;
@ -1133,6 +1134,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t walFsyncPeriod;
int8_t walLevel;
int8_t strict;
@ -1239,6 +1241,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t minRows;
int32_t maxRows;
int32_t walFsyncPeriod;
@ -1591,6 +1594,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t minRows;
int32_t maxRows;
int32_t walFsyncPeriod;
@ -1670,6 +1674,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t walFsyncPeriod;
int8_t walLevel;
int8_t strict;

View File

@ -115,250 +115,252 @@
#define TK_STT_TRIGGER 96
#define TK_TABLE_PREFIX 97
#define TK_TABLE_SUFFIX 98
#define TK_NK_COLON 99
#define TK_BWLIMIT 100
#define TK_START 101
#define TK_TIMESTAMP 102
#define TK_END 103
#define TK_TABLE 104
#define TK_NK_LP 105
#define TK_NK_RP 106
#define TK_STABLE 107
#define TK_COLUMN 108
#define TK_MODIFY 109
#define TK_RENAME 110
#define TK_TAG 111
#define TK_SET 112
#define TK_NK_EQ 113
#define TK_USING 114
#define TK_TAGS 115
#define TK_BOOL 116
#define TK_TINYINT 117
#define TK_SMALLINT 118
#define TK_INT 119
#define TK_INTEGER 120
#define TK_BIGINT 121
#define TK_FLOAT 122
#define TK_DOUBLE 123
#define TK_BINARY 124
#define TK_NCHAR 125
#define TK_UNSIGNED 126
#define TK_JSON 127
#define TK_VARCHAR 128
#define TK_MEDIUMBLOB 129
#define TK_BLOB 130
#define TK_VARBINARY 131
#define TK_GEOMETRY 132
#define TK_DECIMAL 133
#define TK_COMMENT 134
#define TK_MAX_DELAY 135
#define TK_WATERMARK 136
#define TK_ROLLUP 137
#define TK_TTL 138
#define TK_SMA 139
#define TK_DELETE_MARK 140
#define TK_FIRST 141
#define TK_LAST 142
#define TK_SHOW 143
#define TK_PRIVILEGES 144
#define TK_DATABASES 145
#define TK_TABLES 146
#define TK_STABLES 147
#define TK_MNODES 148
#define TK_QNODES 149
#define TK_FUNCTIONS 150
#define TK_INDEXES 151
#define TK_ACCOUNTS 152
#define TK_APPS 153
#define TK_CONNECTIONS 154
#define TK_LICENCES 155
#define TK_GRANTS 156
#define TK_QUERIES 157
#define TK_SCORES 158
#define TK_TOPICS 159
#define TK_VARIABLES 160
#define TK_CLUSTER 161
#define TK_BNODES 162
#define TK_SNODES 163
#define TK_TRANSACTIONS 164
#define TK_DISTRIBUTED 165
#define TK_CONSUMERS 166
#define TK_SUBSCRIPTIONS 167
#define TK_VNODES 168
#define TK_ALIVE 169
#define TK_NORMAL 170
#define TK_CHILD 171
#define TK_LIKE 172
#define TK_TBNAME 173
#define TK_QTAGS 174
#define TK_AS 175
#define TK_SYSTEM 176
#define TK_INDEX 177
#define TK_FUNCTION 178
#define TK_INTERVAL 179
#define TK_COUNT 180
#define TK_LAST_ROW 181
#define TK_META 182
#define TK_ONLY 183
#define TK_TOPIC 184
#define TK_CONSUMER 185
#define TK_GROUP 186
#define TK_DESC 187
#define TK_DESCRIBE 188
#define TK_RESET 189
#define TK_QUERY 190
#define TK_CACHE 191
#define TK_EXPLAIN 192
#define TK_ANALYZE 193
#define TK_VERBOSE 194
#define TK_NK_BOOL 195
#define TK_RATIO 196
#define TK_NK_FLOAT 197
#define TK_OUTPUTTYPE 198
#define TK_AGGREGATE 199
#define TK_BUFSIZE 200
#define TK_LANGUAGE 201
#define TK_REPLACE 202
#define TK_STREAM 203
#define TK_INTO 204
#define TK_PAUSE 205
#define TK_RESUME 206
#define TK_TRIGGER 207
#define TK_AT_ONCE 208
#define TK_WINDOW_CLOSE 209
#define TK_IGNORE 210
#define TK_EXPIRED 211
#define TK_FILL_HISTORY 212
#define TK_UPDATE 213
#define TK_SUBTABLE 214
#define TK_UNTREATED 215
#define TK_KILL 216
#define TK_CONNECTION 217
#define TK_TRANSACTION 218
#define TK_BALANCE 219
#define TK_VGROUP 220
#define TK_LEADER 221
#define TK_MERGE 222
#define TK_REDISTRIBUTE 223
#define TK_SPLIT 224
#define TK_DELETE 225
#define TK_INSERT 226
#define TK_NULL 227
#define TK_NK_QUESTION 228
#define TK_NK_ARROW 229
#define TK_ROWTS 230
#define TK_QSTART 231
#define TK_QEND 232
#define TK_QDURATION 233
#define TK_WSTART 234
#define TK_WEND 235
#define TK_WDURATION 236
#define TK_IROWTS 237
#define TK_ISFILLED 238
#define TK_CAST 239
#define TK_NOW 240
#define TK_TODAY 241
#define TK_TIMEZONE 242
#define TK_CLIENT_VERSION 243
#define TK_SERVER_VERSION 244
#define TK_SERVER_STATUS 245
#define TK_CURRENT_USER 246
#define TK_CASE 247
#define TK_WHEN 248
#define TK_THEN 249
#define TK_ELSE 250
#define TK_BETWEEN 251
#define TK_IS 252
#define TK_NK_LT 253
#define TK_NK_GT 254
#define TK_NK_LE 255
#define TK_NK_GE 256
#define TK_NK_NE 257
#define TK_MATCH 258
#define TK_NMATCH 259
#define TK_CONTAINS 260
#define TK_IN 261
#define TK_JOIN 262
#define TK_INNER 263
#define TK_SELECT 264
#define TK_NK_HINT 265
#define TK_DISTINCT 266
#define TK_WHERE 267
#define TK_PARTITION 268
#define TK_BY 269
#define TK_SESSION 270
#define TK_STATE_WINDOW 271
#define TK_EVENT_WINDOW 272
#define TK_SLIDING 273
#define TK_FILL 274
#define TK_VALUE 275
#define TK_VALUE_F 276
#define TK_NONE 277
#define TK_PREV 278
#define TK_NULL_F 279
#define TK_LINEAR 280
#define TK_NEXT 281
#define TK_HAVING 282
#define TK_RANGE 283
#define TK_EVERY 284
#define TK_ORDER 285
#define TK_SLIMIT 286
#define TK_SOFFSET 287
#define TK_LIMIT 288
#define TK_OFFSET 289
#define TK_ASC 290
#define TK_NULLS 291
#define TK_ABORT 292
#define TK_AFTER 293
#define TK_ATTACH 294
#define TK_BEFORE 295
#define TK_BEGIN 296
#define TK_BITAND 297
#define TK_BITNOT 298
#define TK_BITOR 299
#define TK_BLOCKS 300
#define TK_CHANGE 301
#define TK_COMMA 302
#define TK_CONCAT 303
#define TK_CONFLICT 304
#define TK_COPY 305
#define TK_DEFERRED 306
#define TK_DELIMITERS 307
#define TK_DETACH 308
#define TK_DIVIDE 309
#define TK_DOT 310
#define TK_EACH 311
#define TK_FAIL 312
#define TK_FILE 313
#define TK_FOR 314
#define TK_GLOB 315
#define TK_ID 316
#define TK_IMMEDIATE 317
#define TK_IMPORT 318
#define TK_INITIALLY 319
#define TK_INSTEAD 320
#define TK_ISNULL 321
#define TK_KEY 322
#define TK_MODULES 323
#define TK_NK_BITNOT 324
#define TK_NK_SEMI 325
#define TK_NOTNULL 326
#define TK_OF 327
#define TK_PLUS 328
#define TK_PRIVILEGE 329
#define TK_RAISE 330
#define TK_RESTRICT 331
#define TK_ROW 332
#define TK_SEMI 333
#define TK_STAR 334
#define TK_STATEMENT 335
#define TK_STRICT 336
#define TK_STRING 337
#define TK_TIMES 338
#define TK_VALUES 339
#define TK_VARIABLE 340
#define TK_VIEW 341
#define TK_WAL 342
#define TK_KEEP_TIME_OFFSET 99
#define TK_NK_COLON 100
#define TK_BWLIMIT 101
#define TK_START 102
#define TK_TIMESTAMP 103
#define TK_END 104
#define TK_TABLE 105
#define TK_NK_LP 106
#define TK_NK_RP 107
#define TK_STABLE 108
#define TK_COLUMN 109
#define TK_MODIFY 110
#define TK_RENAME 111
#define TK_TAG 112
#define TK_SET 113
#define TK_NK_EQ 114
#define TK_USING 115
#define TK_TAGS 116
#define TK_BOOL 117
#define TK_TINYINT 118
#define TK_SMALLINT 119
#define TK_INT 120
#define TK_INTEGER 121
#define TK_BIGINT 122
#define TK_FLOAT 123
#define TK_DOUBLE 124
#define TK_BINARY 125
#define TK_NCHAR 126
#define TK_UNSIGNED 127
#define TK_JSON 128
#define TK_VARCHAR 129
#define TK_MEDIUMBLOB 130
#define TK_BLOB 131
#define TK_VARBINARY 132
#define TK_GEOMETRY 133
#define TK_DECIMAL 134
#define TK_COMMENT 135
#define TK_MAX_DELAY 136
#define TK_WATERMARK 137
#define TK_ROLLUP 138
#define TK_TTL 139
#define TK_SMA 140
#define TK_DELETE_MARK 141
#define TK_FIRST 142
#define TK_LAST 143
#define TK_SHOW 144
#define TK_PRIVILEGES 145
#define TK_DATABASES 146
#define TK_TABLES 147
#define TK_STABLES 148
#define TK_MNODES 149
#define TK_QNODES 150
#define TK_FUNCTIONS 151
#define TK_INDEXES 152
#define TK_ACCOUNTS 153
#define TK_APPS 154
#define TK_CONNECTIONS 155
#define TK_LICENCES 156
#define TK_GRANTS 157
#define TK_QUERIES 158
#define TK_SCORES 159
#define TK_TOPICS 160
#define TK_VARIABLES 161
#define TK_CLUSTER 162
#define TK_BNODES 163
#define TK_SNODES 164
#define TK_TRANSACTIONS 165
#define TK_DISTRIBUTED 166
#define TK_CONSUMERS 167
#define TK_SUBSCRIPTIONS 168
#define TK_VNODES 169
#define TK_ALIVE 170
#define TK_NORMAL 171
#define TK_CHILD 172
#define TK_LIKE 173
#define TK_TBNAME 174
#define TK_QTAGS 175
#define TK_AS 176
#define TK_SYSTEM 177
#define TK_INDEX 178
#define TK_FUNCTION 179
#define TK_INTERVAL 180
#define TK_COUNT 181
#define TK_LAST_ROW 182
#define TK_META 183
#define TK_ONLY 184
#define TK_TOPIC 185
#define TK_CONSUMER 186
#define TK_GROUP 187
#define TK_DESC 188
#define TK_DESCRIBE 189
#define TK_RESET 190
#define TK_QUERY 191
#define TK_CACHE 192
#define TK_EXPLAIN 193
#define TK_ANALYZE 194
#define TK_VERBOSE 195
#define TK_NK_BOOL 196
#define TK_RATIO 197
#define TK_NK_FLOAT 198
#define TK_OUTPUTTYPE 199
#define TK_AGGREGATE 200
#define TK_BUFSIZE 201
#define TK_LANGUAGE 202
#define TK_REPLACE 203
#define TK_STREAM 204
#define TK_INTO 205
#define TK_PAUSE 206
#define TK_RESUME 207
#define TK_TRIGGER 208
#define TK_AT_ONCE 209
#define TK_WINDOW_CLOSE 210
#define TK_IGNORE 211
#define TK_EXPIRED 212
#define TK_FILL_HISTORY 213
#define TK_UPDATE 214
#define TK_SUBTABLE 215
#define TK_UNTREATED 216
#define TK_KILL 217
#define TK_CONNECTION 218
#define TK_TRANSACTION 219
#define TK_BALANCE 220
#define TK_VGROUP 221
#define TK_LEADER 222
#define TK_MERGE 223
#define TK_REDISTRIBUTE 224
#define TK_SPLIT 225
#define TK_DELETE 226
#define TK_INSERT 227
#define TK_NULL 228
#define TK_NK_QUESTION 229
#define TK_NK_ARROW 230
#define TK_ROWTS 231
#define TK_QSTART 232
#define TK_QEND 233
#define TK_QDURATION 234
#define TK_WSTART 235
#define TK_WEND 236
#define TK_WDURATION 237
#define TK_IROWTS 238
#define TK_ISFILLED 239
#define TK_CAST 240
#define TK_NOW 241
#define TK_TODAY 242
#define TK_TIMEZONE 243
#define TK_CLIENT_VERSION 244
#define TK_SERVER_VERSION 245
#define TK_SERVER_STATUS 246
#define TK_CURRENT_USER 247
#define TK_CASE 248
#define TK_WHEN 249
#define TK_THEN 250
#define TK_ELSE 251
#define TK_BETWEEN 252
#define TK_IS 253
#define TK_NK_LT 254
#define TK_NK_GT 255
#define TK_NK_LE 256
#define TK_NK_GE 257
#define TK_NK_NE 258
#define TK_MATCH 259
#define TK_NMATCH 260
#define TK_CONTAINS 261
#define TK_IN 262
#define TK_JOIN 263
#define TK_INNER 264
#define TK_SELECT 265
#define TK_NK_HINT 266
#define TK_DISTINCT 267
#define TK_WHERE 268
#define TK_PARTITION 269
#define TK_BY 270
#define TK_SESSION 271
#define TK_STATE_WINDOW 272
#define TK_EVENT_WINDOW 273
#define TK_SLIDING 274
#define TK_FILL 275
#define TK_VALUE 276
#define TK_VALUE_F 277
#define TK_NONE 278
#define TK_PREV 279
#define TK_NULL_F 280
#define TK_LINEAR 281
#define TK_NEXT 282
#define TK_HAVING 283
#define TK_RANGE 284
#define TK_EVERY 285
#define TK_ORDER 286
#define TK_SLIMIT 287
#define TK_SOFFSET 288
#define TK_LIMIT 289
#define TK_OFFSET 290
#define TK_ASC 291
#define TK_NULLS 292
#define TK_ABORT 293
#define TK_AFTER 294
#define TK_ATTACH 295
#define TK_BEFORE 296
#define TK_BEGIN 297
#define TK_BITAND 298
#define TK_BITNOT 299
#define TK_BITOR 300
#define TK_BLOCKS 301
#define TK_CHANGE 302
#define TK_COMMA 303
#define TK_CONCAT 304
#define TK_CONFLICT 305
#define TK_COPY 306
#define TK_DEFERRED 307
#define TK_DELIMITERS 308
#define TK_DETACH 309
#define TK_DIVIDE 310
#define TK_DOT 311
#define TK_EACH 312
#define TK_FAIL 313
#define TK_FILE 314
#define TK_FOR 315
#define TK_GLOB 316
#define TK_ID 317
#define TK_IMMEDIATE 318
#define TK_IMPORT 319
#define TK_INITIALLY 320
#define TK_INSTEAD 321
#define TK_ISNULL 322
#define TK_KEY 323
#define TK_MODULES 324
#define TK_NK_BITNOT 325
#define TK_NK_SEMI 326
#define TK_NOTNULL 327
#define TK_OF 328
#define TK_PLUS 329
#define TK_PRIVILEGE 330
#define TK_RAISE 331
#define TK_RESTRICT 332
#define TK_ROW 333
#define TK_SEMI 334
#define TK_STAR 335
#define TK_STATEMENT 336
#define TK_STRICT 337
#define TK_STRING 338
#define TK_TIMES 339
#define TK_VALUES 340
#define TK_VARIABLE 341
#define TK_VIEW 342
#define TK_WAL 343
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

View File

@ -66,6 +66,7 @@ typedef struct SDatabaseOptions {
int32_t minRowsPerBlock;
SNodeList* pKeep;
int64_t keep[3];
int32_t keepTimeOffset;
int32_t pages;
int32_t pagesize;
int32_t tsdbPageSize;

View File

@ -322,6 +322,9 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_KEEP_TIME_OFFSET 0
#define TSDB_MAX_KEEP_TIME_OFFSET 23
#define TSDB_DEFAULT_KEEP_TIME_OFFSET 0
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100

View File

@ -108,9 +108,6 @@
# time period of keeping log files, in days
# logKeepDays 0
# unit Hour. Latency of data migration
# keepTimeOffset 0
############ 3. Debug Flag and levels #############################################

View File

@ -106,6 +106,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
};
static const SSysDbTableSchema userFuncSchema[] = {

View File

@ -256,7 +256,6 @@ char tsUdfdLdLibPath[512] = "";
bool tsDisableStream = false;
int64_t tsStreamBufferSize = 128 * 1024 * 1024;
bool tsFilterScalarMode = false;
int32_t tsKeepTimeOffset = 0; // latency of data migration
int tsResolveFQDNRetryTime = 100; // seconds
char tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
@ -664,7 +663,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, 0) != 0) return -1;
@ -1093,7 +1091,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
tsKeepTimeOffset = cfgGetItem(pCfg, "keepTimeOffset")->i32;
tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;
tsPQSortMemThreshold = cfgGetItem(pCfg, "pqSortMemThreshold")->i32;
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
@ -1691,13 +1688,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
return;
}
if (strcasecmp(option, "keepTimeOffset") == 0) {
int32_t newKeepTimeOffset = atoi(value);
uInfo("keepTimeOffset set from %d to %d", tsKeepTimeOffset, newKeepTimeOffset);
tsKeepTimeOffset = newKeepTimeOffset;
return;
}
if (strcasecmp(option, "ttlPushInterval") == 0) {
int32_t newTtlPushInterval = atoi(value);
uInfo("ttlPushInterval set from %d to %d", tsTtlPushIntervalSec, newTtlPushInterval);

View File

@ -2694,6 +2694,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1;
}
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -2756,6 +2757,11 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1;
pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -2793,6 +2799,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
// 2nd modification
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -2836,6 +2843,10 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
pReq->walRetentionPeriod = -1;
pReq->walRetentionSize = -1;
}
pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -3462,6 +3473,7 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -3563,6 +3575,7 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
}
if (tEncodeI8(encoder, pRsp->schemaless) < 0) return -1;
if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1;
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
return 0;
}
@ -3631,6 +3644,10 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
}
if (tDecodeI8(decoder, &pRsp->schemaless) < 0) return -1;
if (tDecodeI16(decoder, &pRsp->sstTrigger) < 0) return -1;
pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1;
}
return 0;
}
@ -4680,6 +4697,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeSReplica(&encoder, pReplica) < 0) return -1;
}
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
tEndEncode(&encoder);
@ -4769,6 +4787,10 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->changeVersion) < 0) return -1;
}
pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -4928,6 +4950,7 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
// 2nd modification
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4974,6 +4997,11 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
if (tDecodeI32(&decoder, &pReq->walRetentionPeriod) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->walRetentionSize) < 0) return -1;
}
pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;

View File

@ -126,6 +126,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
pCfg->tsdbCfg.keep1 = pCreate->daysToKeep1;
pCfg->tsdbCfg.keep2 = pCreate->daysToKeep2;
pCfg->tsdbCfg.keepTimeOffset = pCreate->keepTimeOffset;
pCfg->tsdbCfg.minRows = pCreate->minRows;
pCfg->tsdbCfg.maxRows = pCreate->maxRows;
for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) {
@ -231,15 +232,15 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo(
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d"
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel,
req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod,
req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica,
req.learnerSelfIndex, req.strict, req.changeVersion);

View File

@ -312,6 +312,7 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t keepTimeOffset;
int32_t minRows;
int32_t maxRows;
int32_t walFsyncPeriod;

View File

@ -34,7 +34,7 @@
#include "audit.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 46
#define DB_RESERVE_SIZE 42
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -137,6 +137,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -227,6 +228,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -368,6 +370,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) return -1;
if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
if (pCfg->minRows > pCfg->maxRows) return -1;
@ -413,6 +416,7 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) return -1;
if (pCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
@ -445,6 +449,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP;
if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0;
if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1;
if (pCfg->keepTimeOffset < 0) pCfg->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET;
if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK;
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK;
if (pCfg->walFsyncPeriod < 0) pCfg->walFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
@ -590,6 +595,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.daysToKeep0 = pCreate->daysToKeep0,
.daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2,
.keepTimeOffset = pCreate->keepTimeOffset,
.minRows = pCreate->minRows,
.maxRows = pCreate->maxRows,
.walFsyncPeriod = pCreate->walFsyncPeriod,
@ -763,6 +769,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep0:%d", createReq.daysToKeep0);
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep1:%d", createReq.daysToKeep1);
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep2:%d", createReq.daysToKeep2);
mndBuildAuditDetailInt32(detail, tmp, "keepTimeOffset:%d", createReq.keepTimeOffset);
mndBuildAuditDetailInt32(detail, tmp, "hashPrefix:%d", createReq.hashPrefix);
mndBuildAuditDetailInt32(detail, tmp, "hashSuffix:%d", createReq.hashSuffix);
mndBuildAuditDetailInt32(detail, tmp, "ignoreExist:%d", createReq.ignoreExist);
@ -841,6 +848,11 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = 0;
}
if (pAlter->keepTimeOffset >= 0 && pAlter->keepTimeOffset != pDb->cfg.keepTimeOffset) {
pDb->cfg.keepTimeOffset = pAlter->keepTimeOffset;
terrno = 0;
}
if (pAlter->walFsyncPeriod >= 0 && pAlter->walFsyncPeriod != pDb->cfg.walFsyncPeriod) {
pDb->cfg.walFsyncPeriod = pAlter->walFsyncPeriod;
terrno = 0;
@ -1046,6 +1058,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep0:%d", alterReq.daysToKeep0);
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep1:%d", alterReq.daysToKeep1);
mndBuildAuditDetailInt32(detail, tmp, "daysToKeep2:%d", alterReq.daysToKeep2);
mndBuildAuditDetailInt32(detail, tmp, "keepTimeOffset:%d", alterReq.keepTimeOffset);
mndBuildAuditDetailInt32(detail, tmp, "minRows:%d", alterReq.minRows);
mndBuildAuditDetailInt32(detail, tmp, "pages:%d", alterReq.pages);
mndBuildAuditDetailInt32(detail, tmp, "pageSize:%d", alterReq.pageSize);
@ -1087,6 +1100,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->daysToKeep0 = pDb->cfg.daysToKeep0;
cfgRsp->daysToKeep1 = pDb->cfg.daysToKeep1;
cfgRsp->daysToKeep2 = pDb->cfg.daysToKeep2;
cfgRsp->keepTimeOffset = pDb->cfg.keepTimeOffset;
cfgRsp->minRows = pDb->cfg.minRows;
cfgRsp->maxRows = pDb->cfg.maxRows;
cfgRsp->walFsyncPeriod = pDb->cfg.walFsyncPeriod;
@ -2013,6 +2027,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false);
}
taosMemoryFree(buf);

View File

@ -1191,20 +1191,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
strcpy(dcfgReq.config, "monitor");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strncasecmp(cfgReq.config, "keeptimeoffset", 14) == 0) {
int32_t optLen = strlen("keeptimeoffset");
int32_t flag = -1;
int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
if (code < 0) return code;
if (flag < 0 || flag > 23) {
mError("dnode:%d, failed to config keepTimeOffset since value:%d. Valid range: [0, 23]", cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
}
strcpy(dcfgReq.config, "keeptimeoffset");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
int32_t optLen = strlen("ttlpushinterval");
int32_t flag = -1;

View File

@ -261,6 +261,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
createReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
createReq.minRows = pDb->cfg.minRows;
createReq.maxRows = pDb->cfg.maxRows;
createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
@ -377,6 +378,7 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV
alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
alterReq.daysToKeep2 = pDb->cfg.daysToKeep2;
alterReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
alterReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
alterReq.walLevel = pDb->cfg.walLevel;
alterReq.strict = pDb->cfg.strict;

View File

@ -283,6 +283,7 @@ struct STsdbCfg {
int32_t keep0; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keep1; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX];
};

View File

@ -411,6 +411,7 @@ struct STsdbKeepCfg {
int32_t keep0;
int32_t keep1;
int32_t keep2;
int32_t keepTimeOffset;
};
typedef struct SVCommitSched {

View File

@ -27,6 +27,7 @@ static int32_t rsmaRestore(SSma *pSma);
pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \
pKeepCfg->keepTimeOffset = 0; \
} while (0)
#define SMA_OPEN_RSMA_IMPL(v, l) \

View File

@ -501,7 +501,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SDFileSet *pRSet = NULL;
// memory
pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision);
pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec());
@ -799,7 +798,6 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
// commit file data start
code = tsdbCommitFileDataStart(pCommitter);
TSDB_CHECK_CODE(code, lino, _exit);

View File

@ -23,6 +23,7 @@ int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) {
pKeepCfg->keep0 = pCfg->keep0;
pKeepCfg->keep1 = pCfg->keep1;
pKeepCfg->keep2 = pCfg->keep2;
pKeepCfg->keepTimeOffset = pCfg->keepTimeOffset;
return 0;
}
@ -75,8 +76,9 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
goto _err;
}
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days,
pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2);
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,
pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
pTsdb->keepCfg.keepTimeOffset);
*ppTsdb = pTsdb;
return 0;
@ -89,8 +91,9 @@ _err:
int tsdbClose(STsdb **pTsdb) {
if (*pTsdb) {
STsdb *pdb = *pTsdb;
tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d", TD_VID(pdb->pVnode), pdb->path, pdb->keepCfg.days,
pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2);
tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,
pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2,
pdb->keepCfg.keepTimeOffset);
taosThreadRwlockWrlock(&(*pTsdb)->rwLock);
tsdbMemTableDestroy((*pTsdb)->mem, true);
(*pTsdb)->mem = NULL;

View File

@ -542,7 +542,7 @@ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) {
ASSERT(0);
}
nowSec = nowSec - tsKeepTimeOffset * tsTickPerHour[pKeepCfg->precision];
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];
key = nowSec - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision];
aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);

View File

@ -35,7 +35,8 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.maxRows = 4096,
.keep2 = 5256000,
.keep0 = 5256000,
.keep1 = 5256000},
.keep1 = 5256000,
.keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET},
.walCfg =
{
.vgId = -1,
@ -104,6 +105,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (pCfg->tsdbCfg.retentions[0].freq > 0) {
int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -219,6 +221,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code);
if (code < 0) return -1;
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
if (nRetention > TSDB_RETENTION_MAX) {

View File

@ -1722,11 +1722,11 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d walRetentionPeriod:%d "
"walRetentionSize:%d",
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d fsync:%d level:%d "
"walRetentionPeriod:%d walRetentionSize:%d",
TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize);
req.keepTimeOffset, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize);
if (pVnode->config.cacheLastSize != req.cacheLastSize) {
pVnode->config.cacheLastSize = req.cacheLastSize;
@ -1795,6 +1795,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
}
if (pVnode->config.tsdbCfg.keepTimeOffset != req.keepTimeOffset) {
pVnode->config.tsdbCfg.keepTimeOffset = req.keepTimeOffset;
if (!VND_IS_RSMA(pVnode)) {
tsdbChanged = true;
}
}
if (req.sttTrigger != -1 && req.sttTrigger != pVnode->config.sttTrigger) {
pVnode->config.sttTrigger = req.sttTrigger;
}

View File

@ -312,11 +312,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64,
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d",
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize);
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
pCfg->keepTimeOffset);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

View File

@ -63,7 +63,8 @@ typedef enum EDatabaseOptionType {
DB_OPTION_WAL_SEGMENT_SIZE,
DB_OPTION_STT_TRIGGER,
DB_OPTION_TABLE_PREFIX,
DB_OPTION_TABLE_SUFFIX
DB_OPTION_TABLE_SUFFIX,
DB_OPTION_KEEP_TIME_OFFSET
} EDatabaseOptionType;
typedef enum ETableOptionType {

View File

@ -255,6 +255,7 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }
@ -285,6 +286,7 @@ alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C).
t.n = (C.z + C.n) - B.z;
A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t;
}
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
%type integer_list { SNodeList* }
%destructor integer_list { nodesDestroyList($$); }

View File

@ -1201,6 +1201,10 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
nodesDestroyNode((SNode*)pNode);
break;
}
case DB_OPTION_KEEP_TIME_OFFSET: {
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
}
default:
break;
}

View File

@ -293,6 +293,7 @@ static SKeyword keywordTable[] = {
{"_WSTART", TK_WSTART},
{"ALIVE", TK_ALIVE},
{"VARBINARY", TK_VARBINARY},
{"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET},
};
// clang-format on

View File

@ -4314,6 +4314,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->hashPrefix = pStmt->pOptions->tablePrefix;
pReq->hashSuffix = pStmt->pOptions->tableSuffix;
pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize;
pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset;
pReq->ignoreExist = pStmt->ignoreExists;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
}
@ -4414,6 +4415,17 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbKeepTimeOffsetOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (pOptions->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pOptions->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option keep_time_offset: %d"
" valid range: [%d, %d]",
pOptions->keepTimeOffset, TSDB_MIN_KEEP_TIME_OFFSET, TSDB_MAX_KEEP_TIME_OFFSET);
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbCacheModelOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->cacheModelStr[0]) {
if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_NONE_STR)) {
@ -4586,6 +4598,9 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
if (TSDB_CODE_SUCCESS == code) {
code = checkDbKeepOption(pCxt, pOptions); // use precision
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbKeepTimeOffsetOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE);
}
@ -4707,6 +4722,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
pReq->daysToKeep0 = pStmt->pOptions->keep[0];
pReq->daysToKeep1 = pStmt->pOptions->keep[1];
pReq->daysToKeep2 = pStmt->pOptions->keep[2];
pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset;
pReq->walFsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->walLevel = pStmt->pOptions->walLevel;
pReq->strict = pStmt->pOptions->strict;

File diff suppressed because it is too large Load Diff

View File

@ -42,17 +42,17 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
self.restartTaosd(1, dbname='scd')
@ -60,17 +60,17 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.execute('drop database scd')

View File

@ -217,7 +217,7 @@ class TDTestCase:
tdSql.checkEqual(20470,len(tdSql.queryResult))
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdSql.checkEqual(194, len(tdSql.queryResult))
tdSql.checkEqual(195, len(tdSql.queryResult))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(54, len(tdSql.queryResult))