xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #19872 from taosdata/feat/vnode_compact 

feat: data compaction
This commit is contained in:
Xiaoyu Wang 2023-02-09 16:24:50 +08:00 committed by GitHub
parent 6cf363600b b24d3049f5
commit 86d231ce50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 4722 additions and 3283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/tdataformat.h
View File

@ -109,7 +109,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
void tRowDestroy(SRow *pRow);
void tRowSort(SArray *aRowP);
int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
// SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
@ -137,6 +137,7 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn)
void tColDataClear(SColData *pColData);
void tColDataDeepClear(SColData *pColData);
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward);
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
@ -200,6 +201,9 @@ struct SColData {
int16_t cid;
int8_t type;
int8_t smaOn;
int32_t numOfNone; // # of none
int32_t numOfNull; // # of null
int32_t numOfValue; // # of vale
int32_t nVal;
uint8_t flag;
uint8_t *pBitMap;

2
include/common/tmsg.h
View File

@ -1243,7 +1243,7 @@ int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq
typedef struct {
int64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
int64_t reserved[8];
int64_t compactStartTime;
} SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);

448
include/common/ttokendef.h
View File

@ -77,230 +77,230 @@
#define TK_USE 59
#define TK_FLUSH 60
#define TK_TRIM 61
#define TK_IF 62
#define TK_NOT 63
#define TK_EXISTS 64
#define TK_BUFFER 65
#define TK_CACHEMODEL 66
#define TK_CACHESIZE 67
#define TK_COMP 68
#define TK_DURATION 69
#define TK_NK_VARIABLE 70
#define TK_MAXROWS 71
#define TK_MINROWS 72
#define TK_KEEP 73
#define TK_PAGES 74
#define TK_PAGESIZE 75
#define TK_TSDB_PAGESIZE 76
#define TK_PRECISION 77
#define TK_REPLICA 78
#define TK_VGROUPS 79
#define TK_SINGLE_STABLE 80
#define TK_RETENTIONS 81
#define TK_SCHEMALESS 82
#define TK_WAL_LEVEL 83
#define TK_WAL_FSYNC_PERIOD 84
#define TK_WAL_RETENTION_PERIOD 85
#define TK_WAL_RETENTION_SIZE 86
#define TK_WAL_ROLL_PERIOD 87
#define TK_WAL_SEGMENT_SIZE 88
#define TK_STT_TRIGGER 89
#define TK_TABLE_PREFIX 90
#define TK_TABLE_SUFFIX 91
#define TK_NK_COLON 92
#define TK_MAX_SPEED 93
#define TK_TABLE 94
#define TK_NK_LP 95
#define TK_NK_RP 96
#define TK_STABLE 97
#define TK_ADD 98
#define TK_COLUMN 99
#define TK_MODIFY 100
#define TK_RENAME 101
#define TK_TAG 102
#define TK_SET 103
#define TK_NK_EQ 104
#define TK_USING 105
#define TK_TAGS 106
#define TK_COMMENT 107
#define TK_BOOL 108
#define TK_TINYINT 109
#define TK_SMALLINT 110
#define TK_INT 111
#define TK_INTEGER 112
#define TK_BIGINT 113
#define TK_FLOAT 114
#define TK_DOUBLE 115
#define TK_BINARY 116
#define TK_TIMESTAMP 117
#define TK_NCHAR 118
#define TK_UNSIGNED 119
#define TK_JSON 120
#define TK_VARCHAR 121
#define TK_MEDIUMBLOB 122
#define TK_BLOB 123
#define TK_VARBINARY 124
#define TK_DECIMAL 125
#define TK_MAX_DELAY 126
#define TK_WATERMARK 127
#define TK_ROLLUP 128
#define TK_TTL 129
#define TK_SMA 130
#define TK_DELETE_MARK 131
#define TK_FIRST 132
#define TK_LAST 133
#define TK_SHOW 134
#define TK_PRIVILEGES 135
#define TK_DATABASES 136
#define TK_TABLES 137
#define TK_STABLES 138
#define TK_MNODES 139
#define TK_QNODES 140
#define TK_FUNCTIONS 141
#define TK_INDEXES 142
#define TK_ACCOUNTS 143
#define TK_APPS 144
#define TK_CONNECTIONS 145
#define TK_LICENCES 146
#define TK_GRANTS 147
#define TK_QUERIES 148
#define TK_SCORES 149
#define TK_TOPICS 150
#define TK_VARIABLES 151
#define TK_CLUSTER 152
#define TK_BNODES 153
#define TK_SNODES 154
#define TK_TRANSACTIONS 155
#define TK_DISTRIBUTED 156
#define TK_CONSUMERS 157
#define TK_SUBSCRIPTIONS 158
#define TK_VNODES 159
#define TK_ALIVE 160
#define TK_LIKE 161
#define TK_TBNAME 162
#define TK_QTAGS 163
#define TK_AS 164
#define TK_INDEX 165
#define TK_FUNCTION 166
#define TK_INTERVAL 167
#define TK_COUNT 168
#define TK_LAST_ROW 169
#define TK_TOPIC 170
#define TK_WITH 171
#define TK_META 172
#define TK_CONSUMER 173
#define TK_GROUP 174
#define TK_DESC 175
#define TK_DESCRIBE 176
#define TK_RESET 177
#define TK_QUERY 178
#define TK_CACHE 179
#define TK_EXPLAIN 180
#define TK_ANALYZE 181
#define TK_VERBOSE 182
#define TK_NK_BOOL 183
#define TK_RATIO 184
#define TK_NK_FLOAT 185
#define TK_OUTPUTTYPE 186
#define TK_AGGREGATE 187
#define TK_BUFSIZE 188
#define TK_STREAM 189
#define TK_INTO 190
#define TK_TRIGGER 191
#define TK_AT_ONCE 192
#define TK_WINDOW_CLOSE 193
#define TK_IGNORE 194
#define TK_EXPIRED 195
#define TK_FILL_HISTORY 196
#define TK_SUBTABLE 197
#define TK_KILL 198
#define TK_CONNECTION 199
#define TK_TRANSACTION 200
#define TK_BALANCE 201
#define TK_VGROUP 202
#define TK_MERGE 203
#define TK_REDISTRIBUTE 204
#define TK_SPLIT 205
#define TK_DELETE 206
#define TK_INSERT 207
#define TK_NULL 208
#define TK_NK_QUESTION 209
#define TK_NK_ARROW 210
#define TK_ROWTS 211
#define TK_QSTART 212
#define TK_QEND 213
#define TK_QDURATION 214
#define TK_WSTART 215
#define TK_WEND 216
#define TK_WDURATION 217
#define TK_IROWTS 218
#define TK_ISFILLED 219
#define TK_CAST 220
#define TK_NOW 221
#define TK_TODAY 222
#define TK_TIMEZONE 223
#define TK_CLIENT_VERSION 224
#define TK_SERVER_VERSION 225
#define TK_SERVER_STATUS 226
#define TK_CURRENT_USER 227
#define TK_CASE 228
#define TK_END 229
#define TK_WHEN 230
#define TK_THEN 231
#define TK_ELSE 232
#define TK_BETWEEN 233
#define TK_IS 234
#define TK_NK_LT 235
#define TK_NK_GT 236
#define TK_NK_LE 237
#define TK_NK_GE 238
#define TK_NK_NE 239
#define TK_MATCH 240
#define TK_NMATCH 241
#define TK_CONTAINS 242
#define TK_IN 243
#define TK_JOIN 244
#define TK_INNER 245
#define TK_SELECT 246
#define TK_DISTINCT 247
#define TK_WHERE 248
#define TK_PARTITION 249
#define TK_BY 250
#define TK_SESSION 251
#define TK_STATE_WINDOW 252
#define TK_EVENT_WINDOW 253
#define TK_START 254
#define TK_SLIDING 255
#define TK_FILL 256
#define TK_VALUE 257
#define TK_VALUE_F 258
#define TK_NONE 259
#define TK_PREV 260
#define TK_NULL_F 261
#define TK_LINEAR 262
#define TK_NEXT 263
#define TK_HAVING 264
#define TK_RANGE 265
#define TK_EVERY 266
#define TK_ORDER 267
#define TK_SLIMIT 268
#define TK_SOFFSET 269
#define TK_LIMIT 270
#define TK_OFFSET 271
#define TK_ASC 272
#define TK_NULLS 273
#define TK_ABORT 274
#define TK_AFTER 275
#define TK_ATTACH 276
#define TK_BEFORE 277
#define TK_BEGIN 278
#define TK_BITAND 279
#define TK_BITNOT 280
#define TK_BITOR 281
#define TK_BLOCKS 282
#define TK_CHANGE 283
#define TK_COMMA 284
#define TK_COMPACT 285
#define TK_COMPACT 62
#define TK_IF 63
#define TK_NOT 64
#define TK_EXISTS 65
#define TK_BUFFER 66
#define TK_CACHEMODEL 67
#define TK_CACHESIZE 68
#define TK_COMP 69
#define TK_DURATION 70
#define TK_NK_VARIABLE 71
#define TK_MAXROWS 72
#define TK_MINROWS 73
#define TK_KEEP 74
#define TK_PAGES 75
#define TK_PAGESIZE 76
#define TK_TSDB_PAGESIZE 77
#define TK_PRECISION 78
#define TK_REPLICA 79
#define TK_VGROUPS 80
#define TK_SINGLE_STABLE 81
#define TK_RETENTIONS 82
#define TK_SCHEMALESS 83
#define TK_WAL_LEVEL 84
#define TK_WAL_FSYNC_PERIOD 85
#define TK_WAL_RETENTION_PERIOD 86
#define TK_WAL_RETENTION_SIZE 87
#define TK_WAL_ROLL_PERIOD 88
#define TK_WAL_SEGMENT_SIZE 89
#define TK_STT_TRIGGER 90
#define TK_TABLE_PREFIX 91
#define TK_TABLE_SUFFIX 92
#define TK_NK_COLON 93
#define TK_MAX_SPEED 94
#define TK_TABLE 95
#define TK_NK_LP 96
#define TK_NK_RP 97
#define TK_STABLE 98
#define TK_ADD 99
#define TK_COLUMN 100
#define TK_MODIFY 101
#define TK_RENAME 102
#define TK_TAG 103
#define TK_SET 104
#define TK_NK_EQ 105
#define TK_USING 106
#define TK_TAGS 107
#define TK_COMMENT 108
#define TK_BOOL 109
#define TK_TINYINT 110
#define TK_SMALLINT 111
#define TK_INT 112
#define TK_INTEGER 113
#define TK_BIGINT 114
#define TK_FLOAT 115
#define TK_DOUBLE 116
#define TK_BINARY 117
#define TK_TIMESTAMP 118
#define TK_NCHAR 119
#define TK_UNSIGNED 120
#define TK_JSON 121
#define TK_VARCHAR 122
#define TK_MEDIUMBLOB 123
#define TK_BLOB 124
#define TK_VARBINARY 125
#define TK_DECIMAL 126
#define TK_MAX_DELAY 127
#define TK_WATERMARK 128
#define TK_ROLLUP 129
#define TK_TTL 130
#define TK_SMA 131
#define TK_DELETE_MARK 132
#define TK_FIRST 133
#define TK_LAST 134
#define TK_SHOW 135
#define TK_PRIVILEGES 136
#define TK_DATABASES 137
#define TK_TABLES 138
#define TK_STABLES 139
#define TK_MNODES 140
#define TK_QNODES 141
#define TK_FUNCTIONS 142
#define TK_INDEXES 143
#define TK_ACCOUNTS 144
#define TK_APPS 145
#define TK_CONNECTIONS 146
#define TK_LICENCES 147
#define TK_GRANTS 148
#define TK_QUERIES 149
#define TK_SCORES 150
#define TK_TOPICS 151
#define TK_VARIABLES 152
#define TK_CLUSTER 153
#define TK_BNODES 154
#define TK_SNODES 155
#define TK_TRANSACTIONS 156
#define TK_DISTRIBUTED 157
#define TK_CONSUMERS 158
#define TK_SUBSCRIPTIONS 159
#define TK_VNODES 160
#define TK_ALIVE 161
#define TK_LIKE 162
#define TK_TBNAME 163
#define TK_QTAGS 164
#define TK_AS 165
#define TK_INDEX 166
#define TK_FUNCTION 167
#define TK_INTERVAL 168
#define TK_COUNT 169
#define TK_LAST_ROW 170
#define TK_TOPIC 171
#define TK_WITH 172
#define TK_META 173
#define TK_CONSUMER 174
#define TK_GROUP 175
#define TK_DESC 176
#define TK_DESCRIBE 177
#define TK_RESET 178
#define TK_QUERY 179
#define TK_CACHE 180
#define TK_EXPLAIN 181
#define TK_ANALYZE 182
#define TK_VERBOSE 183
#define TK_NK_BOOL 184
#define TK_RATIO 185
#define TK_NK_FLOAT 186
#define TK_OUTPUTTYPE 187
#define TK_AGGREGATE 188
#define TK_BUFSIZE 189
#define TK_STREAM 190
#define TK_INTO 191
#define TK_TRIGGER 192
#define TK_AT_ONCE 193
#define TK_WINDOW_CLOSE 194
#define TK_IGNORE 195
#define TK_EXPIRED 196
#define TK_FILL_HISTORY 197
#define TK_SUBTABLE 198
#define TK_KILL 199
#define TK_CONNECTION 200
#define TK_TRANSACTION 201
#define TK_BALANCE 202
#define TK_VGROUP 203
#define TK_MERGE 204
#define TK_REDISTRIBUTE 205
#define TK_SPLIT 206
#define TK_DELETE 207
#define TK_INSERT 208
#define TK_NULL 209
#define TK_NK_QUESTION 210
#define TK_NK_ARROW 211
#define TK_ROWTS 212
#define TK_QSTART 213
#define TK_QEND 214
#define TK_QDURATION 215
#define TK_WSTART 216
#define TK_WEND 217
#define TK_WDURATION 218
#define TK_IROWTS 219
#define TK_ISFILLED 220
#define TK_CAST 221
#define TK_NOW 222
#define TK_TODAY 223
#define TK_TIMEZONE 224
#define TK_CLIENT_VERSION 225
#define TK_SERVER_VERSION 226
#define TK_SERVER_STATUS 227
#define TK_CURRENT_USER 228
#define TK_CASE 229
#define TK_END 230
#define TK_WHEN 231
#define TK_THEN 232
#define TK_ELSE 233
#define TK_BETWEEN 234
#define TK_IS 235
#define TK_NK_LT 236
#define TK_NK_GT 237
#define TK_NK_LE 238
#define TK_NK_GE 239
#define TK_NK_NE 240
#define TK_MATCH 241
#define TK_NMATCH 242
#define TK_CONTAINS 243
#define TK_IN 244
#define TK_JOIN 245
#define TK_INNER 246
#define TK_SELECT 247
#define TK_DISTINCT 248
#define TK_WHERE 249
#define TK_PARTITION 250
#define TK_BY 251
#define TK_SESSION 252
#define TK_STATE_WINDOW 253
#define TK_EVENT_WINDOW 254
#define TK_START 255
#define TK_SLIDING 256
#define TK_FILL 257
#define TK_VALUE 258
#define TK_VALUE_F 259
#define TK_NONE 260
#define TK_PREV 261
#define TK_NULL_F 262
#define TK_LINEAR 263
#define TK_NEXT 264
#define TK_HAVING 265
#define TK_RANGE 266
#define TK_EVERY 267
#define TK_ORDER 268
#define TK_SLIMIT 269
#define TK_SOFFSET 270
#define TK_LIMIT 271
#define TK_OFFSET 272
#define TK_ASC 273
#define TK_NULLS 274
#define TK_ABORT 275
#define TK_AFTER 276
#define TK_ATTACH 277
#define TK_BEFORE 278
#define TK_BEGIN 279
#define TK_BITAND 280
#define TK_BITNOT 281
#define TK_BITOR 282
#define TK_BLOCKS 283
#define TK_CHANGE 284
#define TK_COMMA 285
#define TK_CONCAT 286
#define TK_CONFLICT 287
#define TK_COPY 288

5
include/libs/nodes/cmdnodes.h
View File

@ -124,6 +124,11 @@ typedef struct STrimDatabaseStmt {
int32_t maxSpeed;
} STrimDatabaseStmt;
typedef struct SCompactDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SCompactDatabaseStmt;
typedef struct STableOptions {
ENodeType type;
bool commentNull;

2
include/libs/nodes/nodes.h
View File

@ -155,7 +155,7 @@ typedef enum ENodeType {
QUERY_NODE_EXPLAIN_STMT,
QUERY_NODE_DESCRIBE_STMT,
QUERY_NODE_RESET_QUERY_CACHE_STMT,
QUERY_NODE_COMPACT_STMT,
QUERY_NODE_COMPACT_DATABASE_STMT,
QUERY_NODE_CREATE_FUNCTION_STMT,
QUERY_NODE_DROP_FUNCTION_STMT,
QUERY_NODE_CREATE_STREAM_STMT,

3
include/util/trbtree.h
View File

@ -33,9 +33,12 @@ typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *);
#define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max)
void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn);
void tRBTreeClear(SRBTree *pTree);
SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z);
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z);
SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey);
SRBTreeNode *tRBTreeDropMin(SRBTree *pTree);
SRBTreeNode *tRBTreeDropMax(SRBTree *pTree);
SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode);
// SRBTreeIter =============================================

1
source/common/src/systable.c
View File

@ -230,6 +230,7 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
// {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema smaSchema[] = {

433
source/common/src/tdataformat.c
View File

@ -21,6 +21,7 @@
#include "tlog.h"
static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData);
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward);
// SBuffer ================================
void tBufferDestroy(SBuffer *pBuffer) {
@ -850,19 +851,19 @@ _exit:
return &pIter->cv;
}
static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) {
static int32_t tRowNoneUpsertColData(SColData *aColData, int32_t nColData, int32_t flag) {
int32_t code = 0;
if (flag) return code;
for (int32_t iColData = 0; iColData < nColData; iColData++) {
SColData *pColData = &aColData[iColData];
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
code = tColDataAppendValueImpl[aColData[iColData].flag][CV_FLAG_NONE](&aColData[iColData], NULL, 0);
if (code) return code;
}
_exit:
return code;
}
static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) {
static int32_t tRowNullUpsertColData(SColData *aColData, int32_t nColData, STSchema *pSchema, int32_t flag) {
int32_t code = 0;
int32_t iColData = 0;
@ -873,21 +874,23 @@ static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STS
while (pColData) {
if (pTColumn) {
if (pTColumn->colId == pColData->cid) { // NULL
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) { // NONE
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
} else { // NONE
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -895,7 +898,8 @@ static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STS
_exit:
return code;
}
static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData,
int32_t flag) {
int32_t code = 0;
int32_t iColData = 0;
@ -955,11 +959,15 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
}
if (bv == BIT_FLG_NONE) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0)))
goto _exit;
goto _continue;
} else if (bv == BIT_FLG_NULL) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
goto _continue;
}
@ -969,11 +977,20 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset);
uint32_t nData;
pData += tGetU32v(pData, &nData);
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0);
}
if (code) goto _exit;
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset,
TYPE_BYTES[pColData->type]);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset,
TYPE_BYTES[pColData->type]);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset,
TYPE_BYTES[pColData->type], flag > 0);
}
if (code) goto _exit;
}
@ -981,17 +998,13 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) { // NONE
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -999,7 +1012,7 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
_exit:
return code;
}
static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) {
int32_t code = 0;
SKVIdx *pKVIdx = (SKVIdx *)pRow->data;
@ -1040,18 +1053,25 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
if (TABS(cid) == pTColumn->colId) {
if (cid < 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
} else {
uint32_t nData;
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
uint32_t nData;
pData += tGetU32v(pData, &nData);
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
if (code) goto _exit;
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0);
if (code) goto _exit;
nData = 0;
}
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0);
}
if (code) goto _exit;
}
iCol++;
goto _continue;
@ -1062,22 +1082,19 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
}
}
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
_continue:
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -1085,30 +1102,23 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
_exit:
return code;
}
int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
/* flag > 0: forward update
* flag == 0: append
* flag < 0: backward update
*/
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) {
ASSERT(pRow->sver == pTSchema->version);
ASSERT(nColData > 0);
int32_t code = 0;
if (pRow->flag == HAS_NONE) {
code = tRowAppendNoneToColData(aColData, nColData);
goto _exit;
return tRowNoneUpsertColData(aColData, nColData, flag);
} else if (pRow->flag == HAS_NULL) {
code = tRowAppendNullToColData(aColData, nColData, pTSchema);
goto _exit;
return tRowNullUpsertColData(aColData, nColData, pTSchema, flag);
} else if (pRow->flag >> 4) { // KV row
return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag);
} else { // TUPLE row
return tRowTupleUpsertColData(pRow, pTSchema, aColData, nColData, flag);
}
if (pRow->flag >> 4) { // KV row
code = tRowAppendKVToColData(pRow, pTSchema, aColData, nColData);
if (code) goto _exit;
} else {
code = tRowAppendTupleToColData(pRow, pTSchema, aColData, nColData);
if (code) goto _exit;
}
_exit:
return code;
}
// STag ========================================
@ -1549,6 +1559,9 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
}
void tColDataClear(SColData *pColData) {
pColData->numOfNone = 0;
pColData->numOfNull = 0;
pColData->numOfValue = 0;
pColData->nVal = 0;
pColData->flag = 0;
pColData->nData = 0;
@ -1594,15 +1607,18 @@ _exit:
}
static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_VALUE;
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_NONE;
pColData->numOfNone++;
pColData->nVal++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_NULL;
pColData->numOfNull++;
pColData->nVal++;
return 0;
}
@ -1617,6 +1633,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
@ -1636,6 +1653,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *p
}
static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->nVal++;
pColData->numOfNone++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) {
@ -1649,6 +1667,7 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
pColData->nVal++;
return code;
@ -1664,6 +1683,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
@ -1692,18 +1712,21 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->nVal++;
pColData->numOfNull++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1740,6 +1763,7 @@ static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
pColData->nVal++;
return code;
@ -1751,15 +1775,20 @@ static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfNull++;
pColData->nVal++;
return code;
}
#define tColDataAppendValue40 tColDataPutValue
static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
@ -1774,6 +1803,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
@ -1791,6 +1821,7 @@ static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1801,6 +1832,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1808,6 +1840,7 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1829,6 +1862,7 @@ static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1836,6 +1870,7 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1857,6 +1892,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNull++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1866,6 +1902,7 @@ static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1875,6 +1912,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1884,6 +1922,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfNull++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1896,6 +1935,8 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pDa
{tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE
{tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL
{tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
// VALUE NONE NULL
};
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
@ -1904,6 +1945,282 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
pColVal->value.nData);
}
static FORCE_INLINE int32_t tColDataUpdateValue10(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue10(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
static FORCE_INLINE int32_t tColDataUpdateValue12(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue12(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue02(pColData, pData, nData);
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue20(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue20(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue30(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE == > VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue30(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
} else if (forward) { // NULL == > VALUE
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue30(pColData, pData, nData);
} else {
pColData->flag = HAS_NONE;
return tColDataAppendValue10(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue32(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->numOfNull++;
if (pColData->numOfNone) {
SET_BIT1(pColData->pBitMap, pColData->nVal - 1, 1);
} else {
pColData->flag = HAS_NULL;
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue40(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue40(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue42(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue42(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue02(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue50(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue50(pColData, pData, nData);
} else {
pColData->flag = HAS_VALUE;
return tColDataAppendValue40(pColData, pData, nData);
}
} else if (forward) { // VALUE ==> VALUE
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue50(pColData, pData, nData);
} else {
pColData->flag = HAS_NONE;
return tColDataAppendValue10(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue52(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue52(pColData, pData, nData);
} else {
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue42(pColData, pData, nData);
}
} else if (forward) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue52(pColData, pData, nData);
} else {
pColData->flag &= ~HAS_VALUE;
return tColDataAppendValue12(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue60(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NULL ==> VALUE
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue60(pColData, pData, nData);
} else {
pColData->flag = HAS_VALUE;
return tColDataAppendValue40(pColData, pData, nData);
}
} else { // VALUE ==> VALUE
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue60(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue62(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward && (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 1)) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue62(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue70(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
int32_t code = 0;
uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
}
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue60(pColData, pData, nData);
}
} else if (bv == 1) { // NULL ==> VALUE
if (forward) {
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) ? 1 : 0);
}
pColData->flag &= ~HAS_NULL;
return tColDataAppendValue50(pColData, pData, nData);
}
}
} else if (bv == 2) { // VALUE ==> VALUE
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
return tColDataPutValue(pColData, pData, nData);
}
}
} else {
ASSERT(0);
}
return 0;
}
static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1);
ASSERT(bv < 3);
if (bv == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue72(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
}
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue62(pColData, pData, nData);
}
} else if (bv == 2 && forward) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue72(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal));
}
pColData->flag &= ~HAS_VALUE;
return tColDataAppendValue32(pColData, pData, nData);
}
}
return 0;
}
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) = {
{NULL, NULL, NULL}, // 0
{tColDataUpdateValue10, NULL, tColDataUpdateValue12}, // HAS_NONE
{tColDataUpdateValue20, NULL, NULL}, // HAS_NULL
{tColDataUpdateValue30, NULL, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE
{tColDataUpdateValue40, NULL, tColDataUpdateValue42}, // HAS_VALUE
{tColDataUpdateValue50, NULL, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE
{tColDataUpdateValue60, NULL, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL
{tColDataUpdateValue70, NULL, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
// VALUE NONE NULL
};
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
ASSERT(pColData->nVal > 0);
if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0;
return tColDataUpdateValueImpl[pColData->flag][pColVal->flag](
pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val,
pColVal->value.nData, forward);
}
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE
*pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
}

8
source/common/src/tmsg.c
View File

@ -3992,9 +3992,7 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
}
if (tEncodeI64(&encoder, pReq->compactStartTime) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4009,9 +4007,7 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
}
if (tDecodeI64(&decoder, &pReq->compactStartTime) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);

1
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -328,6 +328,7 @@ typedef struct {
SDbCfg cfg;
SRWLatch lock;
int64_t stateTs;
int64_t compactStartTime;
} SDbObj;
typedef struct {

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -43,6 +43,7 @@ int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVg
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force);
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray);
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs);
void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

68
source/dnode/mnode/impl/src/mndDb.c
View File

@ -30,7 +30,7 @@
#include "systable.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 54
#define DB_RESERVE_SIZE 46
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -126,6 +126,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER)
SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -215,6 +216,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER)
SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -274,6 +276,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.replications = pNew->cfg.replications;
pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
return 0;
}
@ -1378,7 +1381,63 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
return 0;
}
static int32_t mndCompactDb(SMnode *pMnode, SDbObj *pDb) { return 0; }
static int32_t mndSetCompactDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
SDbObj dbObj = {0};
memcpy(&dbObj, pDb, sizeof(SDbObj));
dbObj.compactStartTime = compactTs;
SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
sdbFreeRaw(pCommitRaw);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
return 0;
}
static int32_t mndSetCompactDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (mndVgroupInDb(pVgroup, pDb->uid)) {
if (mndBuildCompactVgroupAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
}
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
int64_t compactTs = taosGetTimestampMs();
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "compact-db");
if (pTrans == NULL) goto _OVER;
mInfo("trans:%d, used to compact db:%s", pTrans->id, pDb->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
if (mndSetCompactDbCommitLogs(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
if (mndSetCompactDbRedoActions(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
}
static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@ -1402,10 +1461,11 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
goto _OVER;
}
code = mndCompactDb(pMnode, pDb);
code = mndCompactDb(pMnode, pReq, pDb);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0) {
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr());
}

66
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -796,6 +796,13 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
// if (pDb == NULL || pDb->compactStartTime <= 0) {
// colDataAppendNULL(pColInfo, numOfRows);
// } else {
// colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false);
// }
numOfRows++;
sdbRelease(pSdb, pVgroup);
}
@ -1168,7 +1175,8 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
return 0;
}
static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) {
static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
int32_t dnodeId) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
if (pDnode == NULL) return -1;
@ -2192,3 +2200,59 @@ _OVER:
}
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }
static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen,
int64_t compactTs) {
SCompactVnodeReq compactReq = {0};
compactReq.dbUid = pDb->uid;
compactReq.compactStartTime = compactTs;
tstrncpy(compactReq.db, pDb->name, TSDB_DB_FNAME_LEN);
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSCompactVnodeReq(NULL, 0, &compactReq);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
contLen += sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSCompactVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &compactReq);
*pContLen = contLen;
return pReq;
}
static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
int64_t compactTs) {
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t contLen = 0;
void *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs);
if (pReq == NULL) return -1;
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs) {
if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) return -1;
return 0;
}

2
source/dnode/vnode/CMakeLists.txt
View File

@ -14,6 +14,7 @@ target_sources(
"src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c"
"src/vnd/vnodeSnapshot.c"
"src/vnd/vnodeCompact.c"
# meta
"src/meta/metaOpen.c"
@ -53,6 +54,7 @@ target_sources(
"src/tsdb/tsdbDiskData.c"
"src/tsdb/tsdbCompact.c"
"src/tsdb/tsdbMergeTree.c"
"src/tsdb/tsdbDataIter.c"
# tq
"src/tq/tq.c"

85
source/dnode/vnode/src/inc/tsdb.h
View File

@ -69,6 +69,8 @@ typedef struct SDiskCol SDiskCol;
typedef struct SDiskData SDiskData;
typedef struct SDiskDataBuilder SDiskDataBuilder;
typedef struct SBlkInfo SBlkInfo;
typedef struct STsdbDataIter2 STsdbDataIter2;
typedef struct STsdbFilterInfo STsdbFilterInfo;
#define TSDBROW_ROW_FMT ((int8_t)0x0)
#define TSDBROW_COL_FMT ((int8_t)0x1)
@ -163,6 +165,8 @@ void tBlockDataDestroy(SBlockData *pBlockData);
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
void tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid);
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void tBlockDataClear(SBlockData *pBlockData);
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
@ -277,6 +281,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData);
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
// SDelFWriter
@ -308,6 +313,26 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
// tsdbDataIter.c ==============================================================================================
#define TSDB_MEM_TABLE_DATA_ITER 0
#define TSDB_DATA_FILE_DATA_ITER 1
#define TSDB_STT_FILE_DATA_ITER 2
#define TSDB_TOMB_FILE_DATA_ITER 3
#define TSDB_FILTER_FLAG_BY_VERSION 0x1
#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
/* open */
int32_t tsdbOpenDataFileDataIter(SDataFReader *pReader, STsdbDataIter2 **ppIter);
int32_t tsdbOpenSttFileDataIter(SDataFReader *pReader, int32_t iStt, STsdbDataIter2 **ppIter);
int32_t tsdbOpenTombFileDataIter(SDelFReader *pReader, STsdbDataIter2 **ppIter);
/* close */
void tsdbCloseDataIter2(STsdbDataIter2 *pIter);
/* cmpr */
int32_t tsdbDataIterCmprFn(const SRBTreeNode *pNode1, const SRBTreeNode *pNode2);
/* next */
int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo);
// structs =======================
struct STsdbFS {
@ -644,6 +669,7 @@ struct SDataFReader {
uint8_t *aBuf[3];
};
// NOTE: do NOT change the order of the fields
typedef struct {
int64_t suid;
int64_t uid;
@ -830,6 +856,65 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
return pIter->pRow;
}
int32_t tRowInfoCmprFn(const void *p1, const void *p2);
typedef struct {
int64_t suid;
int64_t uid;
SDelData delData;
} SDelInfo;
struct STsdbDataIter2 {
STsdbDataIter2 *next;
SRBTreeNode rbtn;
int32_t type;
SRowInfo rowInfo;
SDelInfo delInfo;
union {
// TSDB_MEM_TABLE_DATA_ITER
struct {
SMemTable *pMemTable;
} mIter;
// TSDB_DATA_FILE_DATA_ITER
struct {
SDataFReader *pReader;
SArray *aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk;
SBlockData bData;
int32_t iBlockIdx;
int32_t iDataBlk;
int32_t iRow;
} dIter;
// TSDB_STT_FILE_DATA_ITER
struct {
SDataFReader *pReader;
int32_t iStt;
SArray *aSttBlk;
SBlockData bData;
int32_t iSttBlk;
int32_t iRow;
} sIter;
// TSDB_TOMB_FILE_DATA_ITER
struct {
SDelFReader *pReader;
SArray *aDelIdx;
SArray *aDelData;
int32_t iDelIdx;
int32_t iDelData;
} tIter;
};
};
struct STsdbFilterInfo {
int32_t flag;
int64_t sver;
int64_t ever;
TABLEID tbid;
};
#ifdef __cplusplus
}
#endif

4
source/dnode/vnode/src/inc/vnd.h
View File

@ -106,6 +106,10 @@ int32_t vnodeSyncCommit(SVnode* pVnode);
int32_t vnodeAsyncCommit(SVnode* pVnode);
bool vnodeShouldRollback(SVnode* pVnode);
// vnodeCompact.c
int32_t vnodeAsyncCompact(SVnode* pVnode);
int32_t vnodeSyncCompact(SVnode* pVnode);
// vnodeSync.c
int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
int32_t vnodeSyncStart(SVnode* pVnode);

4
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -450,8 +450,10 @@ struct SCommitInfo {
SVnodeInfo info;
SVnode* pVnode;
TXN* txn;
};
// APIs
int32_t (*commitFn)(STsdb* pTsdb, SCommitInfo* pInfo);
};
#ifdef __cplusplus
}
#endif

645
source/dnode/vnode/src/tsdb/tsdbCompact.c
View File

@ -15,13 +15,654 @@
#include "tsdb.h"
extern int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo);
extern int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg);
extern int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg);
typedef struct {
STsdb *pTsdb;
int64_t commitID;
int8_t cmprAlg;
int32_t maxRows;
int32_t minRows;
STsdbFS fs;
int32_t fid;
TABLEID tbid;
SSkmInfo tbSkm;
// Tombstone
SDelFReader *pDelFReader;
SArray *aDelIdx; // SArray<SDelIdx>
SArray *aDelData; // SArray<SDelData>
SArray *aSkyLine; // SArray<TSDBKEY>
int32_t iDelIdx;
int32_t iSkyLine;
TSDBKEY *pDKey;
TSDBKEY dKey;
// Reader
SDataFReader *pReader;
STsdbDataIter2 *iterList; // list of iterators
STsdbDataIter2 *pIter;
SRBTree rbt;
// Writer
SDataFWriter *pWriter;
SArray *aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk; // SMapData<SDataBlk>
SArray *aSttBlk; // SArray<SSttBlk>
SBlockData bData;
SBlockData sData;
} STsdbCompactor;
int32_t tsdbCompact(STsdb *pTsdb) {
static int32_t tsdbCommitCompact(STsdbCompactor *pCompactor) {
int32_t code = 0;
// TODO
int32_t lino = 0;
STsdb *pTsdb = pCompactor->pTsdb;
code = tsdbFSPrepareCommit(pTsdb, &pCompactor->fs);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadRwlockWrlock(&pTsdb->rwLock);
code = tsdbFSCommit(pTsdb);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadRwlockUnlock(&pTsdb->rwLock);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbAbortCompact(STsdbCompactor *pCompactor) {
int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCompactor->pTsdb;
// TODO
ASSERT(0);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
} else {
tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
}
return code;
}
static int32_t tsdbCompactWriteTableDataStart(STsdbCompactor *pCompactor, TABLEID *pId) {
int32_t code = 0;
int32_t lino = 0;
pCompactor->tbid = *pId;
// tombstone
for (;;) {
if (pCompactor->iDelIdx >= taosArrayGetSize(pCompactor->aDelIdx)) {
pCompactor->pDKey = NULL;
break;
}
SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(pCompactor->aDelIdx, pCompactor->iDelIdx);
int32_t c = tTABLEIDCmprFn(pDelIdx, &pCompactor->tbid);
if (c < 0) {
pCompactor->iDelIdx++;
} else if (c == 0) {
pCompactor->iDelIdx++;
code = tsdbReadDelData(pCompactor->pDelFReader, pDelIdx, pCompactor->aDelData);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbBuildDeleteSkyline(pCompactor->aDelData, 0, taosArrayGetSize(pCompactor->aDelData) - 1,
pCompactor->aSkyLine);
TSDB_CHECK_CODE(code, lino, _exit);
pCompactor->iSkyLine = 0;
if (pCompactor->iSkyLine < taosArrayGetSize(pCompactor->aSkyLine)) {
TSDBKEY *pKey = (TSDBKEY *)taosArrayGet(pCompactor->aSkyLine, pCompactor->iSkyLine);
pCompactor->dKey.version = 0;
pCompactor->dKey.ts = pKey->ts;
pCompactor->pDKey = &pCompactor->dKey;
} else {
pCompactor->pDKey = NULL;
}
break;
} else {
pCompactor->pDKey = NULL;
break;
}
}
// writer
code = tsdbUpdateTableSchema(pCompactor->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pCompactor->tbSkm);
TSDB_CHECK_CODE(code, lino, _exit);
tMapDataReset(&pCompactor->mDataBlk);
code = tBlockDataInit(&pCompactor->bData, pId, pCompactor->tbSkm.pTSchema, NULL, 0);
TSDB_CHECK_CODE(code, lino, _exit);
if (!TABLE_SAME_SCHEMA(pCompactor->sData.suid, pCompactor->sData.uid, pId->suid, pId->uid)) {
if (pCompactor->sData.nRow > 0) {
code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
TABLEID tbid = {.suid = pId->suid, .uid = pId->suid ? 0 : pId->uid};
code = tBlockDataInit(&pCompactor->sData, &tbid, pCompactor->tbSkm.pTSchema, NULL, 0);
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code));
} else {
tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pId->suid,
pId->uid);
}
return code;
}
static int32_t tsdbCompactWriteTableDataEnd(STsdbCompactor *pCompactor) {
int32_t code = 0;
int32_t lino = 0;
if (pCompactor->bData.nRow > 0) {
if (pCompactor->bData.nRow < pCompactor->minRows) {
for (int32_t iRow = 0; iRow < pCompactor->bData.nRow; iRow++) {
code = tBlockDataAppendRow(&pCompactor->sData, &tsdbRowFromBlockData(&pCompactor->bData, iRow), NULL,
pCompactor->tbid.uid);
TSDB_CHECK_CODE(code, lino, _exit);
if (pCompactor->sData.nRow >= pCompactor->maxRows) {
code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
tBlockDataClear(&pCompactor->bData);
} else {
code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
if (pCompactor->mDataBlk.nItem > 0) {
SBlockIdx *pBlockIdx = (SBlockIdx *)taosArrayReserve(pCompactor->aBlockIdx, 1);
if (pBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pBlockIdx->suid = pCompactor->tbid.suid;
pBlockIdx->uid = pCompactor->tbid.uid;
code = tsdbWriteDataBlk(pCompactor->pWriter, &pCompactor->mDataBlk, pBlockIdx);
TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code));
} else {
tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__,
pCompactor->tbid.suid, pCompactor->tbid.uid);
}
return code;
}
static bool tsdbCompactRowIsDeleted(STsdbCompactor *pCompactor, TSDBROW *pRow) {
TSDBKEY tKey = TSDBROW_KEY(pRow);
TSDBKEY *aKey = (TSDBKEY *)TARRAY_DATA(pCompactor->aSkyLine);
int32_t nKey = TARRAY_SIZE(pCompactor->aSkyLine);
if (tKey.ts > pCompactor->pDKey->ts) {
do {
pCompactor->pDKey->version = aKey[pCompactor->iSkyLine].version;
pCompactor->iSkyLine++;
if (pCompactor->iSkyLine < nKey) {
pCompactor->dKey.ts = aKey[pCompactor->iSkyLine].ts;
} else {
if (pCompactor->pDKey->version == 0) {
pCompactor->pDKey = NULL;
return false;
} else {
pCompactor->pDKey->ts = INT64_MAX;
}
}
} while (tKey.ts > pCompactor->pDKey->ts);
}
if (tKey.ts < pCompactor->pDKey->ts) {
if (tKey.version > pCompactor->pDKey->version) {
return false;
} else {
return true;
}
} else if (tKey.ts == pCompactor->pDKey->ts) {
ASSERT(pCompactor->iSkyLine < nKey);
if (tKey.version > TMAX(pCompactor->pDKey->version, aKey[pCompactor->iSkyLine].version)) {
return false;
} else {
return true;
}
}
return false;
}
static int32_t tsdbCompactWriteTableData(STsdbCompactor *pCompactor, SRowInfo *pRowInfo) {
int32_t code = 0;
int32_t lino = 0;
// start a new table data write if need
if (pRowInfo == NULL || pRowInfo->uid != pCompactor->tbid.uid) {
if (pCompactor->tbid.uid) {
code = tsdbCompactWriteTableDataEnd(pCompactor);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (pRowInfo == NULL) {
if (pCompactor->sData.nRow > 0) {
code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
return code;
}
code = tsdbCompactWriteTableDataStart(pCompactor, (TABLEID *)pRowInfo);
TSDB_CHECK_CODE(code, lino, _exit);
}
// check if row is deleted
if (pCompactor->pDKey && tsdbCompactRowIsDeleted(pCompactor, &pRowInfo->row)) goto _exit;
if (tBlockDataTryUpsertRow(&pCompactor->bData, &pRowInfo->row, pRowInfo->uid) > pCompactor->maxRows) {
code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataUpsertRow(&pCompactor->bData, &pRowInfo->row, NULL, pRowInfo->uid);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code));
} else if (pRowInfo) {
tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " ts:%" PRId64 " version:%" PRId64,
TD_VID(pCompactor->pTsdb->pVnode), __func__, pRowInfo->suid, pRowInfo->uid, TSDBROW_TS(&pRowInfo->row),
TSDBROW_VERSION(&pRowInfo->row));
}
return code;
}
static bool tsdbCompactTableIsDropped(STsdbCompactor *pCompactor) {
SMetaInfo info;
if (pCompactor->pIter->rowInfo.uid == pCompactor->tbid.uid) return false;
if (metaGetInfo(pCompactor->pTsdb->pVnode->pMeta, pCompactor->tbid.uid, &info, NULL)) {
return true;
}
return false;
}
static int32_t tsdbCompactNextRow(STsdbCompactor *pCompactor, SRowInfo **ppRowInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
if (pCompactor->pIter) {
code = tsdbDataIterNext2(pCompactor->pIter, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
if (pCompactor->pIter->rowInfo.suid == 0 && pCompactor->pIter->rowInfo.uid == 0) {
pCompactor->pIter = NULL;
} else {
SRBTreeNode *pNode = tRBTreeMin(&pCompactor->rbt);
if (pNode) {
int32_t c = tsdbDataIterCmprFn(&pCompactor->pIter->rbtn, pNode);
if (c > 0) {
tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
pCompactor->pIter = NULL;
} else if (c == 0) {
ASSERT(0);
}
}
}
}
if (pCompactor->pIter == NULL) {
SRBTreeNode *pNode = tRBTreeDropMin(&pCompactor->rbt);
if (pNode) {
pCompactor->pIter = TSDB_RBTN_TO_DATA_ITER(pNode);
}
}
if (pCompactor->pIter) {
if (tsdbCompactTableIsDropped(pCompactor)) {
TABLEID tbid = {.suid = pCompactor->pIter->rowInfo.suid, .uid = pCompactor->pIter->rowInfo.uid};
tRBTreeClear(&pCompactor->rbt);
for (pCompactor->pIter = pCompactor->iterList; pCompactor->pIter; pCompactor->pIter = pCompactor->pIter->next) {
code = tsdbDataIterNext2(pCompactor->pIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_TABLEID, .tbid = tbid});
TSDB_CHECK_CODE(code, lino, _exit);
if (pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid) {
tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
}
}
} else {
*ppRowInfo = &pCompactor->pIter->rowInfo;
break;
}
} else {
*ppRowInfo = NULL;
break;
}
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code;
}
static int32_t tsdbCompactFileSetStart(STsdbCompactor *pCompactor, SDFileSet *pSet) {
int32_t code = 0;
int32_t lino = 0;
pCompactor->fid = pSet->fid;
pCompactor->tbid = (TABLEID){0};
/* tombstone */
pCompactor->iDelIdx = 0;
/* reader */
code = tsdbDataFReaderOpen(&pCompactor->pReader, pCompactor->pTsdb, pSet);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbOpenDataFileDataIter(pCompactor->pReader, &pCompactor->pIter);
TSDB_CHECK_CODE(code, lino, _exit);
tRBTreeCreate(&pCompactor->rbt, tsdbDataIterCmprFn);
if (pCompactor->pIter) {
pCompactor->pIter->next = pCompactor->iterList;
pCompactor->iterList = pCompactor->pIter;
code = tsdbDataIterNext2(pCompactor->pIter, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid);
tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
}
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
code = tsdbOpenSttFileDataIter(pCompactor->pReader, iStt, &pCompactor->pIter);
TSDB_CHECK_CODE(code, lino, _exit);
if (pCompactor->pIter) {
pCompactor->pIter->next = pCompactor->iterList;
pCompactor->iterList = pCompactor->pIter;
code = tsdbDataIterNext2(pCompactor->pIter, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid);
tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
}
}
pCompactor->pIter = NULL;
/* writer */
code = tsdbDataFWriterOpen(&pCompactor->pWriter, pCompactor->pTsdb,
&(SDFileSet){.fid = pCompactor->fid,
.diskId = pSet->diskId,
.pHeadF = &(SHeadFile){.commitID = pCompactor->commitID},
.pDataF = &(SDataFile){.commitID = pCompactor->commitID},
.pSmaF = &(SSmaFile){.commitID = pCompactor->commitID},
.nSttF = 1,
.aSttF = {&(SSttFile){.commitID = pCompactor->commitID}}});
TSDB_CHECK_CODE(code, lino, _exit);
if (pCompactor->aBlockIdx) {
taosArrayClear(pCompactor->aBlockIdx);
} else if ((pCompactor->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
tMapDataReset(&pCompactor->mDataBlk);
if (pCompactor->aSttBlk) {
taosArrayClear(pCompactor->aSttBlk);
} else if ((pCompactor->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
tBlockDataReset(&pCompactor->bData);
tBlockDataReset(&pCompactor->sData);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code), pCompactor->fid);
} else {
tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid);
}
return code;
}
static int32_t tsdbCompactFileSetEnd(STsdbCompactor *pCompactor) {
int32_t code = 0;
int32_t lino = 0;
ASSERT(pCompactor->bData.nRow == 0);
ASSERT(pCompactor->sData.nRow == 0);
/* update files */
code = tsdbWriteSttBlk(pCompactor->pWriter, pCompactor->aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbWriteBlockIdx(pCompactor->pWriter, pCompactor->aBlockIdx);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbUpdateDFileSetHeader(pCompactor->pWriter);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFSUpsertFSet(&pCompactor->fs, &pCompactor->pWriter->wSet);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFWriterClose(&pCompactor->pWriter, 1);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFReaderClose(&pCompactor->pReader);
TSDB_CHECK_CODE(code, lino, _exit);
/* do clear */
while ((pCompactor->pIter = pCompactor->iterList) != NULL) {
pCompactor->iterList = pCompactor->pIter->next;
tsdbCloseDataIter2(pCompactor->pIter);
}
tBlockDataReset(&pCompactor->bData);
tBlockDataReset(&pCompactor->sData);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code), pCompactor->fid);
} else {
tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid);
}
return code;
}
static int32_t tsdbCompactFileSet(STsdbCompactor *pCompactor, SDFileSet *pSet) {
int32_t code = 0;
int32_t lino = 0;
// start compact
code = tsdbCompactFileSetStart(pCompactor, pSet);
TSDB_CHECK_CODE(code, lino, _exit);
// do compact, end with a NULL row
SRowInfo *pRowInfo;
do {
code = tsdbCompactNextRow(pCompactor, &pRowInfo);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCompactWriteTableData(pCompactor, pRowInfo);
TSDB_CHECK_CODE(code, lino, _exit);
} while (pRowInfo);
// end compact
code = tsdbCompactFileSetEnd(pCompactor);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
tstrerror(code), pCompactor->fid);
if (pCompactor->pWriter) tsdbDataFWriterClose(&pCompactor->pWriter, 0);
while ((pCompactor->pIter = pCompactor->iterList)) {
pCompactor->iterList = pCompactor->pIter->next;
tsdbCloseDataIter2(pCompactor->pIter);
}
if (pCompactor->pReader) tsdbDataFReaderClose(&pCompactor->pReader);
}
return code;
}
static void tsdbEndCompact(STsdbCompactor *pCompactor) {
// writer
tBlockDataDestroy(&pCompactor->sData);
tBlockDataDestroy(&pCompactor->bData);
taosArrayDestroy(pCompactor->aSttBlk);
tMapDataClear(&pCompactor->mDataBlk);
taosArrayDestroy(pCompactor->aBlockIdx);
// reader
// tombstone
taosArrayDestroy(pCompactor->aSkyLine);
taosArrayDestroy(pCompactor->aDelData);
taosArrayDestroy(pCompactor->aDelIdx);
// others
tDestroyTSchema(pCompactor->tbSkm.pTSchema);
tsdbFSDestroy(&pCompactor->fs);
tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->commitID);
}
static int32_t tsdbBeginCompact(STsdb *pTsdb, STsdbCompactor *pCompactor) {
int32_t code = 0;
int32_t lino = 0;
pCompactor->pTsdb = pTsdb;
pCompactor->commitID = 0; // TODO
pCompactor->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pCompactor->maxRows = pTsdb->pVnode->config.tsdbCfg.maxRows;
pCompactor->minRows = pTsdb->pVnode->config.tsdbCfg.minRows;
pCompactor->fid = INT32_MIN;
code = tsdbFSCopy(pTsdb, &pCompactor->fs);
TSDB_CHECK_CODE(code, lino, _exit);
/* tombstone */
if (pCompactor->fs.pDelFile) {
code = tsdbDelFReaderOpen(&pCompactor->pDelFReader, pCompactor->fs.pDelFile, pTsdb);
TSDB_CHECK_CODE(code, lino, _exit);
if ((pCompactor->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if ((pCompactor->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if ((pCompactor->aSkyLine = taosArrayInit(0, sizeof(TSDBKEY))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbReadDelIdx(pCompactor->pDelFReader, pCompactor->aDelIdx);
TSDB_CHECK_CODE(code, lino, _exit);
}
/* reader */
/* writer */
code = tBlockDataCreate(&pCompactor->bData);
TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataCreate(&pCompactor->sData);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, lino,
tstrerror(code), pCompactor->commitID);
tBlockDataDestroy(&pCompactor->sData);
tBlockDataDestroy(&pCompactor->bData);
if (pCompactor->fs.pDelFile) {
taosArrayDestroy(pCompactor->aSkyLine);
taosArrayDestroy(pCompactor->aDelData);
taosArrayDestroy(pCompactor->aDelIdx);
if (pCompactor->pDelFReader) tsdbDelFReaderClose(&pCompactor->pDelFReader);
}
tsdbFSDestroy(&pCompactor->fs);
} else {
tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, pCompactor->commitID);
}
return code;
}
int32_t tsdbCompact(STsdb *pTsdb, int32_t flag) {
int32_t code = 0;
STsdbCompactor *pCompactor = &(STsdbCompactor){0};
if ((code = tsdbBeginCompact(pTsdb, pCompactor))) return code;
for (;;) {
SDFileSet *pSet = (SDFileSet *)taosArraySearch(pCompactor->fs.aDFileSet, &(SDFileSet){.fid = pCompactor->fid},
tDFileSetCmprFn, TD_GT);
if (pSet == NULL) {
pCompactor->fid = INT32_MAX;
break;
}
if ((code = tsdbCompactFileSet(pCompactor, pSet))) goto _exit;
}
if ((code = tsdbFSUpsertDelFile(&pCompactor->fs, NULL))) goto _exit;
_exit:
if (code) {
tsdbAbortCompact(pCompactor);
} else {
tsdbCommitCompact(pCompactor);
}
tsdbEndCompact(pCompactor);
return code;
}

458
source/dnode/vnode/src/tsdb/tsdbDataIter.c Normal file
View File

@ -0,0 +1,458 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
// STsdbDataIter2
/* open */
int32_t tsdbOpenDataFileDataIter(SDataFReader* pReader, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
// create handle
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_DATA_FILE_DATA_ITER;
pIter->dIter.pReader = pReader;
if ((pIter->dIter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pIter->dIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->dIter.iBlockIdx = 0;
pIter->dIter.iDataBlk = 0;
pIter->dIter.iRow = 0;
// read data
code = tsdbReadBlockIdx(pReader, pIter->dIter.aBlockIdx);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->dIter.aBlockIdx) == 0) goto _clear;
_exit:
if (code) {
if (pIter) {
_clear:
tBlockDataDestroy(&pIter->dIter.bData);
taosArrayDestroy(pIter->dIter.aBlockIdx);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
int32_t tsdbOpenSttFileDataIter(SDataFReader* pReader, int32_t iStt, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
// create handle
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_STT_FILE_DATA_ITER;
pIter->sIter.pReader = pReader;
pIter->sIter.iStt = iStt;
pIter->sIter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->sIter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pIter->sIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->sIter.iSttBlk = 0;
pIter->sIter.iRow = 0;
// read data
code = tsdbReadSttBlk(pReader, iStt, pIter->sIter.aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->sIter.aSttBlk) == 0) goto _clear;
_exit:
if (code) {
if (pIter) {
_clear:
taosArrayDestroy(pIter->sIter.aSttBlk);
tBlockDataDestroy(&pIter->sIter.bData);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
int32_t tsdbOpenTombFileDataIter(SDelFReader* pReader, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_TOMB_FILE_DATA_ITER;
pIter->tIter.pReader = pReader;
if ((pIter->tIter.aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if ((pIter->tIter.aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbReadDelIdx(pReader, pIter->tIter.aDelIdx);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->tIter.aDelIdx) == 0) goto _clear;
pIter->tIter.iDelIdx = 0;
pIter->tIter.iDelData = 0;
_exit:
if (code) {
if (pIter) {
_clear:
taosArrayDestroy(pIter->tIter.aDelIdx);
taosArrayDestroy(pIter->tIter.aDelData);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
/* close */
static void tsdbCloseDataFileDataIter(STsdbDataIter2* pIter) {
tBlockDataDestroy(&pIter->dIter.bData);
tMapDataClear(&pIter->dIter.mDataBlk);
taosArrayDestroy(pIter->dIter.aBlockIdx);
taosMemoryFree(pIter);
}
static void tsdbCloseSttFileDataIter(STsdbDataIter2* pIter) {
tBlockDataDestroy(&pIter->sIter.bData);
taosArrayDestroy(pIter->sIter.aSttBlk);
taosMemoryFree(pIter);
}
static void tsdbCloseTombFileDataIter(STsdbDataIter2* pIter) {
taosArrayDestroy(pIter->tIter.aDelData);
taosArrayDestroy(pIter->tIter.aDelIdx);
taosMemoryFree(pIter);
}
void tsdbCloseDataIter2(STsdbDataIter2* pIter) {
if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) {
ASSERT(0);
} else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) {
tsdbCloseDataFileDataIter(pIter);
} else if (pIter->type == TSDB_STT_FILE_DATA_ITER) {
tsdbCloseSttFileDataIter(pIter);
} else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) {
tsdbCloseTombFileDataIter(pIter);
} else {
ASSERT(0);
}
}
/* cmpr */
int32_t tsdbDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) {
STsdbDataIter2* pIter1 = TSDB_RBTN_TO_DATA_ITER(pNode1);
STsdbDataIter2* pIter2 = TSDB_RBTN_TO_DATA_ITER(pNode2);
return tRowInfoCmprFn(&pIter1->rowInfo, &pIter2->rowInfo);
}
/* seek */
/* iter next */
static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->dIter.iRow < pIter->dIter.bData.nRow) {
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) {
pIter->dIter.iRow = pIter->dIter.bData.nRow;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver ||
pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) {
pIter->dIter.iRow++;
continue;
}
}
}
ASSERT(pIter->rowInfo.suid == pIter->dIter.bData.suid);
ASSERT(pIter->rowInfo.uid = pIter->dIter.bData.uid);
pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->dIter.bData, pIter->dIter.iRow);
pIter->dIter.iRow++;
goto _exit;
}
for (;;) {
while (pIter->dIter.iDataBlk < pIter->dIter.mDataBlk.nItem) {
SDataBlk dataBlk;
tMapDataGetItemByIdx(&pIter->dIter.mDataBlk, pIter->dIter.iDataBlk, &dataBlk, tGetDataBlk);
// filter
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) {
pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) {
pIter->dIter.iDataBlk++;
continue;
}
}
}
code = tsdbReadDataBlockEx(pIter->dIter.pReader, &dataBlk, &pIter->dIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->dIter.iDataBlk++;
pIter->dIter.iRow = 0;
break;
}
if (pIter->dIter.iRow < pIter->dIter.bData.nRow) break;
for (;;) {
if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) {
SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx);
if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) {
int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
if (c == 0) {
pIter->dIter.iBlockIdx++;
continue;
} else if (c < 0) {
ASSERT(0);
}
}
code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->rowInfo.suid = pBlockIdx->suid;
pIter->rowInfo.uid = pBlockIdx->uid;
pIter->dIter.iBlockIdx++;
pIter->dIter.iDataBlk = 0;
break;
} else {
pIter->rowInfo = (SRowInfo){0};
goto _exit;
}
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->sIter.iRow < pIter->sIter.bData.nRow) {
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
if (pFilterInfo->tbid.uid == uid) {
pIter->sIter.iRow++;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] ||
pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) {
pIter->sIter.iRow++;
continue;
}
}
}
pIter->rowInfo.suid = pIter->sIter.bData.suid;
pIter->rowInfo.uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->sIter.bData, pIter->sIter.iRow);
pIter->sIter.iRow++;
goto _exit;
}
for (;;) {
if (pIter->sIter.iSttBlk < taosArrayGetSize(pIter->sIter.aSttBlk)) {
SSttBlk* pSttBlk = taosArrayGet(pIter->sIter.aSttBlk, pIter->sIter.iSttBlk);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
if (pSttBlk->suid == pFilterInfo->tbid.suid && pSttBlk->minUid == pFilterInfo->tbid.uid &&
pSttBlk->maxUid == pFilterInfo->tbid.uid) {
pIter->sIter.iSttBlk++;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pSttBlk->maxVer || pFilterInfo->ever < pSttBlk->minVer) {
pIter->sIter.iSttBlk++;
continue;
}
}
}
code = tsdbReadSttBlockEx(pIter->sIter.pReader, pIter->sIter.iStt, pSttBlk, &pIter->sIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->sIter.iRow = 0;
pIter->sIter.iSttBlk++;
break;
} else {
pIter->rowInfo = (SRowInfo){0};
goto _exit;
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->tIter.iDelData < taosArrayGetSize(pIter->tIter.aDelData)) {
SDelData* pDelData = taosArrayGet(pIter->tIter.aDelData, pIter->tIter.iDelData);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pDelData->version || pFilterInfo->ever < pDelData->version) {
pIter->tIter.iDelData++;
continue;
}
}
}
pIter->delInfo.delData = *pDelData;
pIter->tIter.iDelData++;
goto _exit;
}
for (;;) {
if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) {
SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx);
code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->delInfo.suid = pDelIdx->suid;
pIter->delInfo.uid = pDelIdx->uid;
pIter->tIter.iDelData = 0;
pIter->tIter.iDelIdx++;
break;
} else {
pIter->delInfo = (SDelInfo){0};
goto _exit;
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbDataIterNext2(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) {
ASSERT(0);
return code;
} else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) {
return tsdbDataFileDataIterNext(pIter, pFilterInfo);
} else if (pIter->type == TSDB_STT_FILE_DATA_ITER) {
return tsdbSttFileDataIterNext(pIter, pFilterInfo);
} else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) {
return tsdbTombFileDataIterNext(pIter, pFilterInfo);
} else {
ASSERT(0);
return code;
}
}
/* get */
// STsdbFSetIter
typedef struct STsdbFSetDataIter {
STsdb* pTsdb;
int32_t flags;
/* tombstone */
SDelFReader* pDelFReader;
SArray* aDelIdx; // SArray<SDelIdx>
SArray* aDelData; // SArray<SDelData>
SArray* aSkeyLine; // SArray<TABLEID>
int32_t iDelIdx;
int32_t iSkyLine;
/* time-series data */
SDataFReader* pReader;
STsdbDataIter2* iterList;
STsdbDataIter2* pIter;
SRBTree rbt;
} STsdbFSetDataIter;

27
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -629,7 +629,15 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) {
}
}
} else {
ASSERT(pTsdb->fs.pDelFile == NULL);
if (pTsdb->fs.pDelFile) {
nRef = atomic_sub_fetch_32(&pTsdb->fs.pDelFile->nRef, 1);
if (nRef == 0) {
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
(void)taosRemoveFile(fname);
taosMemoryFree(pTsdb->fs.pDelFile);
}
pTsdb->fs.pDelFile = NULL;
}
}
// aDFileSet
@ -906,14 +914,21 @@ _exit:
int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) {
int32_t code = 0;
if (pFS->pDelFile == NULL) {
pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
if (pDelFile) {
if (pFS->pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
if (pFS->pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
*pFS->pDelFile = *pDelFile;
} else {
if (pFS->pDelFile) {
taosMemoryFree(pFS->pDelFile);
pFS->pDelFile = NULL;
}
}
*pFS->pDelFile = *pDelFile;
_exit:
return code;

449
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -15,457 +15,10 @@
#include "tsdb.h"
extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData);
extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo);
extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg);
extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg);
// STsdbDataIter2 ========================================
#define TSDB_MEM_TABLE_DATA_ITER 0
#define TSDB_DATA_FILE_DATA_ITER 1
#define TSDB_STT_FILE_DATA_ITER 2
#define TSDB_TOMB_FILE_DATA_ITER 3
typedef struct STsdbDataIter2 STsdbDataIter2;
typedef struct STsdbFilterInfo STsdbFilterInfo;
typedef struct {
int64_t suid;
int64_t uid;
SDelData delData;
} SDelInfo;
struct STsdbDataIter2 {
STsdbDataIter2* next;
SRBTreeNode rbtn;
int32_t type;
SRowInfo rowInfo;
SDelInfo delInfo;
union {
// TSDB_MEM_TABLE_DATA_ITER
struct {
SMemTable* pMemTable;
} mIter;
// TSDB_DATA_FILE_DATA_ITER
struct {
SDataFReader* pReader;
SArray* aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk;
SBlockData bData;
int32_t iBlockIdx;
int32_t iDataBlk;
int32_t iRow;
} dIter;
// TSDB_STT_FILE_DATA_ITER
struct {
SDataFReader* pReader;
int32_t iStt;
SArray* aSttBlk;
SBlockData bData;
int32_t iSttBlk;
int32_t iRow;
} sIter;
// TSDB_TOMB_FILE_DATA_ITER
struct {
SDelFReader* pReader;
SArray* aDelIdx;
SArray* aDelData;
int32_t iDelIdx;
int32_t iDelData;
} tIter;
};
};
#define TSDB_FILTER_FLAG_BY_VERSION 0x1
struct STsdbFilterInfo {
int32_t flag;
int64_t sver;
int64_t ever;
};
#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2*)(((char*)pNode) - offsetof(STsdbDataIter2, rbtn)))
/* open */
static int32_t tsdbOpenDataFileDataIter(SDataFReader* pReader, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
// create handle
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_DATA_FILE_DATA_ITER;
pIter->dIter.pReader = pReader;
if ((pIter->dIter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pIter->dIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->dIter.iBlockIdx = 0;
pIter->dIter.iDataBlk = 0;
pIter->dIter.iRow = 0;
// read data
code = tsdbReadBlockIdx(pReader, pIter->dIter.aBlockIdx);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->dIter.aBlockIdx) == 0) goto _clear;
_exit:
if (code) {
if (pIter) {
_clear:
tBlockDataDestroy(&pIter->dIter.bData);
taosArrayDestroy(pIter->dIter.aBlockIdx);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
static int32_t tsdbOpenSttFileDataIter(SDataFReader* pReader, int32_t iStt, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
// create handle
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_STT_FILE_DATA_ITER;
pIter->sIter.pReader = pReader;
pIter->sIter.iStt = iStt;
pIter->sIter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->sIter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pIter->sIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->sIter.iSttBlk = 0;
pIter->sIter.iRow = 0;
// read data
code = tsdbReadSttBlk(pReader, iStt, pIter->sIter.aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->sIter.aSttBlk) == 0) goto _clear;
_exit:
if (code) {
if (pIter) {
_clear:
taosArrayDestroy(pIter->sIter.aSttBlk);
tBlockDataDestroy(&pIter->sIter.bData);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
static int32_t tsdbOpenTombFileDataIter(SDelFReader* pReader, STsdbDataIter2** ppIter) {
int32_t code = 0;
int32_t lino = 0;
STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter));
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pIter->type = TSDB_TOMB_FILE_DATA_ITER;
pIter->tIter.pReader = pReader;
if ((pIter->tIter.aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if ((pIter->tIter.aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbReadDelIdx(pReader, pIter->tIter.aDelIdx);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->tIter.aDelIdx) == 0) goto _clear;
pIter->tIter.iDelIdx = 0;
pIter->tIter.iDelData = 0;
_exit:
if (code) {
if (pIter) {
_clear:
taosArrayDestroy(pIter->tIter.aDelIdx);
taosArrayDestroy(pIter->tIter.aDelData);
taosMemoryFree(pIter);
pIter = NULL;
}
}
*ppIter = pIter;
return code;
}
/* close */
static void tsdbCloseDataFileDataIter(STsdbDataIter2* pIter) {
tBlockDataDestroy(&pIter->dIter.bData);
tMapDataClear(&pIter->dIter.mDataBlk);
taosArrayDestroy(pIter->dIter.aBlockIdx);
taosMemoryFree(pIter);
}
static void tsdbCloseSttFileDataIter(STsdbDataIter2* pIter) {
tBlockDataDestroy(&pIter->sIter.bData);
taosArrayDestroy(pIter->sIter.aSttBlk);
taosMemoryFree(pIter);
}
static void tsdbCloseTombFileDataIter(STsdbDataIter2* pIter) {
taosArrayDestroy(pIter->tIter.aDelData);
taosArrayDestroy(pIter->tIter.aDelIdx);
taosMemoryFree(pIter);
}
static void tsdbCloseDataIter2(STsdbDataIter2* pIter) {
if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) {
ASSERT(0);
} else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) {
tsdbCloseDataFileDataIter(pIter);
} else if (pIter->type == TSDB_STT_FILE_DATA_ITER) {
tsdbCloseSttFileDataIter(pIter);
} else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) {
tsdbCloseTombFileDataIter(pIter);
} else {
ASSERT(0);
}
}
/* cmpr */
static int32_t tsdbDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) {
STsdbDataIter2* pIter1 = TSDB_RBTN_TO_DATA_ITER(pNode1);
STsdbDataIter2* pIter2 = TSDB_RBTN_TO_DATA_ITER(pNode2);
return tRowInfoCmprFn(&pIter1->rowInfo, &pIter2->rowInfo);
}
/* seek */
/* iter next */
static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->dIter.iRow < pIter->dIter.bData.nRow) {
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver ||
pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) {
pIter->dIter.iRow++;
continue;
}
}
}
pIter->rowInfo.suid = pIter->dIter.bData.suid;
pIter->rowInfo.uid = pIter->dIter.bData.uid;
pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->dIter.bData, pIter->dIter.iRow);
pIter->dIter.iRow++;
goto _exit;
}
for (;;) {
while (pIter->dIter.iDataBlk < pIter->dIter.mDataBlk.nItem) {
SDataBlk dataBlk;
tMapDataGetItemByIdx(&pIter->dIter.mDataBlk, pIter->dIter.iDataBlk, &dataBlk, tGetDataBlk);
// filter
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) {
pIter->dIter.iDataBlk++;
continue;
}
}
}
code = tsdbReadDataBlockEx(pIter->dIter.pReader, &dataBlk, &pIter->dIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->dIter.iDataBlk++;
pIter->dIter.iRow = 0;
break;
}
if (pIter->dIter.iRow < pIter->dIter.bData.nRow) break;
for (;;) {
if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) {
SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx);
code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->dIter.iBlockIdx++;
pIter->dIter.iDataBlk = 0;
break;
} else {
pIter->rowInfo = (SRowInfo){0};
goto _exit;
}
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->sIter.iRow < pIter->sIter.bData.nRow) {
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] ||
pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) {
pIter->sIter.iRow++;
continue;
}
}
}
pIter->rowInfo.suid = pIter->sIter.bData.suid;
pIter->rowInfo.uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->sIter.bData, pIter->sIter.iRow);
pIter->sIter.iRow++;
goto _exit;
}
for (;;) {
if (pIter->sIter.iSttBlk < taosArrayGetSize(pIter->sIter.aSttBlk)) {
SSttBlk* pSttBlk = taosArrayGet(pIter->sIter.aSttBlk, pIter->sIter.iSttBlk);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pSttBlk->maxVer || pFilterInfo->ever < pSttBlk->minVer) {
pIter->sIter.iSttBlk++;
continue;
}
}
}
code = tsdbReadSttBlockEx(pIter->sIter.pReader, pIter->sIter.iStt, pSttBlk, &pIter->sIter.bData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->sIter.iRow = 0;
pIter->sIter.iSttBlk++;
break;
} else {
pIter->rowInfo = (SRowInfo){0};
goto _exit;
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
int32_t lino = 0;
for (;;) {
while (pIter->tIter.iDelData < taosArrayGetSize(pIter->tIter.aDelData)) {
SDelData* pDelData = taosArrayGet(pIter->tIter.aDelData, pIter->tIter.iDelData);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pDelData->version || pFilterInfo->ever < pDelData->version) {
pIter->tIter.iDelData++;
continue;
}
}
}
pIter->delInfo.delData = *pDelData;
pIter->tIter.iDelData++;
goto _exit;
}
for (;;) {
if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) {
SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx);
code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData);
TSDB_CHECK_CODE(code, lino, _exit);
pIter->delInfo.suid = pDelIdx->suid;
pIter->delInfo.uid = pDelIdx->uid;
pIter->tIter.iDelData = 0;
pIter->tIter.iDelIdx++;
break;
} else {
pIter->delInfo = (SDelInfo){0};
goto _exit;
}
}
}
_exit:
if (code) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbDataIterNext2(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) {
int32_t code = 0;
if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) {
ASSERT(0);
return code;
} else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) {
return tsdbDataFileDataIterNext(pIter, pFilterInfo);
} else if (pIter->type == TSDB_STT_FILE_DATA_ITER) {
return tsdbSttFileDataIterNext(pIter, pFilterInfo);
} else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) {
return tsdbTombFileDataIterNext(pIter, pFilterInfo);
} else {
ASSERT(0);
return code;
}
}
/* get */
// STsdbSnapReader ========================================
struct STsdbSnapReader {
STsdb* pTsdb;
@ -1318,7 +871,7 @@ static int32_t tsdbSnapWriteFileDataStart(STsdbSnapWriter* pWriter, int32_t fid)
TSDB_CHECK_CODE(code, lino, _exit);
if (pWriter->pSIter) {
code = tsdbSttFileDataIterNext(pWriter->pSIter, NULL);
code = tsdbDataIterNext2(pWriter->pSIter, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
// add to tree

76
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -1281,7 +1281,12 @@ void tBlockDataClear(SBlockData *pBlockData) {
}
}
static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) {
/* flag > 0: forward update
* flag == 0: insert
* flag < 0: backward update
*/
static int32_t tBlockDataUpsertBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow,
int32_t flag) {
int32_t code = 0;
SColVal cv = {0};
@ -1296,12 +1301,16 @@ static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlo
}
if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) {
code = tColDataAppendValue(pColDataTo, &COL_VAL_NONE(pColDataTo->cid, pColDataTo->type));
if (code) goto _exit;
cv = COL_VAL_NONE(pColDataTo->cid, pColDataTo->type);
if (flag == 0 && (code = tColDataAppendValue(pColDataTo, &cv))) goto _exit;
} else {
tColDataGetValue(pColDataFrom, iRow, &cv);
code = tColDataAppendValue(pColDataTo, &cv);
if (flag) {
code = tColDataUpdateValue(pColDataTo, &cv, flag > 0);
} else {
code = tColDataAppendValue(pColDataTo, &cv);
}
if (code) goto _exit;
pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL;
@ -1321,36 +1330,77 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
if (pBlockData->uid == 0) {
ASSERT(uid);
code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1));
if (code) goto _err;
if (code) goto _exit;
pBlockData->aUid[pBlockData->nRow] = uid;
}
// version
code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * (pBlockData->nRow + 1));
if (code) goto _err;
if (code) goto _exit;
pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow);
// timestamp
code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * (pBlockData->nRow + 1));
if (code) goto _err;
if (code) goto _exit;
pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow);
SColVal cv = {0};
if (pRow->type == TSDBROW_ROW_FMT) {
code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData);
if (code) goto _err;
code = tRowUpsertColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData, 0 /* append */);
if (code) goto _exit;
} else if (pRow->type == TSDBROW_COL_FMT) {
code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow);
if (code) goto _err;
code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, 0 /* append */);
if (code) goto _exit;
} else {
ASSERT(0);
}
pBlockData->nRow++;
_exit:
return code;
}
static int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) {
int32_t code = 0;
_err:
// version
int64_t lversion = pBlockData->aVersion[pBlockData->nRow - 1];
int64_t rversion = TSDBROW_VERSION(pRow);
ASSERT(lversion != rversion);
if (rversion > lversion) {
pBlockData->aVersion[pBlockData->nRow - 1] = rversion;
}
// update other rows
if (pRow->type == TSDBROW_ROW_FMT) {
code = tRowUpsertColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData,
(rversion > lversion) ? 1 : -1 /* update */);
if (code) goto _exit;
} else if (pRow->type == TSDBROW_COL_FMT) {
code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, (rversion > lversion) ? 1 : -1);
if (code) goto _exit;
} else {
ASSERT(0);
}
_exit:
return code;
}
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid) {
if (pBlockData->nRow == 0) {
return 1;
} else if (pBlockData->aTSKEY[pBlockData->nRow - 1] == TSDBROW_TS(pRow)) {
return pBlockData->nRow;
} else {
return pBlockData->nRow + 1;
}
}
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
if (pBlockData->aTSKEY[pBlockData->nRow - 1] == TSDBROW_TS(pRow)) {
return tBlockDataUpdateRow(pBlockData, pRow, pTSchema);
} else {
return tBlockDataAppendRow(pBlockData, pRow, pTSchema, uid);
}
}
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) {
ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
int32_t lidx = 0;

2
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -289,7 +289,7 @@ _err:
return -1;
}
static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
int32_t code = 0;
int32_t lino = 0;
char dir[TSDB_FILENAME_LEN] = {0};

81
source/dnode/vnode/src/vnd/vnodeCompact.c Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnd.h"
extern int32_t tsdbCompact(STsdb *pTsdb, int32_t flag);
int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo);
static int32_t vnodeCompactImpl(SCommitInfo *pInfo) {
int32_t code = 0;
int32_t lino = 0;
// TODO
SVnode *pVnode = pInfo->pVnode;
code = tsdbCompact(pVnode->pTsdb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d %s failed since %s", TD_VID(pInfo->pVnode), __func__, tstrerror(code));
} else {
vDebug("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__);
}
return code;
}
static int32_t vnodeCompactTask(void *param) {
int32_t code = 0;
SCommitInfo *pInfo = (SCommitInfo *)param;
// compact
vnodeCompactImpl(pInfo);
// end compact
tsem_post(&pInfo->pVnode->canCommit);
_exit:
taosMemoryFree(pInfo);
return code;
}
int32_t vnodeAsyncCompact(SVnode *pVnode) {
int32_t code = 0;
// schedule compact task
SCommitInfo *pInfo = taosMemoryCalloc(1, sizeof(*pInfo));
if (NULL == pInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
vnodePrepareCommit(pVnode, pInfo);
vnodeScheduleTask(vnodeCompactTask, pInfo);
_exit:
if (code) {
vError("vgId:%d %s failed since %s", TD_VID(pInfo->pVnode), __func__, tstrerror(code));
}
return code;
}
int32_t vnodeSyncCompact(SVnode *pVnode) {
vnodeAsyncCompact(pVnode);
tsem_wait(&pVnode->canCommit);
tsem_post(&pVnode->canCommit);
return 0;
}

19
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -29,6 +29,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t code = 0;
@ -318,6 +319,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
case TDMT_VND_COMMIT:
needCommit = true;
break;
case TDMT_VND_COMPACT:
vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp);
goto _exit;
default:
vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType);
return -1;
@ -1404,3 +1408,18 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
_err:
return code;
}
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SCompactVnodeReq req = {0};
if (tDeserializeSCompactVnodeReq(pReq, len, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return TSDB_CODE_INVALID_MSG;
}
vInfo("vgId:%d, compact msg will be processed, db:%s dbUid:%" PRId64 " compactStartTime:%" PRId64, TD_VID(pVnode),
req.db, req.dbUid, req.compactStartTime);
vnodeAsyncCompact(pVnode);
vnodeBegin(pVnode);
return 0;
}

18
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -165,6 +165,8 @@ const char* nodesNodeName(ENodeType type) {
return "ExplainStmt";
case QUERY_NODE_DESCRIBE_STMT:
return "DescribeStmt";
case QUERY_NODE_COMPACT_DATABASE_STMT:
return "CompactDatabaseStmt";
case QUERY_NODE_CREATE_STREAM_STMT:
return "CreateStreamStmt";
case QUERY_NODE_DROP_STREAM_STMT:
@ -5693,6 +5695,18 @@ static int32_t jsonToDescribeStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkCompactDatabaseStmtDbName = "DbName";
static int32_t compactDatabaseStmtToJson(const void* pObj, SJson* pJson) {
const SCompactDatabaseStmt* pNode = (const SCompactDatabaseStmt*)pObj;
return tjsonAddStringToObject(pJson, jkCompactDatabaseStmtDbName, pNode->dbName);
}
static int32_t jsonToCompactDatabaseStmt(const SJson* pJson, void* pObj) {
SCompactDatabaseStmt* pNode = (SCompactDatabaseStmt*)pObj;
return tjsonGetStringValue(pJson, jkCompactDatabaseStmtDbName, pNode->dbName);
}
static const char* jkCreateStreamStmtStreamName = "StreamName";
static const char* jkCreateStreamStmtTargetDbName = "TargetDbName";
static const char* jkCreateStreamStmtTargetTabName = "TargetTabName";
@ -6404,6 +6418,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return explainStmtToJson(pObj, pJson);
case QUERY_NODE_DESCRIBE_STMT:
return describeStmtToJson(pObj, pJson);
case QUERY_NODE_COMPACT_DATABASE_STMT:
return compactDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_STREAM_STMT:
return createStreamStmtToJson(pObj, pJson);
case QUERY_NODE_DROP_STREAM_STMT:
@ -6710,6 +6726,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToExplainStmt(pJson, pObj);
case QUERY_NODE_DESCRIBE_STMT:
return jsonToDescribeStmt(pJson, pObj);
case QUERY_NODE_COMPACT_DATABASE_STMT:
return jsonToCompactDatabaseStmt(pJson, pObj);
case QUERY_NODE_CREATE_STREAM_STMT:
return jsonToCreateStreamStmt(pJson, pObj);
case QUERY_NODE_DROP_STREAM_STMT:

6
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -374,8 +374,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SDescribeStmt));
case QUERY_NODE_RESET_QUERY_CACHE_STMT:
return makeNode(type, sizeof(SNode));
case QUERY_NODE_COMPACT_STMT:
break;
case QUERY_NODE_COMPACT_DATABASE_STMT:
return makeNode(type, sizeof(SCompactDatabaseStmt));
case QUERY_NODE_CREATE_FUNCTION_STMT:
return makeNode(type, sizeof(SCreateFunctionStmt));
case QUERY_NODE_DROP_FUNCTION_STMT:
@ -923,7 +923,7 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFree(((SDescribeStmt*)pNode)->pMeta);
break;
case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field
case QUERY_NODE_COMPACT_STMT: // no pointer field
case QUERY_NODE_COMPACT_DATABASE_STMT: // no pointer field
case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field
case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field
break;

2
source/libs/parser/inc/parAst.h
View File

@ -151,6 +151,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal);
@ -210,7 +211,6 @@ SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* p
SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery);
SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups);
SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName,
const SToken* pLibPath, SDataType dataType, int32_t bufSize);
SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);

6
source/libs/parser/inc/sql.y
View File

@ -167,6 +167,7 @@ cmd ::= USE db_name(A).
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
cmd ::= COMPACT DATABASE db_name(A). { pCxt->pRootNode = createCompactStmt(pCxt, &A); }
%type not_exists_opt { bool }
%destructor not_exists_opt { }
@ -528,9 +529,6 @@ explain_options(A) ::= .
explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C). { A = setExplainVerbose(pCxt, B, &C); }
explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C). { A = setExplainRatio(pCxt, B, &C); }
/************************************************ compact *************************************************************/
//cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
/************************************************ create/drop function ************************************************/
cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B)
AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E); }
@ -1106,6 +1104,6 @@ null_ordering_opt(A) ::= .
null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; }
null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; }
%fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL
%fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL
FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT
STRICT STRING TIMES UPDATE VALUES VARIABLE VIEW WAL.

18
source/libs/parser/src/parAstCreater.c
View File

@ -1105,6 +1105,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t
return (SNode*)pStmt;
}
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) {
return NULL;
}
SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)nodesMakeNode(QUERY_NODE_COMPACT_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
return (SNode*)pStmt;
}
SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt);
STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
@ -1768,13 +1779,6 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) {
return pStmt;
}
SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) {
CHECK_PARSER_STATUS(pCxt);
SNode* pStmt = nodesMakeNode(QUERY_NODE_COMPACT_STMT);
CHECK_OUT_OF_MEM(pStmt);
return pStmt;
}
SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName,
const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
CHECK_PARSER_STATUS(pCxt);

1
source/libs/parser/src/parTokenizer.c
View File

@ -61,6 +61,7 @@ static SKeyword keywordTable[] = {
{"COLUMN", TK_COLUMN},
{"COMMENT", TK_COMMENT},
{"COMP", TK_COMP},
{"COMPACT", TK_COMPACT},
{"CONNECTION", TK_CONNECTION},
{"CONNECTIONS", TK_CONNECTIONS},
{"CONNS", TK_CONNS},

13
source/libs/parser/src/parTranslater.c
View File

@ -5530,7 +5530,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt
}
static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) {
// todo
// The statement is executed directly on the client without constructing a message.
return TSDB_CODE_SUCCESS;
}
@ -5545,6 +5545,14 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
}
static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) {
SCompactDbReq compactReq = {0};
SName name;
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, compactReq.db);
return buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq);
}
static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) {
SKillConnReq killReq = {0};
killReq.connId = pStmt->targetId;
@ -6478,6 +6486,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DESCRIBE_STMT:
code = translateDescribe(pCxt, (SDescribeStmt*)pNode);
break;
case QUERY_NODE_COMPACT_DATABASE_STMT:
code = translateCompact(pCxt, (SCompactDatabaseStmt*)pNode);
break;
case QUERY_NODE_KILL_CONNECTION_STMT:
code = translateKillConnection(pCxt, (SKillStmt*)pNode);
break;

5005
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

20
source/libs/parser/test/parInitialCTest.cpp
View File

@ -21,6 +21,26 @@ namespace ParserTest {
class ParserInitialCTest : public ParserDdlTest {};
/*
* COMPACT DATABASE db_name
*/
TEST_F(ParserInitialCTest, compact) {
SCompactDbReq expect = {0};
auto setCompactDbReq = [&](const char* pDb) { snprintf(expect.db, sizeof(expect.db), "0.%s", pDb); };
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_COMPACT_DATABASE_STMT);
ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_COMPACT_DB);
SCompactDbReq req = {0};
ASSERT_EQ(tDeserializeSCompactDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
ASSERT_EQ(std::string(req.db), std::string(expect.db));
});
setCompactDbReq("wxy_db");
run("COMPACT DATABASE wxy_db");
}
/*
* CREATE ACCOUNT account_name PASS value [create_account_options]
*

20
source/util/src/trbtree.c
View File

@ -147,6 +147,10 @@ static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) {
void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) {
pTree->cmprFn = cmprFn;
tRBTreeClear(pTree);
}
void tRBTreeClear(SRBTree *pTree) {
pTree->n = 0;
pTree->NIL = &pTree->NILNODE;
pTree->NIL->color = BLACK;
@ -423,6 +427,22 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) {
return pNode;
}
SRBTreeNode *tRBTreeDropMin(SRBTree *pTree) {
SRBTreeNode *pNode = tRBTreeMin(pTree);
if (pNode) {
tRBTreeDrop(pTree, pNode);
}
return pNode;
}
SRBTreeNode *tRBTreeDropMax(SRBTree *pTree) {
SRBTreeNode *pNode = tRBTreeMax(pTree);
if (pNode) {
tRBTreeDrop(pTree, pNode);
}
return pNode;
}
SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) {
SRBTreeNode *pNode = pTree->root;