xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

This commit is contained in:
Minghao Li 2022-07-06 17:52:44 +08:00
parent 0e12b29ca9 7aebf05699
commit f35fccac04
112 changed files with 17047 additions and 15807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/test/CMakeLists.txt
View File

@ -24,3 +24,4 @@ if(${BUILD_WITH_TRAFT})
endif(${BUILD_WITH_TRAFT})
add_subdirectory(tdev)
add_subdirectory(lz4)

6
contrib/test/lz4/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
add_executable(lz4_test "")
target_sources(lz4_test
PRIVATE
"main.c"
)
target_link_libraries(lz4_test lz4_static)

8
contrib/test/lz4/main.c Normal file
View File

@ -0,0 +1,8 @@
#include <stdio.h>
#include "lz4.h"
int main(int argc, char const *argv[]) {
printf("%d\n", LZ4_compressBound(1024));
return 0;
}

2
examples/rust

@ -1 +1 @@
Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df
Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12

8
include/common/tcommon.h
View File

@ -73,15 +73,17 @@ typedef struct {
uint64_t suid;
} STableListInfo;
#pragma pack(push, 1)
typedef struct SColumnDataAgg {
int16_t colId;
int16_t maxIndex;
int16_t minIndex;
int16_t maxIndex;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
} SColumnDataAgg;
#pragma pack(pop)
typedef struct SDataBlockInfo {
STimeWindow window;
@ -122,13 +124,11 @@ typedef struct SColumnInfoData {
} SColumnInfoData;
typedef struct SQueryTableDataCond {
// STimeWindow twindow;
uint64_t suid;
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo* colList;
bool loadExternalRows; // load external rows or not
int32_t type; // data block load type:
int32_t type; // data block load type:
int32_t numOfTWindows;
STimeWindow* twindows;
int64_t startVersion;

36
include/common/tdataformat.h
View File

@ -34,21 +34,40 @@ typedef struct SValue SValue;
typedef struct SColVal SColVal;
typedef struct STSRow2 STSRow2;
typedef struct STSRowBuilder STSRowBuilder;
typedef struct SColData SColData;
typedef struct STagVal STagVal;
typedef struct STag STag;
// bitmap
#define N1(n) ((1 << (n)) - 1)
#define BIT1_SIZE(n) (((n)-1) / 8 + 1)
#define BIT2_SIZE(n) (((n)-1) / 4 + 1)
#define SET_BIT1(p, i, v) \
do { \
(p)[(i) / 8] &= N1((i) % 8); \
(p)[(i) / 8] |= (((uint8_t)(v)) << (((i) % 8))); \
} while (0)
#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
#define SET_BIT2(p, i, v) \
do { \
p[(i) / 4] &= N1((i) % 4 * 2); \
(p)[(i) / 4] |= (((uint8_t)(v)) << (((i) % 4) * 2)); \
} while (0)
#define GET_BIT2(p, i) (((p)[(i) / 4] >> (((i) % 4) * 2)) & ((uint8_t)3))
// STSchema
int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
void tTSchemaDestroy(STSchema *pTSchema);
// SValue
int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type);
int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
// STSRow2
#define COL_VAL_NONE(CID) ((SColVal){.cid = (CID), .isNone = 1})
#define COL_VAL_NULL(CID) ((SColVal){.cid = (CID), .isNull = 1})
#define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .value = (V)})
#define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1})
#define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1})
#define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)})
int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow);
int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow);
@ -140,6 +159,7 @@ struct SValue {
struct SColVal {
int16_t cid;
int8_t type;
int8_t isNone;
int8_t isNull;
SValue value;
@ -172,12 +192,6 @@ struct STag {
};
#pragma pack(pop)
struct SColData {
int16_t cid;
uint32_t nData;
uint8_t *pData;
};
#if 1 //================================================================================================================================================
// Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
#define TD_SUPPORT_BITMAP

11
include/common/tmsg.h
View File

@ -3024,6 +3024,17 @@ typedef struct {
int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq);
int32_t tDecodeSVDeleteRsp(SDecoder* pCoder, SVDeleteRsp* pReq);
typedef struct SDeleteRes {
uint64_t suid;
SArray* uidList;
int64_t skey;
int64_t ekey;
int64_t affectedRows;
} SDeleteRes;
int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);
int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes);
#pragma pack(pop)
#ifdef __cplusplus

1
include/common/tmsgdef.h
View File

@ -200,6 +200,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "commit vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_SCH_MSG)

4
include/common/trow.h
View File

@ -299,6 +299,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp
int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset,
int16_t colIdx);
int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx);
void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal);
typedef struct {
STSchema *pSchema;
@ -312,6 +313,7 @@ typedef struct {
void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow);
void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema);
int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow);
bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal);
bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal);
@ -320,7 +322,7 @@ STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema
int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode);
bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx,
SCellVal *pVal);
bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal);
bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal);
int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode);
void tdSCellValPrint(SCellVal *pVal, int8_t colType);
void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag);

389
include/common/ttokendef.h
View File

@ -73,200 +73,201 @@
#define TK_MNODE 55
#define TK_DATABASE 56
#define TK_USE 57
#define TK_IF 58
#define TK_NOT 59
#define TK_EXISTS 60
#define TK_BUFFER 61
#define TK_CACHELAST 62
#define TK_COMP 63
#define TK_DURATION 64
#define TK_NK_VARIABLE 65
#define TK_FSYNC 66
#define TK_MAXROWS 67
#define TK_MINROWS 68
#define TK_KEEP 69
#define TK_PAGES 70
#define TK_PAGESIZE 71
#define TK_PRECISION 72
#define TK_REPLICA 73
#define TK_STRICT 74
#define TK_WAL 75
#define TK_VGROUPS 76
#define TK_SINGLE_STABLE 77
#define TK_RETENTIONS 78
#define TK_SCHEMALESS 79
#define TK_NK_COLON 80
#define TK_TABLE 81
#define TK_NK_LP 82
#define TK_NK_RP 83
#define TK_STABLE 84
#define TK_ADD 85
#define TK_COLUMN 86
#define TK_MODIFY 87
#define TK_RENAME 88
#define TK_TAG 89
#define TK_SET 90
#define TK_NK_EQ 91
#define TK_USING 92
#define TK_TAGS 93
#define TK_COMMENT 94
#define TK_BOOL 95
#define TK_TINYINT 96
#define TK_SMALLINT 97
#define TK_INT 98
#define TK_INTEGER 99
#define TK_BIGINT 100
#define TK_FLOAT 101
#define TK_DOUBLE 102
#define TK_BINARY 103
#define TK_TIMESTAMP 104
#define TK_NCHAR 105
#define TK_UNSIGNED 106
#define TK_JSON 107
#define TK_VARCHAR 108
#define TK_MEDIUMBLOB 109
#define TK_BLOB 110
#define TK_VARBINARY 111
#define TK_DECIMAL 112
#define TK_MAX_DELAY 113
#define TK_WATERMARK 114
#define TK_ROLLUP 115
#define TK_TTL 116
#define TK_SMA 117
#define TK_FIRST 118
#define TK_LAST 119
#define TK_SHOW 120
#define TK_DATABASES 121
#define TK_TABLES 122
#define TK_STABLES 123
#define TK_MNODES 124
#define TK_MODULES 125
#define TK_QNODES 126
#define TK_FUNCTIONS 127
#define TK_INDEXES 128
#define TK_ACCOUNTS 129
#define TK_APPS 130
#define TK_CONNECTIONS 131
#define TK_LICENCE 132
#define TK_GRANTS 133
#define TK_QUERIES 134
#define TK_SCORES 135
#define TK_TOPICS 136
#define TK_VARIABLES 137
#define TK_BNODES 138
#define TK_SNODES 139
#define TK_CLUSTER 140
#define TK_TRANSACTIONS 141
#define TK_DISTRIBUTED 142
#define TK_CONSUMERS 143
#define TK_SUBSCRIPTIONS 144
#define TK_LIKE 145
#define TK_INDEX 146
#define TK_FUNCTION 147
#define TK_INTERVAL 148
#define TK_TOPIC 149
#define TK_AS 150
#define TK_WITH 151
#define TK_META 152
#define TK_CONSUMER 153
#define TK_GROUP 154
#define TK_DESC 155
#define TK_DESCRIBE 156
#define TK_RESET 157
#define TK_QUERY 158
#define TK_CACHE 159
#define TK_EXPLAIN 160
#define TK_ANALYZE 161
#define TK_VERBOSE 162
#define TK_NK_BOOL 163
#define TK_RATIO 164
#define TK_NK_FLOAT 165
#define TK_COMPACT 166
#define TK_VNODES 167
#define TK_IN 168
#define TK_OUTPUTTYPE 169
#define TK_AGGREGATE 170
#define TK_BUFSIZE 171
#define TK_STREAM 172
#define TK_INTO 173
#define TK_TRIGGER 174
#define TK_AT_ONCE 175
#define TK_WINDOW_CLOSE 176
#define TK_IGNORE 177
#define TK_EXPIRED 178
#define TK_KILL 179
#define TK_CONNECTION 180
#define TK_TRANSACTION 181
#define TK_BALANCE 182
#define TK_VGROUP 183
#define TK_MERGE 184
#define TK_REDISTRIBUTE 185
#define TK_SPLIT 186
#define TK_SYNCDB 187
#define TK_DELETE 188
#define TK_INSERT 189
#define TK_NULL 190
#define TK_NK_QUESTION 191
#define TK_NK_ARROW 192
#define TK_ROWTS 193
#define TK_TBNAME 194
#define TK_QSTARTTS 195
#define TK_QENDTS 196
#define TK_WSTARTTS 197
#define TK_WENDTS 198
#define TK_WDURATION 199
#define TK_CAST 200
#define TK_NOW 201
#define TK_TODAY 202
#define TK_TIMEZONE 203
#define TK_CLIENT_VERSION 204
#define TK_SERVER_VERSION 205
#define TK_SERVER_STATUS 206
#define TK_CURRENT_USER 207
#define TK_COUNT 208
#define TK_LAST_ROW 209
#define TK_BETWEEN 210
#define TK_IS 211
#define TK_NK_LT 212
#define TK_NK_GT 213
#define TK_NK_LE 214
#define TK_NK_GE 215
#define TK_NK_NE 216
#define TK_MATCH 217
#define TK_NMATCH 218
#define TK_CONTAINS 219
#define TK_JOIN 220
#define TK_INNER 221
#define TK_SELECT 222
#define TK_DISTINCT 223
#define TK_WHERE 224
#define TK_PARTITION 225
#define TK_BY 226
#define TK_SESSION 227
#define TK_STATE_WINDOW 228
#define TK_SLIDING 229
#define TK_FILL 230
#define TK_VALUE 231
#define TK_NONE 232
#define TK_PREV 233
#define TK_LINEAR 234
#define TK_NEXT 235
#define TK_HAVING 236
#define TK_RANGE 237
#define TK_EVERY 238
#define TK_ORDER 239
#define TK_SLIMIT 240
#define TK_SOFFSET 241
#define TK_LIMIT 242
#define TK_OFFSET 243
#define TK_ASC 244
#define TK_NULLS 245
#define TK_ID 246
#define TK_NK_BITNOT 247
#define TK_VALUES 248
#define TK_IMPORT 249
#define TK_NK_SEMI 250
#define TK_FILE 251
#define TK_FLUSH 58
#define TK_IF 59
#define TK_NOT 60
#define TK_EXISTS 61
#define TK_BUFFER 62
#define TK_CACHELAST 63
#define TK_COMP 64
#define TK_DURATION 65
#define TK_NK_VARIABLE 66
#define TK_FSYNC 67
#define TK_MAXROWS 68
#define TK_MINROWS 69
#define TK_KEEP 70
#define TK_PAGES 71
#define TK_PAGESIZE 72
#define TK_PRECISION 73
#define TK_REPLICA 74
#define TK_STRICT 75
#define TK_WAL 76
#define TK_VGROUPS 77
#define TK_SINGLE_STABLE 78
#define TK_RETENTIONS 79
#define TK_SCHEMALESS 80
#define TK_NK_COLON 81
#define TK_TABLE 82
#define TK_NK_LP 83
#define TK_NK_RP 84
#define TK_STABLE 85
#define TK_ADD 86
#define TK_COLUMN 87
#define TK_MODIFY 88
#define TK_RENAME 89
#define TK_TAG 90
#define TK_SET 91
#define TK_NK_EQ 92
#define TK_USING 93
#define TK_TAGS 94
#define TK_COMMENT 95
#define TK_BOOL 96
#define TK_TINYINT 97
#define TK_SMALLINT 98
#define TK_INT 99
#define TK_INTEGER 100
#define TK_BIGINT 101
#define TK_FLOAT 102
#define TK_DOUBLE 103
#define TK_BINARY 104
#define TK_TIMESTAMP 105
#define TK_NCHAR 106
#define TK_UNSIGNED 107
#define TK_JSON 108
#define TK_VARCHAR 109
#define TK_MEDIUMBLOB 110
#define TK_BLOB 111
#define TK_VARBINARY 112
#define TK_DECIMAL 113
#define TK_MAX_DELAY 114
#define TK_WATERMARK 115
#define TK_ROLLUP 116
#define TK_TTL 117
#define TK_SMA 118
#define TK_FIRST 119
#define TK_LAST 120
#define TK_SHOW 121
#define TK_DATABASES 122
#define TK_TABLES 123
#define TK_STABLES 124
#define TK_MNODES 125
#define TK_MODULES 126
#define TK_QNODES 127
#define TK_FUNCTIONS 128
#define TK_INDEXES 129
#define TK_ACCOUNTS 130
#define TK_APPS 131
#define TK_CONNECTIONS 132
#define TK_LICENCE 133
#define TK_GRANTS 134
#define TK_QUERIES 135
#define TK_SCORES 136
#define TK_TOPICS 137
#define TK_VARIABLES 138
#define TK_BNODES 139
#define TK_SNODES 140
#define TK_CLUSTER 141
#define TK_TRANSACTIONS 142
#define TK_DISTRIBUTED 143
#define TK_CONSUMERS 144
#define TK_SUBSCRIPTIONS 145
#define TK_LIKE 146
#define TK_INDEX 147
#define TK_FUNCTION 148
#define TK_INTERVAL 149
#define TK_TOPIC 150
#define TK_AS 151
#define TK_WITH 152
#define TK_META 153
#define TK_CONSUMER 154
#define TK_GROUP 155
#define TK_DESC 156
#define TK_DESCRIBE 157
#define TK_RESET 158
#define TK_QUERY 159
#define TK_CACHE 160
#define TK_EXPLAIN 161
#define TK_ANALYZE 162
#define TK_VERBOSE 163
#define TK_NK_BOOL 164
#define TK_RATIO 165
#define TK_NK_FLOAT 166
#define TK_COMPACT 167
#define TK_VNODES 168
#define TK_IN 169
#define TK_OUTPUTTYPE 170
#define TK_AGGREGATE 171
#define TK_BUFSIZE 172
#define TK_STREAM 173
#define TK_INTO 174
#define TK_TRIGGER 175
#define TK_AT_ONCE 176
#define TK_WINDOW_CLOSE 177
#define TK_IGNORE 178
#define TK_EXPIRED 179
#define TK_KILL 180
#define TK_CONNECTION 181
#define TK_TRANSACTION 182
#define TK_BALANCE 183
#define TK_VGROUP 184
#define TK_MERGE 185
#define TK_REDISTRIBUTE 186
#define TK_SPLIT 187
#define TK_SYNCDB 188
#define TK_DELETE 189
#define TK_INSERT 190
#define TK_NULL 191
#define TK_NK_QUESTION 192
#define TK_NK_ARROW 193
#define TK_ROWTS 194
#define TK_TBNAME 195
#define TK_QSTARTTS 196
#define TK_QENDTS 197
#define TK_WSTARTTS 198
#define TK_WENDTS 199
#define TK_WDURATION 200
#define TK_CAST 201
#define TK_NOW 202
#define TK_TODAY 203
#define TK_TIMEZONE 204
#define TK_CLIENT_VERSION 205
#define TK_SERVER_VERSION 206
#define TK_SERVER_STATUS 207
#define TK_CURRENT_USER 208
#define TK_COUNT 209
#define TK_LAST_ROW 210
#define TK_BETWEEN 211
#define TK_IS 212
#define TK_NK_LT 213
#define TK_NK_GT 214
#define TK_NK_LE 215
#define TK_NK_GE 216
#define TK_NK_NE 217
#define TK_MATCH 218
#define TK_NMATCH 219
#define TK_CONTAINS 220
#define TK_JOIN 221
#define TK_INNER 222
#define TK_SELECT 223
#define TK_DISTINCT 224
#define TK_WHERE 225
#define TK_PARTITION 226
#define TK_BY 227
#define TK_SESSION 228
#define TK_STATE_WINDOW 229
#define TK_SLIDING 230
#define TK_FILL 231
#define TK_VALUE 232
#define TK_NONE 233
#define TK_PREV 234
#define TK_LINEAR 235
#define TK_NEXT 236
#define TK_HAVING 237
#define TK_RANGE 238
#define TK_EVERY 239
#define TK_ORDER 240
#define TK_SLIMIT 241
#define TK_SOFFSET 242
#define TK_LIMIT 243
#define TK_OFFSET 244
#define TK_ASC 245
#define TK_NULLS 246
#define TK_ID 247
#define TK_NK_BITNOT 248
#define TK_VALUES 249
#define TK_IMPORT 250
#define TK_NK_SEMI 251
#define TK_FILE 252
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

2
include/libs/executor/executor.h
View File

@ -36,6 +36,8 @@ typedef struct SReadHandle {
void* vnode;
void* mnd;
SMsgCb* pMsgCb;
// int8_t initTsdbReader;
bool tqReader;
} SReadHandle;

5
include/libs/nodes/cmdnodes.h
View File

@ -97,6 +97,11 @@ typedef struct SAlterDatabaseStmt {
SDatabaseOptions* pOptions;
} SAlterDatabaseStmt;
typedef struct SFlushDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SFlushDatabaseStmt;
typedef struct STableOptions {
ENodeType type;
bool commentNull;

1
include/libs/nodes/nodes.h
View File

@ -111,6 +111,7 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT,
QUERY_NODE_FLUSH_DATABASE_STMT,
QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLE_STMT,

4
include/libs/nodes/plannodes.h
View File

@ -128,10 +128,12 @@ typedef struct SVnodeModifyLogicNode {
SVgDataBlocks* pVgDataBlocks;
SNode* pAffectedRows; // SColumnNode
uint64_t tableId;
uint64_t stableId;
int8_t tableType; // table type
char tableFName[TSDB_TABLE_FNAME_LEN];
STimeWindow deleteTimeRange;
SVgroupsInfo* pVgroupList;
SNodeList* pInsertCols;
} SVnodeModifyLogicNode;
typedef struct SExchangeLogicNode {
@ -459,7 +461,9 @@ typedef struct SDataInserterNode {
typedef struct SQueryInserterNode {
SDataSinkNode sink;
SNodeList* pCols;
uint64_t tableId;
uint64_t stableId;
int8_t tableType; // table type
char tableFName[TSDB_TABLE_FNAME_LEN];
int32_t vgId;

17
include/libs/qworker/qworker.h
View File

@ -20,9 +20,9 @@
extern "C" {
#endif
#include "executor.h"
#include "tmsgcb.h"
#include "trpc.h"
#include "executor.h"
enum {
NODE_TYPE_VNODE = 1,
@ -31,13 +31,6 @@ enum {
NODE_TYPE_MNODE,
};
typedef struct SDeleteRes {
uint64_t suid;
SArray* uidList;
int64_t skey;
int64_t ekey;
} SDeleteRes;
typedef struct SQWorkerCfg {
uint32_t maxSchedulerNum;
uint32_t maxTaskNum;
@ -46,19 +39,19 @@ typedef struct SQWorkerCfg {
typedef struct {
uint64_t cacheDataSize;
uint64_t queryProcessed;
uint64_t cqueryProcessed;
uint64_t fetchProcessed;
uint64_t dropProcessed;
uint64_t hbProcessed;
uint64_t deleteProcessed;
uint64_t numOfQueryInQueue;
uint64_t numOfFetchInQueue;
uint64_t timeInQueryQueue;
uint64_t timeInFetchQueue;
uint64_t numOfErrors;
} SQWorkerStat;
@ -82,7 +75,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SRpcMsg *pRsp, SDeleteRes *pRes);
int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SDeleteRes *pRes);
void qWorkerDestroy(void **qWorkerMgmt);

64
include/util/tRealloc.h Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_TREALLOC_H_
#define _TD_UTIL_TREALLOC_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
static FORCE_INLINE int32_t tRealloc(uint8_t **ppBuf, int64_t size) {
int32_t code = 0;
int64_t bsize = 0;
uint8_t *pBuf;
if (*ppBuf) {
bsize = *(int64_t *)((*ppBuf) - sizeof(int64_t));
}
if (bsize >= size) goto _exit;
if (bsize == 0) bsize = 64;
while (bsize < size) {
bsize *= 2;
}
pBuf = (uint8_t *)taosMemoryRealloc(*ppBuf ? (*ppBuf) - sizeof(int64_t) : *ppBuf, bsize + sizeof(int64_t));
if (pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
*(int64_t *)pBuf = bsize;
*ppBuf = pBuf + sizeof(int64_t);
_exit:
return code;
}
static FORCE_INLINE void tFree(uint8_t *pBuf) {
if (pBuf) {
taosMemoryFree(pBuf - sizeof(int64_t));
}
}
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_TREALLOC_H_*/

1
include/util/taoserror.h
View File

@ -72,6 +72,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_INVALID_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x0030)
#define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031)
#define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0032)
#define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0033)
#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040)
#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041)

55
include/util/tmallocator.h
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_MALLOCATOR_H_
#define _TD_UTIL_MALLOCATOR_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Memory allocator
#define TD_MEM_ALCT(TYPE) \
struct { \
void *(*malloc_)(struct TYPE *, uint64_t size); \
void (*free_)(struct TYPE *, void *ptr); \
}
#define TD_MA_MALLOC_FUNC(TMA) (TMA)->malloc_
#define TD_MA_FREE_FUNC(TMA) (TMA)->free_
#define TD_MA_MALLOC(TMA, SIZE) (*((TMA)->malloc_))(TMA, (SIZE))
#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR))
typedef struct SMemAllocator {
void *impl;
TD_MEM_ALCT(SMemAllocator);
} SMemAllocator;
#define tMalloc(pMA, SIZE) TD_MA_MALLOC(PMA, SIZE)
#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR)
typedef struct SMemAllocatorFactory {
void *impl;
SMemAllocator *(*create)(struct SMemAllocatorFactory *);
void (*destroy)(struct SMemAllocatorFactory *, SMemAllocator *);
} SMemAllocatorFactory;
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_MALLOCATOR_H_*/

123
source/client/test/clientTests.cpp
View File

@ -669,13 +669,13 @@ TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
@ -700,54 +700,55 @@ TEST(testCase, projection_query_tables) {
printf("create table :%d\n", i);
createNewTable(pConn, i);
}
// pRes = taos_query(pConn, "select * from tu");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
pRes = taos_query(pConn, "select * from tu");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
// taos_free_result(pRes);
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
//TEST(testCase, projection_query_stables) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "select ts from st1");
// if (taos_errno(pRes) != 0) {
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
TEST(testCase, projection_query_stables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "select ts from st1");
if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, agg_query_tables) {
@ -773,7 +774,7 @@ TEST(testCase, agg_query_tables) {
taos_free_result(pRes);
taos_close(pConn);
}
#endif
/*
--- copy the following script in the shell to setup the environment ---
@ -819,5 +820,27 @@ TEST(testCase, async_api_test) {
getchar();
taos_close(pConn);
}
#endif
TEST(testCase, update_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
taos_query(pConn, "use abc1");
TAOS_RES* pRes = taos_query(pConn, "create table tup (ts timestamp, k int);");
if (taos_errno(pRes) != 0) {
printf("failed to create table, reason:%s", taos_errstr(pRes));
}
taos_free_result(pRes);
char s[256] = {0};
for(int32_t i = 0; i < 7000; ++i) {
sprintf(s, "insert into tup values('2020-1-1 1:1:1', %d)", i);
pRes = taos_query(pConn, s);
taos_free_result(pRes);
}
}
#pragma GCC diagnostic pop

1
source/common/src/tdatablock.c
View File

@ -1831,6 +1831,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
pSubmitBlk->suid = suid;
pSubmitBlk->uid = pDataBlock->info.groupId;
pSubmitBlk->numOfRows = rows;
pSubmitBlk->sversion = pTSchema->version;
msgLen += sizeof(SSubmitBlk);
int32_t dataLen = 0;

26
source/common/src/tdataformat.c
View File

@ -29,15 +29,9 @@ typedef struct {
#pragma pack(pop)
#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
#define BIT1_SIZE(n) (((n)-1) / 8 + 1)
#define BIT2_SIZE(n) (((n)-1) / 4 + 1)
#define SET_BIT1(p, i, v) ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8)))
#define SET_BIT2(p, i, v) ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4)))
#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
#define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
// SValue
static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) {
@ -88,7 +82,7 @@ static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
return n;
}
static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) {
@ -421,7 +415,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
_set_none:
if ((flags & 0xf0) == 0) {
setBitMap(pb, 0, iColumn - 1, flags);
if (flags & TSROW_HAS_VAL) { // set 0
if (flags & TSROW_HAS_VAL) { // set 0
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
*(VarDataOffsetT *)(pf + pTColumn->offset) = 0;
} else {
@ -434,7 +428,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
_set_null:
if ((flags & 0xf0) == 0) {
setBitMap(pb, 1, iColumn - 1, flags);
if (flags & TSROW_HAS_VAL) { // set 0
if (flags & TSROW_HAS_VAL) { // set 0
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
*(VarDataOffsetT *)(pf + pTColumn->offset) = 0;
} else {
@ -639,15 +633,15 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal
}
_return_none:
*pColVal = COL_VAL_NONE(pTColumn->colId);
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
return;
_return_null:
*pColVal = COL_VAL_NULL(pTColumn->colId);
*pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
return;
_return_value:
*pColVal = COL_VAL_VALUE(pTColumn->colId, value);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value);
return;
}
@ -1105,9 +1099,9 @@ _err:
#if 1 // ===================================================================================================================
static void dataColSetNEleNull(SDataCol *pCol, int nEle);
int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
int spaceNeeded = pCol->bytes * maxPoints;
if (IS_VAR_DATA_TYPE(pCol->type)) {
spaceNeeded += sizeof(VarDataOffsetT) * maxPoints;
int spaceNeeded = pCol->bytes * maxPoints;
if (IS_VAR_DATA_TYPE(pCol->type)) {
spaceNeeded += sizeof(VarDataOffsetT) * maxPoints;
}
#ifdef TD_SUPPORT_BITMAP
int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(maxPoints);

31
source/common/src/tmsg.c
View File

@ -5445,6 +5445,37 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) {
return 0;
}
int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
int32_t nUid = taosArrayGetSize(pRes->uidList);
if (tEncodeU64(pCoder, pRes->suid) < 0) return -1;
if (tEncodeI32v(pCoder, nUid) < 0) return -1;
for (int32_t iUid = 0; iUid < nUid; iUid++) {
if (tEncodeU64(pCoder, *(uint64_t *)taosArrayGet(pRes->uidList, iUid)) < 0) return -1;
}
if (tEncodeI64(pCoder, pRes->skey) < 0) return -1;
if (tEncodeI64(pCoder, pRes->ekey) < 0) return -1;
if (tEncodeI64v(pCoder, pRes->affectedRows) < 0) return -1;
return 0;
}
int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
int32_t nUid;
uint64_t uid;
if (tDecodeU64(pCoder, &pRes->suid) < 0) return -1;
if (tDecodeI32v(pCoder, &nUid) < 0) return -1;
for (int32_t iUid = 0; iUid < nUid; iUid++) {
if (tDecodeU64(pCoder, &uid) < 0) return -1;
taosArrayPush(pRes->uidList, &uid);
}
if (tDecodeI64(pCoder, &pRes->skey) < 0) return -1;
if (tDecodeI64(pCoder, &pRes->ekey) < 0) return -1;
if (tDecodeI64v(pCoder, &pRes->affectedRows) < 0) return -1;
return 0;
}
int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1;
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;

167
source/common/src/trow.c
View File

@ -34,6 +34,7 @@ const uint8_t tdVTypeByte[2][3] = {{
// declaration
static uint8_t tdGetBitmapByte(uint8_t byte);
static int32_t tdCompareColId(const void *arg1, const void *arg2);
static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2);
// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
@ -339,9 +340,9 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8
}
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
uint8_t *qBitmap = (uint8_t*)pBitmap;
int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
uint8_t *qBitmap = (uint8_t *)pBitmap;
for (int i = 0; i < nBytes; ++i) {
if (*qBitmap != vTypeByte) {
return false;
@ -1045,13 +1046,28 @@ int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) {
return result;
}
bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) {
bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
return true;
}
int16_t nCols = tdRowGetNCols(pRow) - 1;
if (nCols <= 0) {
pVal->valType = TD_VTYPE_NONE;
return true;
}
SKvRowIdx *pColIdx =
(SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), nCols, sizeof(SKvRowIdx), compareKvRowColId, TD_EQ);
if (!pColIdx) {
pVal->valType = TD_VTYPE_NONE;
return true;
}
void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx);
tdGetKvRowValOfCol(pVal, pRow, pBitmap, pColIdx->offset,
POINTER_DISTANCE(pColIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx));
return true;
}
@ -1204,6 +1220,112 @@ static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2
}
}
int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
STColumn *pTColumn;
SColVal *pColVal;
int32_t nColVal = taosArrayGetSize(pArray);
int32_t varDataLen = 0;
int32_t maxVarDataLen = 0;
int32_t iColVal = 0;
void *varBuf = NULL;
ASSERT(nColVal > 1);
for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) {
pTColumn = &pTSchema->columns[iColumn];
if (iColVal < nColVal) {
pColVal = (SColVal *)taosArrayGet(pArray, iColVal);
} else {
pColVal = NULL;
}
if (iColumn == 0) {
ASSERT(pColVal->cid == pTColumn->colId);
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
} else {
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
if (pColVal) {
varDataLen += (pColVal->value.nData + sizeof(VarDataLenT));
if (maxVarDataLen < (pColVal->value.nData + sizeof(VarDataLenT))) {
maxVarDataLen = pColVal->value.nData + sizeof(VarDataLenT);
}
} else {
varDataLen += sizeof(VarDataLenT);
if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR) {
varDataLen += CHAR_BYTES;
if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) {
maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT);
}
} else {
varDataLen += INT_BYTES;
if (maxVarDataLen < INT_BYTES + sizeof(VarDataLenT)) {
maxVarDataLen = INT_BYTES + sizeof(VarDataLenT);
}
}
}
}
}
++iColVal;
}
*ppRow = (STSRow *)taosMemoryCalloc(
1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
if (!(*ppRow)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (maxVarDataLen > 0) {
varBuf = taosMemoryMalloc(maxVarDataLen);
if (!varBuf) {
taosMemoryFreeClear(*ppRow);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
SRowBuilder rb = {0};
tdSRowInit(&rb, pTSchema->version);
tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen);
tdSRowResetBuf(&rb, *ppRow);
iColVal = 0;
for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) {
pTColumn = &pTSchema->columns[iColumn];
TDRowValT valType = TD_VTYPE_NORM;
const void *val = NULL;
if (iColVal < nColVal) {
pColVal = (SColVal *)taosArrayGet(pArray, iColVal);
if (pColVal->isNone) {
valType = TD_VTYPE_NONE;
} else if (pColVal->isNull) {
valType = TD_VTYPE_NULL;
} else if (IS_VAR_DATA_TYPE(pTColumn->type)) {
varDataSetLen(varBuf, pColVal->value.nData);
memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
val = varBuf;
} else {
val = (const void *)&pColVal->value.i64;
}
} else {
pColVal = NULL;
valType = TD_VTYPE_NONE;
}
tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
++iColVal;
}
taosMemoryFreeClear(varBuf);
return 0;
}
bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
pVal->val = &pIter->pRow->ts;
@ -1593,7 +1715,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou
} else {
pBuilder->rowType = TD_ROW_TP;
}
pBuilder->flen = flen;
pBuilder->nCols = nCols;
pBuilder->nBoundCols = nBoundCols;
@ -1855,4 +1976,36 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) {
void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) {
pIter->pSchema = pSchema;
pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId;
}
}
void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) {
STColumn *pTColumn = &pTSchema->columns[iCol];
SCellVal cv;
SValue value;
ASSERT(iCol > 0);
if (TD_IS_TP_ROW(pRow)) {
tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv);
} else if (TD_IS_KV_ROW(pRow)) {
ASSERT(iCol > 0);
tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv);
} else {
ASSERT(0);
}
if (tdValTypeIsNone(cv.valType)) {
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
} else if (tdValTypeIsNull(cv.valType)) {
*pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
} else {
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
value.nData = varDataLen(cv.val);
value.pData = varDataVal(cv.val);
} else {
tGetValue(cv.val, &value, pTColumn->type);
}
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value);
}
}

1
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -350,6 +350,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;

4
source/dnode/vnode/CMakeLists.txt
View File

@ -44,8 +44,12 @@ target_sources(
"src/tsdb/tsdbMemTable.c"
"src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbCache.c"
"src/tsdb/tsdbWrite.c"
"src/tsdb/tsdbReaderWriter.c"
"src/tsdb/tsdbUtil.c"
"src/tsdb/tsdbSnapshot.c"
"src/tsdb/tsdbCacheRead.c"
# tq
"src/tq/tq.c"

51
source/dnode/vnode/inc/vnode.h
View File

@ -28,7 +28,6 @@
#include "tcommon.h"
#include "tfs.h"
#include "tmallocator.h"
#include "tmsg.h"
#include "trow.h"
@ -70,6 +69,10 @@ int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int
int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
void *vnodeGetIdx(SVnode *pVnode);
void *vnodeGetIvtIdx(SVnode *pVnode);
void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
@ -110,33 +113,31 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur);
// tsdb
// typedef struct STsdb STsdb;
typedef void *tsdbReaderT;
typedef struct STsdbReader STsdbReader;
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3
#define BLOCK_LOAD_OFFSET_ORDER 1
#define BLOCK_LOAD_TABLESEQ_ORDER 2
#define BLOCK_LOAD_EXTERN_ORDER 3
int32_t tsdbSetTableId(tsdbReaderT reader, int64_t uid);
int32_t tsdbSetTableList(tsdbReaderT reader, SArray *tableList);
tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
uint64_t taskId);
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
void *pMemRef);
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
bool isTsdbCacheLastRow(tsdbReaderT *pReader);
int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
int32_t tsdbGetStbIdList(SMeta *pMeta, int64_t suid, SArray *list);
void *tsdbGetIdx(SMeta *pMeta);
void *tsdbGetIvtIdx(SMeta *pMeta);
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
#define LASTROW_RETRIEVE_TYPE_ALL 0x1
#define LASTROW_RETRIEVE_TYPE_SINGLE 0x2
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg ***pBlockStatis, bool *allHave);
SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond, int32_t tWinIdx);
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
int32_t tsdbSetTableId(STsdbReader *pReader, int64_t uid);
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTableList, STsdbReader **ppReader,
const char *idstr);
void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader);
void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols,
void **pReader);
int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds);
int32_t tsdbLastrowReaderClose(void *pReader);
// tq

8
source/dnode/vnode/src/inc/sma.h
View File

@ -62,6 +62,7 @@ struct STSmaStat {
struct SRSmaStat {
SSma *pSma;
int64_t submitVer;
int64_t refId; // shared by fetch tasks
void *tmrHandle; // shared by fetch tasks
int8_t triggerStat; // shared by fetch tasks
@ -84,6 +85,7 @@ struct SSmaStat {
#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
#define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
#define RSMA_REF_ID(r) ((r)->refId)
#define RSMA_SUBMIT_VER(r) ((r)->submitVer)
enum {
TASK_TRIGGER_STAT_INIT = 0,
@ -208,11 +210,15 @@ struct STFInfo {
// specific fields
union {
struct {
int64_t applyVer[2];
int64_t submitVer;
} qTaskInfo;
};
};
enum {
TD_FTYPE_RSMA_QTASKINFO = 0,
};
struct STFile {
uint8_t state;
STFInfo info;

952
source/dnode/vnode/src/inc/tsdb.h
View File

File diff suppressed because it is too large Load Diff

14
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -20,6 +20,7 @@
#include "filter.h"
#include "qworker.h"
#include "sync.h"
#include "tRealloc.h"
#include "tchecksum.h"
#include "tcoding.h"
#include "tcompare.h"
@ -27,15 +28,15 @@
#include "tdatablock.h"
#include "tdb.h"
#include "tencode.h"
#include "tref.h"
#include "tfs.h"
#include "tglobal.h"
#include "tjson.h"
#include "tlist.h"
#include "tlockfree.h"
#include "tlosertree.h"
#include "tmallocator.h"
#include "tlrucache.h"
#include "tmsgcb.h"
#include "tref.h"
#include "tskiplist.h"
#include "tstream.h"
#include "ttime.h"
@ -94,6 +95,7 @@ int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema);
int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name);
int metaGetTbNum(SMeta* pMeta);
@ -127,9 +129,7 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
SSubmitBlkRsp* pRsp);
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
uint64_t taskId);
tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
void* pMemRef);
int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever);
int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
@ -157,6 +157,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list);
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId);
@ -171,6 +172,7 @@ int32_t smaPostCommit(SSma* pSma);
int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
int64_t tdRSmaGetMaxSubmitVer(SSma* pSma, int8_t level);
int32_t tdProcessRSmaCreate(SVnode* pVnode, SVCreateStbReq* pReq);
int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
@ -196,9 +198,9 @@ typedef struct {
// SVState
struct SVState {
// int64_t processed;
int64_t committed;
int64_t applied;
int64_t commitID;
};
struct SVnodeInfo {

115
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -31,7 +31,7 @@ void metaReaderClear(SMetaReader *pReader) {
}
int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
STbDbKey tbDbKey = {.version = version, .uid = uid};
// query table.db
@ -54,7 +54,7 @@ _err:
}
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
int64_t version;
// query uid.idx
@ -68,7 +68,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
}
int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
tb_uid_t uid;
// query name.idx
@ -82,7 +82,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
}
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
void * pData = NULL;
void *pData = NULL;
int nData = 0;
tb_uid_t uid = 0;
@ -138,7 +138,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
int metaTbCursorNext(SMTbCursor *pTbCur) {
int ret;
void * pBuf;
void *pBuf;
STbCfg tbCfg;
for (;;) {
@ -159,7 +159,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
void * pData = NULL;
void *pData = NULL;
int nData = 0;
int64_t version;
SSchemaWrapper schema = {0};
@ -218,9 +218,9 @@ _err:
return NULL;
}
int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){
TBC * pCur;
int ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL);
int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) {
TBC *pCur;
int ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL);
if (ret < 0) {
return ret;
}
@ -235,13 +235,13 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){
}
void *pKey = NULL;
int kLen = 0;
while(1){
int kLen = 0;
while (1) {
ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL);
if (ret < 0) {
break;
}
ttlKey = *(STtlIdxKey*)pKey;
ttlKey = *(STtlIdxKey *)pKey;
taosArrayPush(uidList, &ttlKey.uid);
}
tdbTbcClose(pCur);
@ -252,11 +252,11 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){
}
struct SMCtbCursor {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
@ -388,15 +388,15 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) {
if (ret < 0) {
return 0;
}
return *(tb_uid_t*)pStbCur->pKey;
return *(tb_uid_t *)pStbCur->pKey;
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
// SMetaReader mr = {0};
STSchema * pTSchema = NULL;
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
STSchemaBuilder sb = {0};
SSchema * pSchema;
SSchema *pSchema;
pSW = metaGetTableSchema(pMeta, uid, sver, 0);
if (!pSW) return NULL;
@ -415,6 +415,51 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
return pTSchema;
}
int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema **ppTSchema) {
int32_t code = 0;
STSchema *pTSchema = NULL;
SSkmDbKey skmDbKey = {.uid = suid ? suid : uid, .sver = sver};
void *pData = NULL;
int nData = 0;
// query
metaRLock(pMeta);
if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), &pData, &nData) < 0) {
code = TSDB_CODE_NOT_FOUND;
metaULock(pMeta);
goto _err;
}
metaULock(pMeta);
// decode
SDecoder dc = {0};
SSchemaWrapper schema;
SSchemaWrapper *pSchemaWrapper = &schema;
tDecoderInit(&dc, pData, nData);
tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
tDecoderClear(&dc);
// convert
STSchemaBuilder sb = {0};
tdInitTSchemaBuilder(&sb, pSchemaWrapper->version);
for (int i = 0; i < pSchemaWrapper->nCols; i++) {
SSchema *pSchema = pSchemaWrapper->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
*ppTSchema = pTSchema;
taosMemoryFree(pSchemaWrapper->pSchema);
return code;
_err:
*ppTSchema = NULL;
return code;
}
int metaGetTbNum(SMeta *pMeta) {
// TODO
// ASSERT(0);
@ -422,11 +467,11 @@ int metaGetTbNum(SMeta *pMeta) {
}
typedef struct {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t uid;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int kLen;
int vLen;
} SMSmaCursor;
@ -498,7 +543,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) {
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
STSmaWrapper *pSW = NULL;
SArray * pSmaIds = NULL;
SArray *pSmaIds = NULL;
if (!(pSmaIds = metaGetSmaIdsByTable(pMeta, uid))) {
return NULL;
@ -522,7 +567,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
metaReaderInit(&mr, pMeta, 0);
int64_t smaId;
int smaIdx = 0;
STSma * pTSma = NULL;
STSma *pTSma = NULL;
for (int i = 0; i < pSW->number; ++i) {
smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i);
if (metaGetTableEntryByUid(&mr, smaId) < 0) {
@ -570,7 +615,7 @@ _err:
}
STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
STSma * pTSma = NULL;
STSma *pTSma = NULL;
SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByUid(&mr, indexUid) < 0) {
@ -592,7 +637,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
}
SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
SArray * pUids = NULL;
SArray *pUids = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
@ -630,7 +675,7 @@ SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
}
SArray *metaGetSmaTbUids(SMeta *pMeta) {
SArray * pUids = NULL;
SArray *pUids = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t lastUid = 0;
@ -689,20 +734,20 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) {
}
typedef struct {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
int16_t cid;
int16_t type;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int32_t kLen;
int32_t vLen;
} SIdxCursor;
int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
SIdxCursor *pCursor = NULL;
char * buf = NULL;
char *buf = NULL;
int32_t maxSize = 0;
int32_t ret = 0, valid = 0;
@ -721,7 +766,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
int32_t nKey = 0;
int32_t nTagData = 0;
void * tagData = NULL;
void *tagData = NULL;
if (param->val == NULL) {
metaError("vgId:%d, failed to filter NULL data", TD_VID(pMeta->pVnode));
@ -757,7 +802,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
goto END;
}
void * entryKey = NULL, *entryVal = NULL;
void *entryKey = NULL, *entryVal = NULL;
int32_t nEntryKey, nEntryVal;
bool first = true;
while (1) {

49
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -523,6 +523,17 @@ static void tdDestroySDataBlockArray(SArray *pArray) {
taosArrayDestroy(pArray);
}
int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
if (level == TSDB_RETENTION_L0) {
return pSma->pVnode->state.applied;
}
SSmaEnv *pRSmaEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pRSmaStat = (SRSmaStat *)(SMA_ENV_STAT(pRSmaEnv));
return atomic_load_64(&pRSmaStat->submitVer);
}
static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
SArray *pResult = NULL;
SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
@ -562,7 +573,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
goto _err;
}
if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) {
if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
taosMemoryFreeClear(pReq);
goto _err;
}
@ -814,6 +825,7 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
}
if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
*committed = 0;
if (pVnode->state.committed > 0) {
smaWarn("vgId:%d, rsma restore for version %" PRIi64 ", not start as %s not exist", TD_VID(pVnode),
pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
@ -828,6 +840,18 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
goto _err;
}
STFInfo tFileInfo = {0};
if (tdLoadTFileHeader(&tFile, &tFileInfo) < 0) {
goto _err;
}
ASSERT(tFileInfo.qTaskInfo.submitVer > 0);
SSmaEnv *pRSmaEnv = pSma->pRSmaEnv;
SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pRSmaEnv);
atomic_store_64(&pRSmaStat->submitVer, tFileInfo.qTaskInfo.submitVer);
smaDebug("%s:%d tFileInfo.qTaskInfo.submitVer = %" PRIi64, __func__, __LINE__, tFileInfo.qTaskInfo.submitVer);
SRSmaQTaskInfoIter fIter = {0};
if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
tdRSmaQTaskInfoIterDestroy(&fIter);
@ -1094,6 +1118,22 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
}
STFile tFile = {0};
if (RSMA_SUBMIT_VER(pRSmaStat) > 0) {
char qTaskInfoFName[TSDB_FILENAME_LEN];
tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
goto _err;
}
if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) {
smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
goto _err;
}
smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile));
isFileCreated = true;
}
while (infoHash) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
@ -1129,12 +1169,12 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
goto _err;
}
if (tdCreateTFile(&tFile, true, -1) < 0) {
if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) {
smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
goto _err;
}
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
i + 1, TD_TFILE_FULL_NAME(&tFile));
smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
TD_TFILE_FULL_NAME(&tFile));
isFileCreated = true;
}
@ -1161,6 +1201,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
}
if (isFileCreated) {
tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->submitVer);
if (tdUpdateTFileHeader(&tFile) < 0) {
smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
tstrerror(terrno));

8
source/dnode/vnode/src/sma/smaUtil.c
View File

@ -32,6 +32,9 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) {
tlen += taosEncodeFixedU32(buf, pInfo->ftype);
tlen += taosEncodeFixedU32(buf, pInfo->fver);
tlen += taosEncodeFixedI64(buf, pInfo->fsize);
if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) {
tlen += taosEncodeFixedI64(buf, pInfo->qTaskInfo.submitVer);
}
return tlen;
}
@ -41,6 +44,11 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) {
buf = taosDecodeFixedU32(buf, &(pInfo->ftype));
buf = taosDecodeFixedU32(buf, &(pInfo->fver));
buf = taosDecodeFixedI64(buf, &(pInfo->fsize));
// specific
if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) {
buf = taosDecodeFixedI64(buf, &(pInfo->qTaskInfo.submitVer));
}
return buf;
}

4
source/dnode/vnode/src/tq/tq.c
View File

@ -462,8 +462,8 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
pHandle->execHandle.execTb.suid = req.suid;
SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
tsdbGetCtbIdList(pTq->pVnode->pMeta, req.suid, tbUidList);
tqDebug("vg %d, tq try get suid: %ld", TD_VID(pTq->pVnode), req.suid);
vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid);
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid);

1
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -87,6 +87,7 @@ int32_t tqMetaOpen(STQ* pTq) {
.reader = handle.execHandle.pExecReader[i],
.meta = pTq->pVnode->pMeta,
.pMsgCb = &pTq->pVnode->msgCb,
.vnode = pTq->pVnode,
};
handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
ASSERT(handle.execHandle.execCol.task[i]);

1178
source/dnode/vnode/src/tsdb/tsdbCache.c Normal file
View File

File diff suppressed because it is too large Load Diff

175
source/dnode/vnode/src/tsdb/tsdbCacheRead.c Normal file
View File

@ -0,0 +1,175 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "taoserror.h"
#include "tarray.h"
#include "tcommon.h"
#include "tsdb.h"
typedef struct SLastrowReader {
SVnode* pVnode;
STSchema* pSchema;
uint64_t uid;
// int32_t* pSlotIds;
char** transferBuf; // todo remove it soon
int32_t numOfCols;
int32_t type;
int32_t tableIndex; // currently returned result tables
SArray* pTableList; // table id list
} SLastrowReader;
static void saveOneRow(STSRow* pRow, SSDataBlock* pBlock, SLastrowReader* pReader, const int32_t* slotIds) {
int32_t numOfRows = pBlock->info.rows;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
SColVal colVal = {0};
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (slotIds[i] == -1) {
colDataAppend(pColInfoData, numOfRows, (const char*)&pRow->ts, false);
} else {
tTSRowGetVal(pRow, pReader->pSchema, slotIds[i], &colVal);
if (IS_VAR_DATA_TYPE(colVal.type)) {
if (colVal.isNull || colVal.isNone) {
colDataAppendNULL(pColInfoData, numOfRows);
} else {
varDataSetLen(pReader->transferBuf[i], colVal.value.nData);
memcpy(varDataVal(pReader->transferBuf[i]), colVal.value.pData, colVal.value.nData);
colDataAppend(pColInfoData, numOfRows, pReader->transferBuf[i], false);
}
} else {
colDataAppend(pColInfoData, numOfRows, (const char*)&colVal.value, colVal.isNull || colVal.isNone);
}
}
}
pBlock->info.rows += 1;
}
int32_t tsdbLastRowReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList, int32_t* colId, int32_t numOfCols,
void** pReader) {
SLastrowReader* p = taosMemoryCalloc(1, sizeof(SLastrowReader));
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
p->type = type;
p->pVnode = pVnode;
p->numOfCols = numOfCols;
p->transferBuf = taosMemoryCalloc(p->numOfCols, POINTER_BYTES);
STableKeyInfo* pKeyInfo = taosArrayGet(pTableIdList, 0);
p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1);
p->pTableList = pTableIdList;
for(int32_t i = 0; i < p->numOfCols; ++i) {
if (IS_VAR_DATA_TYPE(colId[i])) {
p->transferBuf[i] = taosMemoryMalloc(p->pSchema->columns[i].bytes);
}
}
*pReader = p;
return TSDB_CODE_SUCCESS;
}
int32_t tsdbLastrowReaderClose(void* pReader) {
SLastrowReader* p = pReader;
for (int32_t i = 0; i < p->numOfCols; ++i) {
taosMemoryFreeClear(p->transferBuf[i]);
}
taosMemoryFree(p->transferBuf);
taosMemoryFree(pReader);
return TSDB_CODE_SUCCESS;
}
int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds) {
if (pReader == NULL || pResBlock == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SLastrowReader* pr = pReader;
SLRUCache* lruCache = pr->pVnode->pTsdb->lruCache;
LRUHandle* h = NULL;
STSRow* pRow = NULL;
size_t numOfTables = taosArrayGetSize(pr->pTableList);
// retrieve the only one last row of all tables in the uid list.
if (pr->type == LASTROW_RETRIEVE_TYPE_SINGLE) {
int64_t lastKey = INT64_MIN;
bool internalResult = false;
for (int32_t i = 0; i < numOfTables; ++i) {
STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i);
int32_t code = tsdbCacheGetLastrowH(lruCache, pKeyInfo->uid, pr->pVnode->pTsdb, &h);
// int32_t code = tsdbCacheGetLastH(lruCache, pKeyInfo->uid, pr->pVnode->pTsdb, &h);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (h == NULL) {
continue;
}
pRow = (STSRow*)taosLRUCacheValue(lruCache, h);
if (pRow->ts > lastKey) {
// Set result row into the same rowIndex repeatly, so we need to check if the internal result row has already
// appended or not.
if (internalResult) {
pResBlock->info.rows -= 1;
}
saveOneRow(pRow, pResBlock, pr, slotIds);
internalResult = true;
lastKey = pRow->ts;
}
tsdbCacheRelease(lruCache, h);
}
} else if (pr->type == LASTROW_RETRIEVE_TYPE_ALL) {
for (int32_t i = pr->tableIndex; i < numOfTables; ++i) {
STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i);
int32_t code = tsdbCacheGetLastrowH(lruCache, pKeyInfo->uid, pr->pVnode->pTsdb, &h);
// int32_t code = tsdbCacheGetLastH(lruCache, pKeyInfo->uid, pr->pVnode->pTsdb, &h);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// no data in the table of Uid
if (h == NULL) {
continue;
}
pRow = (STSRow*)taosLRUCacheValue(lruCache, h);
saveOneRow(pRow, pResBlock, pr, slotIds);
tsdbCacheRelease(lruCache, h);
pr->tableIndex += 1;
if (pResBlock->info.rows >= pResBlock->info.capacity) {
return TSDB_CODE_SUCCESS;
}
}
} else {
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
}

2686
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

File diff suppressed because it is too large Load Diff

14
source/dnode/vnode/src/tsdb/tsdbDelete.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

1621
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

File diff suppressed because it is too large Load Diff

719
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -15,568 +15,287 @@
#include "tsdb.h"
static const char *TSDB_FNAME_SUFFIX[] = {
"head", // TSDB_FILE_HEAD
"data", // TSDB_FILE_DATA
"last", // TSDB_FILE_LAST
"smad", // TSDB_FILE_SMAD
"smal", // TSDB_FILE_SMAL
"", // TSDB_FILE_MAX
"meta", // TSDB_FILE_META
};
static int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t n = 0;
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname);
static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
static int tsdbRollBackDFile(SDFile *pDFile);
n += tPutI64v(p ? p + n : p, pHeadFile->commitID);
n += tPutI64v(p ? p + n : p, pHeadFile->size);
n += tPutI64v(p ? p + n : p, pHeadFile->offset);
// ============== Operations on SDFile
void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
char fname[TSDB_FILENAME_LEN];
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK);
TSDB_FILE_SET_CLOSED(pDFile);
memset(&(pDFile->info), 0, sizeof(pDFile->info));
pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
pDFile->info.fver = tsdbGetDFSVersion(ftype);
tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname);
tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
return n;
}
void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
*pDFile = *pODFile;
TSDB_FILE_SET_CLOSED(pDFile);
static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t n = 0;
n += tGetI64v(p + n, &pHeadFile->commitID);
n += tGetI64v(p + n, &pHeadFile->size);
n += tGetI64v(p + n, &pHeadFile->offset);
return n;
}
int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
int tlen = 0;
static int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t n = 0;
tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
tlen += tfsEncodeFile(buf, &(pDFile->f));
n += tPutI64v(p ? p + n : p, pDataFile->commitID);
n += tPutI64v(p ? p + n : p, pDataFile->size);
return tlen;
return n;
}
void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile) {
buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
buf = tfsDecodeFile(REPO_TFS(pRepo), buf, &(pDFile->f));
TSDB_FILE_SET_CLOSED(pDFile);
static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t n = 0;
return buf;
n += tGetI64v(p + n, &pDataFile->commitID);
n += tGetI64v(p + n, &pDataFile->size);
return n;
}
static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) {
int tlen = 0;
static int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) {
int32_t n = 0;
tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile));
n += tPutI64v(p ? p + n : p, pLastFile->commitID);
n += tPutI64v(p ? p + n : p, pLastFile->size);
return tlen;
return n;
}
static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
char *aname = NULL;
static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) {
int32_t n = 0;
buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
buf = taosDecodeString(buf, &aname);
strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN);
TSDB_FILE_SET_CLOSED(pDFile);
taosMemoryFreeClear(aname);
n += tGetI64v(p + n, &pLastFile->commitID);
n += tGetI64v(p + n, &pLastFile->size);
return buf;
return n;
}
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
static int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
int32_t n = 0;
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile == NULL) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
if (tfsMkdirRecurAt(REPO_TFS(pRepo), taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) {
taosMemoryFreeClear(s);
return -1;
}
taosMemoryFreeClear(s);
n += tPutI64v(p ? p + n : p, pSmaFile->commitID);
n += tPutI64v(p ? p + n : p, pSmaFile->size);
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
} else {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (!updateHeader) {
return 0;
}
pDFile->info.size += TSDB_FILE_HEAD_SIZE;
pDFile->info.fver = tsdbGetDFSVersion(fType);
if (tsdbUpdateDFileHeader(pDFile) < 0) {
tsdbCloseDFile(pDFile);
tsdbRemoveDFile(pDFile);
return -1;
}
return 0;
return n;
}
int tsdbUpdateDFileHeader(SDFile *pDFile) {
char buf[TSDB_FILE_HEAD_SIZE] = "\0";
static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
int32_t n = 0;
if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
return -1;
}
n += tGetI64v(p + n, &pSmaFile->commitID);
n += tGetI64v(p + n, &pSmaFile->size);
void *ptr = buf;
// taosEncodeFixedU32(&ptr, 0); // fver moved to SDFInfo and saved to current
tsdbEncodeDFInfo(&ptr, &(pDFile->info));
taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
return -1;
}
return 0;
return n;
}
int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
char buf[TSDB_FILE_HEAD_SIZE] = "\0";
uint32_t _version;
// EXPOSED APIS ==================================================
void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]) {
STfs *pTfs = pTsdb->pVnode->pTfs;
ASSERT(TSDB_FILE_OPENED(pDFile));
if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
return -1;
}
if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
void *pBuf = buf;
// pBuf = taosDecodeFixedU32(pBuf, &_version);
pBuf = tsdbDecodeDFInfo(pBuf, pInfo);
return 0;
}
static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
SDFile df;
tsdbInitDFileEx(&df, pDFile);
if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) {
tsdbError("vgId:%d, data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile));
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
return 0;
}
int64_t file_size = 0;
if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (pDFile->info.size < file_size) {
// if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1;
}
if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
}
if (tsdbUpdateDFileHeader(&df) < 0) {
tsdbCloseDFile(&df);
return -1;
}
tsdbCloseDFile(&df);
tsdbInfo("vgId:%d, file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
file_size, pDFile->info.size);
} else if (pDFile->info.size > file_size) {
tsdbError("vgId:%d, data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size);
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return 0;
} else {
tsdbDebug("vgId:%d, file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile));
}
return 0;
}
static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) {
int tlen = 0;
tlen += taosEncodeFixedU32(buf, pInfo->magic);
tlen += taosEncodeFixedU32(buf, pInfo->fver);
tlen += taosEncodeFixedU32(buf, pInfo->len);
tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
tlen += taosEncodeFixedU32(buf, pInfo->offset);
tlen += taosEncodeFixedU64(buf, pInfo->size);
tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
return tlen;
}
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) {
buf = taosDecodeFixedU32(buf, &(pInfo->magic));
buf = taosDecodeFixedU32(buf, &(pInfo->fver));
buf = taosDecodeFixedU32(buf, &(pInfo->len));
buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
buf = taosDecodeFixedU32(buf, &(pInfo->offset));
buf = taosDecodeFixedU64(buf, &(pInfo->size));
buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
return buf;
}
static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
ASSERT(from != NULL || to != NULL);
if (from != NULL) {
if (to == NULL) {
tsdbRemoveDFile(from);
} else {
if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
if (from->info.size > to->info.size) {
tsdbRollBackDFile(to);
}
} else {
(void)tsdbRemoveDFile(from);
}
}
}
return 0;
}
static int tsdbRollBackDFile(SDFile *pDFile) {
SDFile df = *pDFile;
// if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1;
}
if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
}
if (tsdbUpdateDFileHeader(&df) < 0) {
tsdbCloseDFile(&df);
return -1;
}
TSDB_FILE_FSYNC(&df);
tsdbCloseDFile(&df);
return 0;
}
// ============== Operations on SDFileSet
void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) {
TSDB_FSET_FID(pSet) = fid;
TSDB_FSET_VER(pSet) = TSDB_LATEST_FSET_VER;
TSDB_FSET_STATE(pSet) = 0;
pSet->reserve = 0;
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
tsdbInitDFile(pRepo, pDFile, did, fid, ver, ftype);
}
}
void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) {
TSDB_FSET_FID(pSet) = TSDB_FSET_FID(pOSet);
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype));
}
}
int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet));
// state not included
tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet));
tlen += taosEncodeFixedU16(buf, pSet->reserve);
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
}
return tlen;
}
void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) {
buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet)));
TSDB_FSET_STATE(pSet) = 0;
buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet)));
buf = taosDecodeFixedU16(buf, &(pSet->reserve));
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype));
}
return buf;
}
int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) {
int tlen = 0;
tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet));
tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet));
tlen += taosEncodeFixedU16(buf, pSet->reserve);
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
}
return tlen;
}
void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) {
buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet)));
buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet)));
buf = taosDecodeFixedU16(buf, &(pSet->reserve));
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
}
return buf;
}
int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL;
SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL;
if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) {
return -1;
}
}
return 0;
}
int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader, ftype) < 0) {
tsdbCloseDFileSet(pSet);
tsdbRemoveDFileSet(pSet);
return -1;
}
}
return 0;
}
int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
return -1;
}
}
return 0;
}
int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
return -1;
}
}
return 0;
}
int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *_version) {
char *p = NULL;
*_version = 0;
*ftype = TSDB_FILE_MAX;
sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, _version);
for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) {
if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) {
*ftype = i;
switch (ftype) {
case TSDB_HEAD_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fHead.commitID,
".head");
break;
case TSDB_DATA_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fData.commitID,
".data");
break;
case TSDB_LAST_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fLast.commitID,
".last");
break;
case TSDB_SMA_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fSma.commitID,
".sma");
break;
default:
ASSERT(0);
break;
}
}
taosMemoryFreeClear(p);
return 0;
}
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) {
ASSERT(ftype != TSDB_FILE_MAX);
if (ftype < TSDB_FILE_MAX) {
if (ver == 0) {
snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid,
TSDB_FNAME_SUFFIX[ftype]);
} else {
snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid,
TSDB_FNAME_SUFFIX[ftype], ver);
}
} else {
if (ver == 0) {
snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]);
} else {
snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
}
}
}
int tsdbOpenDFile(SDFile *pDFile, int flags) {
ASSERT(!TSDB_FILE_OPENED(pDFile));
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype) {
if (pDFileSet1->diskId.level != pDFileSet2->diskId.level || pDFileSet1->diskId.id != pDFileSet2->diskId.id) {
return false;
}
return 0;
}
void tsdbCloseDFile(SDFile *pDFile) {
if (TSDB_FILE_OPENED(pDFile)) {
taosCloseFile(&pDFile->pFile);
TSDB_FILE_SET_CLOSED(pDFile);
switch (ftype) {
case TSDB_HEAD_FILE:
return pDFileSet1->fHead.commitID == pDFileSet2->fHead.commitID;
case TSDB_DATA_FILE:
return pDFileSet1->fData.commitID == pDFileSet2->fData.commitID;
case TSDB_LAST_FILE:
return pDFileSet1->fLast.commitID == pDFileSet2->fLast.commitID;
case TSDB_SMA_FILE:
return pDFileSet1->fSma.commitID == pDFileSet2->fSma.commitID;
default:
ASSERT(0);
break;
}
}
int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) {
// ASSERT(TSDB_FILE_OPENED(pDFile));
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0;
int64_t n;
char hdr[TSDB_FHDR_SIZE];
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
memset(hdr, 0, TSDB_FHDR_SIZE);
tPutDataFileHdr(hdr, pSet, ftype);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
return loffset;
n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
_exit:
return code;
}
int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0;
int64_t size;
TdFilePtr pFD;
char fname[TSDB_FILENAME_LEN];
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
if (nwrite < nbyte) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
tsdbDataFileName(pTsdb, pSet, ftype, fname);
// open
pFD = taosOpenFile(fname, TD_FILE_WRITE);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
return nwrite;
// truncate
switch (ftype) {
case TSDB_HEAD_FILE:
size = pSet->fHead.size;
break;
case TSDB_DATA_FILE:
size = pSet->fData.size;
break;
case TSDB_LAST_FILE:
size = pSet->fLast.size;
break;
case TSDB_SMA_FILE:
size = pSet->fSma.size;
break;
default:
ASSERT(0);
}
if (taosFtruncateFile(pFD, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// update header
code = tsdbUpdateDFileHdr(pFD, pSet, ftype);
if (code) goto _err;
// sync
if (taosFsyncFile(pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// close
taosCloseFile(&pFD);
return code;
_err:
return code;
}
void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) {
pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM));
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype) {
int32_t n = 0;
switch (ftype) {
case TSDB_HEAD_FILE:
n += tPutHeadFile(p ? p + n : p, &pSet->fHead);
break;
case TSDB_DATA_FILE:
n += tPutDataFile(p ? p + n : p, &pSet->fData);
break;
case TSDB_LAST_FILE:
n += tPutLastFile(p ? p + n : p, &pSet->fLast);
break;
case TSDB_SMA_FILE:
n += tPutSmaFile(p ? p + n : p, &pSet->fSma);
break;
default:
ASSERT(0);
}
return n;
}
int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0;
int64_t toffset;
n += tPutI32v(p ? p + n : p, pSet->diskId.level);
n += tPutI32v(p ? p + n : p, pSet->diskId.id);
n += tPutI32v(p ? p + n : p, pSet->fid);
n += tPutHeadFile(p ? p + n : p, &pSet->fHead);
n += tPutDataFile(p ? p + n : p, &pSet->fData);
n += tPutLastFile(p ? p + n : p, &pSet->fLast);
n += tPutSmaFile(p ? p + n : p, &pSet->fSma);
if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
return -1;
}
ASSERT(pDFile->info.size == toffset);
if (offset) {
*offset = toffset;
}
if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
return -1;
}
pDFile->info.size += nbyte;
return (int)nbyte;
return n;
}
int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0;
int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
n += tGetI32v(p + n, &pSet->diskId.level);
n += tGetI32v(p + n, &pSet->diskId.id);
n += tGetI32v(p + n, &pSet->fid);
n += tGetHeadFile(p + n, &pSet->fHead);
n += tGetDataFile(p + n, &pSet->fData);
n += tGetLastFile(p + n, &pSet->fLast);
n += tGetSmaFile(p + n, &pSet->fSma);
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
if (nread < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return nread;
return n;
}
int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) {
if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// SDelFile ===============================================
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) {
STfs *pTfs = pTsdb->pVnode->pTfs;
pDest->info = pSrc->info;
return 0;
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, pTsdb->path,
TD_DIRSEP, TD_VID(pTsdb->pVnode), pFile->commitID, ".del");
}
void tsdbCloseDFileSet(SDFileSet *pSet) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
}
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) {
int32_t n = 0;
n += tPutI64v(p ? p + n : p, pDelFile->commitID);
n += tPutI64v(p ? p + n : p, pDelFile->size);
n += tPutI64v(p ? p + n : p, pDelFile->offset);
return n;
}
int tsdbOpenDFileSet(SDFileSet *pSet, int flags) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
tsdbCloseDFileSet(pSet);
return -1;
}
}
return 0;
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile) {
int32_t n = 0;
n += tGetI64v(p + n, &pDelFile->commitID);
n += tGetI64v(p + n, &pDelFile->size);
n += tGetI64v(p + n, &pDelFile->offset);
return n;
}
void tsdbRemoveDFileSet(SDFileSet *pSet) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
(void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
}
}
int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
tsdbRemoveDFileSet(pDest);
return -1;
}
}
return 0;
}
void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) {
*minKey = fid * days * tsTickPerMin[precision];
*maxKey = *minKey + days * tsTickPerMin[precision] - 1;
}

141
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -25,8 +25,6 @@
#define SL_MOVE_BACKWARD 0x1
#define SL_MOVE_FROM_POS 0x2
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version,
@ -44,10 +42,12 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
taosInitRWLatch(&pMemTable->latch);
pMemTable->pTsdb = pTsdb;
pMemTable->nRef = 1;
pMemTable->minKey = (TSDBKEY){.ts = TSKEY_MAX, .version = INT64_MAX};
pMemTable->maxKey = (TSDBKEY){.ts = TSKEY_MIN, .version = -1};
pMemTable->minKey = TSKEY_MAX;
pMemTable->maxKey = TSKEY_MIN;
pMemTable->minVersion = VERSION_MAX;
pMemTable->maxVersion = VERSION_MIN;
pMemTable->nRow = 0;
pMemTable->nDelOp = 0;
pMemTable->nDel = 0;
pMemTable->aTbData = taosArrayInit(128, sizeof(STbData *));
if (pMemTable->aTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -146,6 +146,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
SMemTable *pMemTable = pTsdb->mem;
STbData *pTbData = NULL;
SVBufPool *pPool = pTsdb->pVnode->inUse;
TSDBKEY lastKey = {.version = version, .ts = eKey};
// check if table exists (todo)
@ -155,26 +156,32 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
}
// do delete
SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
if (pDelOp == NULL) {
SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
if (pDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pDelOp->version = version;
pDelOp->sKey = sKey;
pDelOp->eKey = eKey;
pDelOp->pNext = NULL;
pDelData->version = version;
pDelData->sKey = sKey;
pDelData->eKey = eKey;
pDelData->pNext = NULL;
if (pTbData->pHead == NULL) {
ASSERT(pTbData->pTail == NULL);
pTbData->pHead = pTbData->pTail = pDelOp;
pTbData->pHead = pTbData->pTail = pDelData;
} else {
pTbData->pTail->pNext = pDelOp;
pTbData->pTail = pDelOp;
pTbData->pTail->pNext = pDelData;
pTbData->pTail = pDelData;
}
// update the state of pMemTable and other (todo)
pMemTable->nDelOp++;
pMemTable->minVersion = TMIN(pMemTable->minVersion, version);
pMemTable->maxVersion = TMAX(pMemTable->maxVersion, version);
pMemTable->nDel++;
if (tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) {
tsdbCacheDelete(pTsdb->lruCache, pTbData->uid, eKey);
}
tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
" since %s",
@ -213,9 +220,15 @@ void *tsdbTbDataIterDestroy(STbDataIter *pIter) {
void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter) {
SMemSkipListNode *pos[SL_MAX_LEVEL];
SMemSkipListNode *pHead;
SMemSkipListNode *pTail;
pHead = pTbData->sl.pHead;
pTail = pTbData->sl.pTail;
pIter->pTbData = pTbData;
pIter->backward = backward;
pIter->pRow = NULL;
pIter->row.type = 0;
if (pFrom == NULL) {
// create from head or tail
if (backward) {
@ -239,6 +252,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
SMemSkipListNode *pHead = pIter->pTbData->sl.pHead;
SMemSkipListNode *pTail = pIter->pTbData->sl.pTail;
pIter->pRow = NULL;
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
@ -266,31 +280,29 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
return true;
}
bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) {
SMemSkipListNode *pHead = pIter->pTbData->sl.pHead;
SMemSkipListNode *pTail = pIter->pTbData->sl.pTail;
TSDBROW row = {0};
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
// we add here for commit usage
if (pIter == NULL) return NULL;
if (pRow == NULL) {
pRow = &row;
if (pIter->pRow) {
goto _exit;
}
if (pIter->backward) {
ASSERT(pIter->pNode != pTail);
if (pIter->pNode == pHead) {
return false;
if (pIter->pNode == pIter->pTbData->sl.pHead) {
goto _exit;
}
} else {
ASSERT(pIter->pNode != pHead);
if (pIter->pNode == pTail) {
return false;
if (pIter->pNode == pIter->pTbData->sl.pTail) {
goto _exit;
}
}
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), pRow);
return true;
tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
pIter->pRow = &pIter->row;
_exit:
return pIter->pRow;
}
static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
@ -317,8 +329,11 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
}
pTbData->suid = suid;
pTbData->uid = uid;
pTbData->minKey = (TSDBKEY){.ts = TSKEY_MAX, .version = INT64_MAX};
pTbData->maxKey = (TSDBKEY){.ts = TSKEY_MIN, .version = -1};
pTbData->minKey = TSKEY_MAX;
pTbData->maxKey = TSKEY_MIN;
pTbData->minVersion = VERSION_MAX;
pTbData->maxVersion = VERSION_MIN;
pTbData->maxSkmVer = -1;
pTbData->pHead = NULL;
pTbData->pTail = NULL;
pTbData->sl.seed = taosRand();
@ -493,8 +508,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
SSubmitBlkIter blkIter = {0};
TSDBKEY key = {.version = version};
SMemSkipListNode *pos[SL_MAX_LEVEL];
TSDBROW row = {.version = version, .pTSRow = NULL};
TSDBROW row = tsdbRowFromTSRow(version, NULL);
int32_t nRow = 0;
STSRow *pLastRow = NULL;
tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter);
@ -508,13 +524,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
goto _err;
}
if (tsdbKeyCmprFn(&key, &pTbData->minKey) < 0) {
pTbData->minKey = key;
}
pTbData->minKey = TMIN(pTbData->minKey, key.ts);
if (tsdbKeyCmprFn(&key, &pMemTable->minKey) < 0) {
pMemTable->minKey = key;
}
pLastRow = row.pTSRow;
// forward put rest data
row.pTSRow = tGetSubmitBlkNext(&blkIter);
@ -531,18 +543,35 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
goto _err;
}
pLastRow = row.pTSRow;
row.pTSRow = tGetSubmitBlkNext(&blkIter);
} while (row.pTSRow);
}
if (tsdbKeyCmprFn(&key, &pTbData->maxKey) > 0) {
pTbData->maxKey = key;
if (key.ts >= pTbData->maxKey) {
if (key.ts > pTbData->maxKey) {
pTbData->maxKey = key.ts;
}
if (pLastRow != NULL) {
tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true);
}
}
if (tsdbKeyCmprFn(&key, &pMemTable->maxKey) > 0) {
pMemTable->maxKey = key;
}
pMemTable->nRef++;
tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow);
pTbData->minVersion = TMIN(pTbData->minVersion, version);
pTbData->maxVersion = TMAX(pTbData->maxVersion, version);
pTbData->maxSkmVer = TMAX(pTbData->maxSkmVer, pMsgIter->sversion);
// SMemTable
pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey);
pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey);
pMemTable->minVersion = TMIN(pMemTable->minVersion, pTbData->minVersion);
pMemTable->maxVersion = TMAX(pMemTable->maxVersion, pTbData->maxVersion);
pMemTable->nRow += nRow;
pRsp->numOfRows = nRow;
pRsp->affectedRows = nRow;
@ -552,22 +581,4 @@ _err:
return code;
}
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tPutI64(p, pRow->version);
if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len);
n += pRow->pTSRow->len;
return n;
}
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
int32_t n = 0;
n += tGetI64(p, &pRow->version);
pRow->pTSRow = (STSRow *)(p + n);
n += pRow->pTSRow->len;
return n;
}
int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; }

32
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -17,7 +17,6 @@
static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg);
// implementation
static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) {
@ -42,7 +41,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
int slen = 0;
*ppTsdb = NULL;
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3;
slen = strlen(pVnode->path) + strlen(dir) + 2;
// create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
@ -51,10 +50,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
return -1;
}
ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN);
memcpy(pTsdb->dir, dir, strlen(dir));
pTsdb->path = (char *)&pTsdb[1];
sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir);
sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir);
taosRealPath(pTsdb->path, NULL, slen);
pTsdb->pVnode = pVnode;
pTsdb->repoLocked = false;
@ -64,13 +61,17 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
} else {
memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));
}
pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb));
// pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb));
// create dir (TODO: use tfsMkdir)
taosMkDir(pTsdb->path);
// create dir
tfsMkdir(pVnode->pTfs, pTsdb->path);
// open tsdb
if (tsdbOpenFS(pTsdb) < 0) {
if (tsdbFSOpen(pTsdb, &pTsdb->fs) < 0) {
goto _err;
}
if (tsdbOpenCache(pTsdb) < 0) {
goto _err;
}
@ -87,10 +88,9 @@ _err:
int tsdbClose(STsdb **pTsdb) {
if (*pTsdb) {
// TODO: destroy mem/imem
taosThreadMutexDestroy(&(*pTsdb)->mutex);
tsdbCloseFS(*pTsdb);
tsdbFreeFS((*pTsdb)->fs);
tsdbFSClose((*pTsdb)->fs);
tsdbCloseCache((*pTsdb)->lruCache);
taosMemoryFreeClear(*pTsdb);
}
return 0;
@ -99,7 +99,7 @@ int tsdbClose(STsdb **pTsdb) {
int tsdbLockRepo(STsdb *pTsdb) {
int code = taosThreadMutexLock(&pTsdb->mutex);
if (code != 0) {
tsdbError("vgId:%d, failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
tsdbError("vgId:%d, failed to lock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
@ -108,13 +108,13 @@ int tsdbLockRepo(STsdb *pTsdb) {
}
int tsdbUnlockRepo(STsdb *pTsdb) {
ASSERT(IS_REPO_LOCKED(pTsdb));
// ASSERT(IS_REPO_LOCKED(pTsdb));
pTsdb->repoLocked = false;
int code = taosThreadMutexUnlock(&pTsdb->mutex);
if (code != 0) {
tsdbError("vgId:%d, failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
tsdbError("vgId:%d, failed to unlock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
}

6713
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

File diff suppressed because it is too large Load Diff

961
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -14,964 +14,3 @@
*/
#include "tsdb.h"
#define TSDB_KEY_COL_OFFSET 0
static void tsdbResetReadTable(SReadH *pReadh);
static void tsdbResetReadFile(SReadH *pReadh);
static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode);
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize);
static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
int numOfColIds, int8_t bitmapMode);
static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
ASSERT(pReadh != NULL && pRepo != NULL);
STsdbCfg *pCfg = REPO_CFG(pRepo);
memset((void *)pReadh, 0, sizeof(*pReadh));
pReadh->pRepo = pRepo;
TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
if (pReadh->aBlkIdx == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRows);
if (pReadh->pDCols[0] == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyReadH(pReadh);
return -1;
}
pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRows);
if (pReadh->pDCols[1] == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyReadH(pReadh);
return -1;
}
return 0;
}
void tsdbDestroyReadH(SReadH *pReadh) {
if (pReadh == NULL) return;
pReadh->pExBuf = taosTZfree(pReadh->pExBuf);
pReadh->pCBuf = taosTZfree(pReadh->pCBuf);
pReadh->pBuf = taosTZfree(pReadh->pBuf);
pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]);
pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]);
pReadh->pAggrBlkData = taosTZfree(pReadh->pAggrBlkData);
pReadh->pBlkData = taosTZfree(pReadh->pBlkData);
pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo);
pReadh->cidx = 0;
pReadh->pBlkIdx = NULL;
pReadh->pTable = NULL;
pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx);
tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
pReadh->pRepo = NULL;
}
int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
ASSERT(pSet != NULL);
tsdbResetReadFile(pReadh);
pReadh->rSet = *pSet;
TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
// if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) {
if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) {
tsdbError("vgId:%d, failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet),
tstrerror(terrno));
return -1;
}
return 0;
}
void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
int tsdbLoadBlockIdx(SReadH *pReadh) {
SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
SBlockIdx blkIdx;
ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
// No data at all, just return
if (pHeadf->info.offset <= 0) return 0;
if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
pHeadf->info.len);
return -1;
}
if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1;
int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len);
if (nread < 0) {
tsdbError("vgId:%d, failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
pHeadf->info.len);
return -1;
}
if (nread < pHeadf->info.len) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len);
return -1;
}
void *ptr = TSDB_READ_BUF(pReadh);
int tsize = 0;
while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) {
ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx);
ASSERT(ptr != NULL);
if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
tsize++;
// ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
// ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
}
return 0;
}
static int32_t tsdbBlockIdxCmprFn(const void *p1, const void *p2) {
SBlockIdx *pBlockIdx1 = (SBlockIdx *)p1;
SBlockIdx *pBlockIdx2 = (SBlockIdx *)p2;
if (pBlockIdx1->suid < pBlockIdx2->suid) {
return -1;
} else if (pBlockIdx1->suid > pBlockIdx2->suid) {
return 1;
}
if (pBlockIdx1->uid < pBlockIdx2->uid) {
return -1;
} else if (pBlockIdx1->uid > pBlockIdx2->uid) {
return 1;
}
return 0;
}
int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
STSchema *pSchema = tsdbGetTableSchemaImpl(TSDB_READ_REPO(pReadh), pTable, false, false, -1);
pReadh->pTable = pTable;
if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
uint8_t *p = taosArraySearch(pReadh->aBlkIdx, &(SBlockIdx){.suid = pTable->suid, .uid = pTable->uid},
tsdbBlockIdxCmprFn, TD_EQ);
if (p == NULL) {
pReadh->pBlkIdx = NULL;
} else {
pReadh->pBlkIdx = (SBlockIdx *)p;
}
return 0;
}
int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
ASSERT(pReadh->pBlkIdx != NULL);
SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
return -1;
}
if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1;
int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
if (nread < 0) {
tsdbError("vgId:%d, failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
return -1;
}
if (nread < pBlkIdx->len) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len);
return -1;
}
// ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
if (pTarget) {
memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
}
return 0;
}
static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) {
SDataCol *pCols = pDest->cols;
memcpy(pDest, pSrc, sizeof(SDataCols));
pSrc->cols = pCols;
}
static void printTsdbLoadBlkData(SReadH *readh, SDataCols *pDCols, SBlock *pBlock, const char *tag, int32_t ln) {
printf("%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId());
if (pBlock) {
SDFile *pHeadf = TSDB_READ_HEAD_FILE(readh);
printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
pHeadf->f.aname);
SDFile *pDFile = pBlock->last ? TSDB_READ_LAST_FILE(readh) : TSDB_READ_DATA_FILE(readh);
printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
pDFile->f.aname);
}
SDataCol *pDCol = pDCols->cols + 0;
if (TSKEY_MIN == *(int64_t *)pDCol->pData) {
ASSERT(0);
}
int rows = pDCols->numOfRows;
for (int r = 0; r < rows; ++r) {
if (pBlock) {
printf("%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
rows, r);
} else {
printf("%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r);
}
int nDataCols = pDCols->numOfCols;
int j = 0;
SCellVal sVal = {0};
while (j < nDataCols) {
SDataCol *pDataCol = pDCols->cols + j;
tdGetColDataOfRow(&sVal, pDataCol, r, pDCols->bitmapMode);
tdSCellValPrint(&sVal, pDataCol->type);
++j;
}
printf("\n");
}
fflush(stdout);
}
int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
ASSERT(pBlock->numOfSubBlocks > 0);
STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo);
int8_t update = pCfg->update;
SBlock *iBlock = pBlock;
if (pBlock->numOfSubBlocks > 1) {
if (pBlkInfo) {
iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset);
} else {
iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
}
}
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, __func__, __LINE__);
#endif
for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
iBlock++;
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkData(pReadh, pReadh->pDCols[1], iBlock, __func__, __LINE__);
#endif
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === MERGE === ", __LINE__);
#endif
}
// if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
if (pBlock->numOfSubBlocks == 1) {
tdResetDataCols(pReadh->pDCols[1]);
pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
return -1;
}
tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === UPDATE FILTER === ", __LINE__);
#endif
}
ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts);
ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts);
return 0;
}
static void printTsdbLoadBlkDataCols(SReadH *readh, SDataCols *pDCols, SBlock *pBlock, const int16_t *colIds,
int numOfColsIds, const char *tag, int32_t ln) {
printf("%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId());
if (pBlock) {
SDFile *pHeadf = TSDB_READ_HEAD_FILE(readh);
printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
pHeadf->f.aname);
SDFile *pDFile = pBlock->last ? TSDB_READ_LAST_FILE(readh) : TSDB_READ_DATA_FILE(readh);
printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
pDFile->f.aname);
}
int rows = pDCols->numOfRows;
for (int r = 0; r < rows; ++r) {
if (pBlock) {
printf("%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len,
rows, r);
} else {
printf("%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r);
}
int nDataCols = pDCols->numOfCols;
int j = 0, k = 0;
SCellVal sVal = {0};
while (j < nDataCols) {
if (k >= numOfColsIds) break;
SDataCol *pDataCol = pDCols->cols + j;
int16_t colId1 = pDataCol->colId;
int16_t colId2 = *(colIds + k);
if (colId1 < colId2) {
++j;
} else if (colId1 > colId2) {
++k; // colId2 not exists in SDataCols
printf("NotExists ");
} else {
tdGetColDataOfRow(&sVal, pDataCol, r, pDCols->bitmapMode);
tdSCellValPrint(&sVal, pDataCol->type);
++j;
++k;
}
}
printf("\n");
}
fflush(stdout);
}
// TODO: filter by Multi-Version
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds,
bool mergeBitmap) {
ASSERT(pBlock->numOfSubBlocks > 0);
int8_t update = pReadh->pRepo->pVnode->config.tsdbCfg.update;
SBlock *iBlock = pBlock;
if (pBlock->numOfSubBlocks > 1) {
if (pBlkInfo) {
iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset);
} else {
iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
}
}
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0)
return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], iBlock, colIds, numOfColsIds, __func__, __LINE__);
#endif
for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
iBlock++;
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0)
return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[1], iBlock, colIds, numOfColsIds, __func__, __LINE__);
#endif
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
return -1;
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, __func__, __LINE__);
#endif
}
// if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
if (pBlock->numOfSubBlocks == 1) {
tdResetDataCols(pReadh->pDCols[1]);
pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
return -1;
}
tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds,
" === update filter === ", __LINE__);
#endif
}
if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) {
for (int i = 0; i < numOfColsIds; ++i) {
SDataCol *pDataCol = pReadh->pDCols[0]->cols + i;
if (pDataCol->len > 0 && pDataCol->bitmap) {
tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap);
tdDataColsSetBitmapI(pReadh->pDCols[0]);
}
}
#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS
printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, " === merge bitmap === ", __LINE__);
#endif
}
ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts);
ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts);
return 0;
}
int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
ASSERT(pBlock->numOfSubBlocks <= 1);
if (!pBlock->aggrStat) {
tsdbDebug("vgId:%d, no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo),
TSDB_READ_TABLE_UID(pReadh));
return TSDB_STATIS_NONE;
}
SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh);
if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64
" since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, tstrerror(terrno));
return -1;
}
size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfBSma, (uint32_t)pBlock->blkVer);
if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1;
int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);
if (nreadAggr < 0) {
tsdbError("vgId:%d, failed to load block statis part for uid %" PRIu64
" while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr);
return -1;
}
if (nreadAggr < sizeAggr) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64
" expected bytes:%" PRIzu " read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block statis part for uid %" PRIu64
"in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
(uint64_t)pBlock->aggrOffset, sizeAggr);
return -1;
}
return 0;
}
static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
ASSERT(pBlock->numOfSubBlocks <= 1);
SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load block head part while seek file %s to offset %" PRId64 " since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
return -1;
}
size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1;
int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
if (nread < 0) {
tsdbError("vgId:%d, failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
return -1;
}
if (nread < size) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
" read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
return -1;
}
if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
return -1;
}
return 0;
}
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
int tlen = 0;
tlen += taosEncodeFixedU64(buf, pIdx->suid);
tlen += taosEncodeFixedU64(buf, pIdx->uid);
tlen += taosEncodeVariantU32(buf, pIdx->len);
tlen += taosEncodeVariantU32(buf, pIdx->offset);
tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
tlen += taosEncodeFixedU64(buf, pIdx->maxKey.ts);
return tlen;
}
void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
uint8_t hasLast = 0;
uint32_t numOfBlocks = 0;
uint64_t value = 0;
// if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->suid = (int64_t)value;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->uid = (int64_t)value;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
pIdx->hasLast = hasLast;
if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
pIdx->numOfBlocks = numOfBlocks;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->maxKey.ts = (TSKEY)value;
return buf;
}
void tsdbGetBlockStatis(SReadH *pReadh, SColumnDataAgg *pStatis, int numOfCols, SBlock *pBlock) {
#ifdef TD_REFACTOR_3
SBlockData *pBlockData = pReadh->pBlkData;
for (int i = 0, j = 0; i < numOfCols;) {
if (j >= pBlockData->numOfCols) {
pStatis[i].numOfNull = -1;
++i;
continue;
}
if (pStatis[i].colId == pBlockData->cols[j].colId) {
pStatis[i].sum = pBlockData->cols[j].sum;
pStatis[i].max = pBlockData->cols[j].max;
pStatis[i].min = pBlockData->cols[j].min;
pStatis[i].maxIndex = pBlockData->cols[j].maxIndex;
pStatis[i].minIndex = pBlockData->cols[j].minIndex;
pStatis[i].numOfNull = pBlockData->cols[j].numOfNull;
++i;
++j;
} else if (pStatis[i].colId < pBlockData->cols[j].colId) {
pStatis[i].numOfNull = -1;
++i;
} else {
++j;
}
}
#else
if (pBlock->aggrStat) {
SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData;
for (int i = 0, j = 0; i < numOfCols;) {
if (j >= pBlock->numOfBSma) {
pStatis[i].numOfNull = -1;
++i;
continue;
}
SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j;
if (pStatis[i].colId == pAggrBlkCol->colId) {
pStatis[i].sum = pAggrBlkCol->sum;
pStatis[i].max = pAggrBlkCol->max;
pStatis[i].min = pAggrBlkCol->min;
pStatis[i].maxIndex = pAggrBlkCol->maxIndex;
pStatis[i].minIndex = pAggrBlkCol->minIndex;
pStatis[i].numOfNull = pAggrBlkCol->numOfNull;
++i;
++j;
} else if (pStatis[i].colId < pAggrBlkCol->colId) {
pStatis[i].numOfNull = -1;
++i;
} else {
++j;
}
}
}
#endif
}
static void tsdbResetReadTable(SReadH *pReadh) {
tdResetDataCols(pReadh->pDCols[0]);
tdResetDataCols(pReadh->pDCols[1]);
pReadh->cidx = 0;
pReadh->pBlkIdx = NULL;
pReadh->pTable = NULL;
}
static void tsdbResetReadFile(SReadH *pReadh) {
tsdbResetReadTable(pReadh);
taosArrayClear(pReadh->aBlkIdx);
tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
}
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode) {
ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
tdResetDataCols(pDataCols);
pDataCols->bitmapMode = bitmapMode;
if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1;
SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh);
if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load block data part while seek file %s to offset %" PRId64 " since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
return -1;
}
int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len);
if (nread < 0) {
tsdbError("vgId:%d, failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset,
pBlock->len);
return -1;
}
if (nread < pBlock->len) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block data part in file %s is corrupted, offset:%" PRId64
" expected bytes:%d read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread);
return -1;
}
int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
return -1;
}
ASSERT(tsize < pBlock->len);
ASSERT(pBlockData->numOfCols == pBlock->numOfCols);
pDataCols->numOfRows = pBlock->numOfRows;
// Recover the data
int ccol = 0; // loop iter for SBlockCol object
int dcol = 0; // loop iter for SDataCols object
int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
SBlockCol *pBlockCol = NULL;
while (dcol < pDataCols->numOfCols) {
SDataCol *pDataCol = &(pDataCols->cols[dcol]);
if (dcol != 0 && ccol >= pBlockData->numOfCols) {
// Set current column as NULL and forward
dataColReset(pDataCol);
++dcol;
continue;
}
int16_t tcolId = PRIMARYKEY_TIMESTAMP_COL_ID;
uint32_t toffset = TSDB_KEY_COL_OFFSET;
int32_t tlen = pBlock->keyLen;
if (dcol != 0) {
pBlockCol = &(pBlockData->cols[ccol]);
tcolId = pBlockCol->colId;
toffset = pBlockCol->offset;
tlen = pBlockCol->len;
pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0;
} else {
ASSERT(pDataCol->colId == tcolId);
TD_SET_COL_ROWS_NORM(pDataCol);
}
// int32_t tBitmaps = 0;
int32_t tLenBitmap = 0;
if ((dcol != 0) && (pBlockCol->blen > 0)) {
tLenBitmap = nBitmaps;
}
if (tcolId == pDataCol->colId) {
if (pBlock->algorithm == TWO_STAGE_COMP) {
int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
}
if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen,
pBlockCol ? pBlockCol->blen : 0, pBlock->algorithm, pBlock->numOfRows,
tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
(int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
tsdbError("vgId:%d, file %s is broken at column %d block offset %" PRId64 " column offset %u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
return -1;
}
if (dcol != 0) {
++ccol;
}
++dcol;
} else if (tcolId < pDataCol->colId) {
++ccol;
} else {
// Set current column as NULL and forward
dataColReset(pDataCol);
++dcol;
}
}
return 0;
}
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) {
if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
tdAllocMemForCol(pDataCol, maxPoints);
// Decode the data
if (comp) {
// Need to decompress
int tlen =
(*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows,
pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
if (tlen <= 0) {
tsdbError(
"Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
(int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
pDataCol->len = tlen;
if (numOfBitmaps > 0) {
tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps,
pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize);
if (tlen <= 0) {
tsdbError(
"Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d "
"bufferSize:%d",
bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
// pDataCol->blen = tlen;
}
} else {
// No need to decompress, just memcpy it
pDataCol->len = len - bitmapLen - sizeof(TSCKSUM);
memcpy(pDataCol->pData, content, pDataCol->len);
if (numOfBitmaps > 0) {
// pDataCol->blen = bitmapLen;
memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen);
}
}
#if 0
if (lenOfBitmaps > 0) {
pDataCol->len -= lenOfBitmaps;
void *pSrcBitmap = NULL;
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pSrcBitmap = dataColSetOffset(pDataCol, numOfRows);
} else {
pSrcBitmap = POINTER_SHIFT(pDataCol->pData, numOfRows * TYPE_BYTES[pDataCol->type]);
}
void *pDestBitmap = POINTER_SHIFT(pDataCol->pData, pDataCol->bytes * maxPoints);
// restore the bitmap parts
memcpy(pDestBitmap, pSrcBitmap, lenOfBitmaps);
} else if (IS_VAR_DATA_TYPE(pDataCol->type)) {
dataColSetOffset(pDataCol, numOfRows);
}
#endif
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
dataColSetOffset(pDataCol, numOfRows);
}
return 0;
}
static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
int numOfColIds, int8_t bitmapMode) {
ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);
SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
SBlockCol blockCol = {0};
tdResetDataCols(pDataCols);
pDataCols->bitmapMode = bitmapMode;
// If only load timestamp column, no need to load SBlockData part
if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1;
pDataCols->numOfRows = pBlock->numOfRows;
int dcol = 0;
int ccol = 0;
for (int i = 0; i < numOfColIds; i++) {
int16_t colId = colIds[i];
SDataCol *pDataCol = NULL;
SBlockCol *pBlockCol = NULL;
while (true) {
if (dcol >= pDataCols->numOfCols) {
pDataCol = NULL;
break;
}
pDataCol = &pDataCols->cols[dcol];
if (pDataCol->colId > colId) {
pDataCol = NULL;
break;
} else {
dcol++;
if (pDataCol->colId == colId) break;
}
}
if (pDataCol == NULL) continue;
ASSERT(pDataCol->colId == colId);
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // load the key row
blockCol.colId = colId;
blockCol.blen = 0; // default is NORM for the primary key column
blockCol.len = pBlock->keyLen;
blockCol.type = pDataCol->type;
blockCol.offset = TSDB_KEY_COL_OFFSET;
pBlockCol = &blockCol;
} else { // load non-key rows
while (true) {
if (ccol >= pBlock->numOfCols) {
pBlockCol = NULL;
break;
}
pBlockCol = &(pReadh->pBlkData->cols[ccol]);
if (pBlockCol->colId > colId) {
pBlockCol = NULL;
break;
} else {
ccol++;
if (pBlockCol->colId == colId) break;
}
}
if (pBlockCol == NULL) {
dataColReset(pDataCol);
continue;
}
ASSERT(pBlockCol->colId == pDataCol->colId);
}
// set the bitmap
pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0;
if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
}
return 0;
}
static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
ASSERT(pDataCol->colId == pBlockCol->colId);
STsdb *pRepo = TSDB_READ_REPO(pReadh);
STsdbCfg *pCfg = REPO_CFG(pRepo);
int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
// int32_t tBitmaps = 0;
int32_t tLenBitmap = 0;
if (pBlockCol->blen) {
tLenBitmap = nBitmaps;
}
int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
int64_t offset =
pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + pBlockCol->offset;
if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
tsdbError("vgId:%d, failed to load block column data while seek file %s to offset %" PRId64 " since %s",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));
return -1;
}
int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len);
if (nread < 0) {
tsdbError("vgId:%d, failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len);
return -1;
}
if (nread < pBlockCol->len) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
tsdbError("vgId:%d, block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu
" read bytes: %" PRId64,
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread);
return -1;
}
if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm,
pBlock->numOfRows, tLenBitmap, pCfg->maxRows, pReadh->pCBuf,
(int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
tsdbError("vgId:%d, file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
pBlockCol->colId, offset);
return -1;
}
return 0;
}

1906
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c Normal file
View File

File diff suppressed because it is too large Load Diff

1270
source/dnode/vnode/src/tsdb/tsdbUtil.c Normal file
View File

File diff suppressed because it is too large Load Diff

6
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -28,7 +28,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
// scan and convert
if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d, failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno));
}
return -1;
}
@ -77,7 +77,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro
if (rowKey < minKey || rowKey > maxKey) {
tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
" maxKey %" PRId64 " row key %" PRId64,
REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey);
TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey);
terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
return -1;
}
@ -92,7 +92,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
SSubmitBlk *pBlock = NULL;
SSubmitBlkIter blkIter = {0};
STSRow *row = NULL;
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb);
STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY maxKey = now + tsTickPerMin[pCfg->precision] * pCfg->days;

34
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -40,6 +40,7 @@ int vnodeBegin(SVnode *pVnode) {
/* pthread_mutex_unlock(); */
pVnode->state.commitID++;
// begin meta
if (metaBegin(pVnode->pMeta) < 0) {
vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno));
@ -47,26 +48,21 @@ int vnodeBegin(SVnode *pVnode) {
}
// begin tsdb
if (pVnode->pSma) {
if (tsdbBegin(VND_RSMA0(pVnode)) < 0) {
vError("vgId:%d, failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbBegin(pVnode->pTsdb) < 0) {
vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbBegin(VND_RSMA1(pVnode)) < 0) {
if (pVnode->pSma) {
if (VND_RSMA1(pVnode) && tsdbBegin(VND_RSMA1(pVnode)) < 0) {
vError("vgId:%d, failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbBegin(VND_RSMA2(pVnode)) < 0) {
if (VND_RSMA2(pVnode) && tsdbBegin(VND_RSMA2(pVnode)) < 0) {
vError("vgId:%d, failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
} else {
if (tsdbBegin(pVnode->pTsdb) < 0) {
vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
}
// begin sma
@ -218,7 +214,8 @@ int vnodeCommit(SVnode *pVnode) {
SVnodeInfo info = {0};
char dir[TSDB_FILENAME_LEN];
vInfo("vgId:%d, start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied);
vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID,
pVnode->state.applied);
pVnode->onCommit = pVnode->inUse;
pVnode->inUse = NULL;
@ -226,6 +223,7 @@ int vnodeCommit(SVnode *pVnode) {
// save info
info.config = pVnode->config;
info.state.committed = pVnode->state.applied;
info.state.commitID = pVnode->state.commitID;
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
if (vnodeSaveInfo(dir, &info) < 0) {
ASSERT(0);
@ -294,7 +292,7 @@ static int vnodeCommitImpl(void *arg) {
// metaCommit(pVnode->pMeta);
tqCommit(pVnode->pTq);
tsdbCommit(pVnode->pTsdb);
// tsdbCommit(pVnode->pTsdb, );
// vnodeBufPoolRecycle(pVnode);
tsem_post(&(pVnode->canCommit));
@ -317,7 +315,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "applied version", pState->applied) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
return 0;
}
@ -327,9 +325,9 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
int32_t code;
tjsonGetNumberValue(pJson, "commit version", pState->committed, code);
if(code < 0) return -1;
tjsonGetNumberValue(pJson, "applied version", pState->applied, code);
if(code < 0) return -1;
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
if (code < 0) return -1;
return 0;
}

2
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -37,6 +37,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
info.config = *pCfg;
info.state.committed = -1;
info.state.applied = -1;
info.state.commitID = 0;
if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno));
@ -79,6 +80,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->config = info.config;
pVnode->state.committed = info.state.committed;
pVnode->state.applied = info.state.committed;
pVnode->state.commitID = info.state.commitID;
pVnode->pTfs = pTfs;
pVnode->msgCb = msgCb;
pVnode->blockCount = 0;

52
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -261,11 +261,49 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
}
}
// wrapper of tsdb read interface
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableList, uint64_t qId,
void *pMemRef) {
#if 0
return tsdbQueryCacheLastT(pVnode->pTsdb, pCond, groupList, qId, pMemRef);
#endif
return 0;
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid);
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
if (id == 0) {
break;
}
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
taosArrayPush(list, &info);
}
metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS;
}
int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid);
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
if (id == 0) {
break;
}
taosArrayPush(list, &id);
}
metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS;
}
void *vnodeGetIdx(SVnode *pVnode) {
if (pVnode == NULL) {
return NULL;
}
return metaGetIdx(pVnode->pMeta);
}
void *vnodeGetIvtIdx(SVnode *pVnode) {
if (pVnode == NULL) {
return NULL;
}
return metaGetIvtIdx(pVnode->pMeta);
}

86
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -25,8 +25,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t code = 0;
@ -93,11 +93,44 @@ int32_t vnodePreProcessReq(SVnode *pVnode, SRpcMsg *pMsg) {
}
} break;
case TDMT_VND_DELETE: {
int32_t size;
int32_t ret;
uint8_t *pCont;
SEncoder *pCoder = &(SEncoder){0};
SDeleteRes res = {0};
SReadHandle handle = {
.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res);
if (code) goto _err;
// malloc and encode
tEncodeSize(tEncodeDeleteRes, &res, size, ret);
pCont = rpcMallocCont(size + sizeof(SMsgHead));
((SMsgHead *)pCont)->contLen = htonl(size + sizeof(SMsgHead));
((SMsgHead *)pCont)->vgId = htonl(TD_VID(pVnode));
tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size);
tEncodeDeleteRes(pCoder, &res);
tEncoderClear(pCoder);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = pCont;
pMsg->contLen = size + sizeof(SMsgHead);
taosArrayDestroy(res.uidList);
} break;
default:
break;
}
return code;
_err:
vError("vgId%d, preprocess request failed since %s", TD_VID(pVnode), tstrerror(code));
return code;
}
int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) {
@ -146,7 +179,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
break;
case TDMT_VND_DELETE:
if (vnodeProcessWriteMsg(pVnode, version, pMsg, pRsp) < 0) goto _err;
if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
/* TQ */
case TDMT_VND_MQ_VG_CHANGE:
@ -185,6 +218,8 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
break;
case TDMT_VND_ALTER_CONFIG:
break;
case TDMT_VND_COMMIT:
goto _do_commit;
default:
ASSERT(0);
break;
@ -199,6 +234,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
// commit if need
if (vnodeShouldCommit(pVnode)) {
_do_commit:
vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
// commit current change
vnodeCommit(pVnode);
@ -280,22 +316,6 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
}
}
int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp) {
vTrace("message in write queue is processing");
char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
SDeleteRes res = {0};
SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
switch (pMsg->msgType) {
case TDMT_VND_DELETE:
return qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, pRsp, &res);
default:
vError("unknown msg type:%d in write queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
}
}
// TODO: remove the function
void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
// TODO
@ -316,7 +336,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
int32_t t = ntohl(*(int32_t *)pReq);
vDebug("vgId:%d, recv ttl msg, time:%d", pVnode->config.vgId, t);
vError("rec ttl time:%d", t);
int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
if (ret != 0) {
goto end;
@ -856,3 +876,31 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo
// 3. reload sync
return 0;
}
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SDecoder *pCoder = &(SDecoder){0};
SDeleteRes *pRes = &(SDeleteRes){0};
pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t));
if (pRes->uidList == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
tDecoderInit(pCoder, pReq, len);
tDecodeDeleteRes(pCoder, pRes);
for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) {
code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
pRes->skey, pRes->ekey);
if (code) goto _err;
}
tDecoderClear(pCoder);
taosArrayDestroy(pRes->uidList);
return code;
_err:
return code;
}

2
source/libs/executor/inc/executil.h
View File

@ -106,7 +106,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
int32_t getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
SArray* createSortInfo(SNodeList* pNodeList);
SArray* extractPartitionColInfo(SNodeList* pNodeList);
SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,

19
source/libs/executor/inc/executorimpl.h
View File

@ -149,8 +149,8 @@ typedef struct SExecTaskInfo {
struct {
char *tablename;
char *dbname;
int32_t sversion;
int32_t tversion;
SSchemaWrapper*sw;
} schemaVer;
STableListInfo tableqinfoList; // this is a table list
@ -261,7 +261,7 @@ enum {
};
typedef struct STableScanInfo {
void* dataReader;
STsdbReader* dataReader;
SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder;
@ -306,6 +306,15 @@ typedef struct STagScanInfo {
STableListInfo *pTableList;
} STagScanInfo;
typedef struct SLastrowScanInfo {
SSDataBlock *pRes;
SArray *pTableList;
SReadHandle readHandle;
void *pLastrowReader;
SArray *pColMatchInfo;
int32_t *pSlotIds;
} SLastrowScanInfo;
typedef enum EStreamScanMode {
STREAM_SCAN_FROM_READERHANDLE = 1,
STREAM_SCAN_FROM_RES,
@ -442,6 +451,8 @@ typedef struct SIntervalAggOperatorInfo {
SArray* pDelWins; // SWinRes
int32_t delIndex;
SSDataBlock* pDelRes;
SNode *pCondition;
} SIntervalAggOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo {
@ -757,7 +768,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
@ -770,6 +781,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SArray* pTableList, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,

59
source/libs/executor/src/executil.c
View File

@ -63,7 +63,8 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
rowSize += pCtx[i].resDataInfo.interBufSize;
}
rowSize += (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData)
rowSize +=
(numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData)
return rowSize;
}
@ -114,7 +115,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
p->pos = *(SResultRowPosition*)pData;
memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
#ifdef BUF_PAGE_DEBUG
qDebug("page_groupRes, groupId:%"PRIu64",pageId:%d,offset:%d\n", p->groupId, p->pos.pageId, p->pos.offset);
qDebug("page_groupRes, groupId:%" PRIu64 ",pageId:%d,offset:%d\n", p->groupId, p->pos.pageId, p->pos.offset);
#endif
taosArrayPush(pGroupResInfo->pRows, &p);
}
@ -288,10 +289,13 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
return result;
}
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
int32_t code = TSDB_CODE_SUCCESS;
pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
if (pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (pListInfo->pTableList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
uint64_t tableUid = pScanNode->uid;
@ -301,14 +305,11 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
if (pScanNode->tableType == TSDB_SUPER_TABLE) {
if (pTagIndexCond) {
SIndexMetaArg metaArg = {
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
SArray* res = taosArrayInit(8, sizeof(uint64_t));
// code = doFilterTag(pTagIndexCond, &metaArg, res);
code = TSDB_CODE_INDEX_REBUILDING;
if (code == TSDB_CODE_INDEX_REBUILDING) {
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList);
} else if (code != TSDB_CODE_SUCCESS) {
qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid);
taosArrayDestroy(res);
@ -324,7 +325,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
taosArrayDestroy(res);
} else {
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList);
}
if (code != TSDB_CODE_SUCCESS) {
@ -333,13 +334,13 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
return code;
}
if(pTagCond){
if (pTagCond) {
int32_t i = 0;
while (i < taosArrayGetSize(pListInfo->pTableList)) {
STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
bool isOk = isTableOk(info, pTagCond, metaHandle);
if(terrno) return terrno;
if(!isOk){
bool isOk = isTableOk(info, pTagCond, metaHandle);
if (terrno) return terrno;
if (!isOk) {
taosArrayRemove(pListInfo->pTableList, i);
continue;
}
@ -350,8 +351,11 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
if (pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (pListInfo->pGroupList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// put into list as default group, remove it if grouping sorting is required later
taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
@ -742,8 +746,6 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
}
int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
pCond->loadExternalRows = false;
pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
@ -759,25 +761,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
pCond->twindows[0] = pTableScanNode->scanRange;
pCond->suid = pTableScanNode->scan.suid;
#if 1
// todo work around a problem, remove it later
for (int32_t i = 0; i < pCond->numOfTWindows; ++i) {
if ((pCond->order == TSDB_ORDER_ASC && pCond->twindows[i].skey > pCond->twindows[i].ekey) ||
(pCond->order == TSDB_ORDER_DESC && pCond->twindows[i].skey < pCond->twindows[i].ekey)) {
TSWAP(pCond->twindows[i].skey, pCond->twindows[i].ekey);
}
}
#endif
for (int32_t i = 0; i < pCond->numOfTWindows; ++i) {
if ((pCond->order == TSDB_ORDER_ASC && pCond->twindows[i].skey > pCond->twindows[i].ekey) ||
(pCond->order == TSDB_ORDER_DESC && pCond->twindows[i].skey < pCond->twindows[i].ekey)) {
TSWAP(pCond->twindows[i].skey, pCond->twindows[i].ekey);
}
}
taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow);
pCond->type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
pCond->type = BLOCK_LOAD_OFFSET_ORDER;
// pCond->type = pTableScanNode->scanFlag;
int32_t j = 0;
@ -798,10 +782,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
return TSDB_CODE_SUCCESS;
}
void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) {
taosMemoryFree(pCond->twindows);
taosMemoryFree(pCond->colList);
}
void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFree(pCond->colList); }
int32_t convertFillType(int32_t mode) {
int32_t type = TSDB_FILL_NONE;

6
source/libs/executor/src/executor.c
View File

@ -189,7 +189,11 @@ int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tab
ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
*sversion = pTaskInfo->schemaVer.sversion;
if (pTaskInfo->schemaVer.sw == NULL) {
return TSDB_CODE_SUCCESS;
}
*sversion = pTaskInfo->schemaVer.sw->version;
*tversion = pTaskInfo->schemaVer.tversion;
if (pTaskInfo->schemaVer.dbname) {
strcpy(dbName, pTaskInfo->schemaVer.dbname);

1
source/libs/executor/src/executorMain.c
View File

@ -143,6 +143,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));
int64_t st = taosGetTimestampUs();
*pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
uint64_t el = (taosGetTimestampUs() - st);

79
source/libs/executor/src/executorimpl.c
View File

@ -2823,7 +2823,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN ||
type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN) {
type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN) {
*order = TSDB_ORDER_ASC;
*scanFlag = MAIN_SCAN;
return TSDB_CODE_SUCCESS;
@ -2885,7 +2885,7 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
tsdbSetTableId(pInfo->dataReader, uid);
int64_t oldSkey = pInfo->cond.twindows[0].skey;
pInfo->cond.twindows[0].skey = ts + 1;
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
pInfo->cond.twindows[0].skey = oldSkey;
pInfo->scanTimes = 0;
pInfo->curTWinIdx = 0;
@ -3889,6 +3889,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->exprSupp.numOfExprs = numOfExpr;
pOperator->pTaskInfo = pTaskInfo;
@ -3991,29 +3992,32 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
return pTaskInfo;
}
static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, const char* idstr);
static SArray* extractColumnInfo(SNodeList* pNodeList);
int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
SMetaReader mr = {0};
metaReaderInit(&mr, pHandle->meta, 0);
int32_t code = metaGetTableEntryByUid(&mr, uid);
if (code) {
if (code != TSDB_CODE_SUCCESS) {
metaReaderClear(&mr);
return code;
return terrno;
}
pTaskInfo->schemaVer.tablename = strdup(mr.me.name);
if (mr.me.type == TSDB_SUPER_TABLE) {
pTaskInfo->schemaVer.sversion = mr.me.stbEntry.schemaRow.version;
pTaskInfo->schemaVer.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
pTaskInfo->schemaVer.tversion = mr.me.stbEntry.schemaTag.version;
} else if (mr.me.type == TSDB_CHILD_TABLE) {
tb_uid_t suid = mr.me.ctbEntry.suid;
metaGetTableEntryByUid(&mr, suid);
pTaskInfo->schemaVer.sversion = mr.me.stbEntry.schemaRow.version;
pTaskInfo->schemaVer.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
pTaskInfo->schemaVer.tversion = mr.me.stbEntry.schemaTag.version;
} else {
pTaskInfo->schemaVer.sversion = mr.me.ntbEntry.schemaRow.version;
pTaskInfo->schemaVer.sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
}
metaReaderClear(&mr);
@ -4193,13 +4197,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pTaskInfo->code = code;
return NULL;
}
code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
if (code) {
pTaskInfo->code = terrno;
return NULL;
}
SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
@ -4252,20 +4257,19 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
int32_t code = getTableList(pHandle->meta, pScanPhyNode, pTableListInfo);
int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = terrno;
return NULL;
}
return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo));
if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
int32_t code = tsdbGetAllTableList(pHandle->meta, pBlockNode->uid, pTableListInfo->pTableList);
int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pTableListInfo->pTableList);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = terrno;
return NULL;
@ -4294,12 +4298,42 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
cond.twindows = taosMemoryCalloc(1, sizeof(STimeWindow));
cond.twindows[0] = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
cond.suid = pBlockNode->suid;
cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
cond.type = BLOCK_LOAD_OFFSET_ORDER;
}
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
STsdbReader* pReader = NULL;
tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, "");
cleanupQueryTableDataCond(&cond);
return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) {
SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode;
// int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
// if (code) {
// pTaskInfo->code = code;
// return NULL;
// }
int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
return NULL;
}
pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo));
if (pScanNode->tableType == TSDB_SUPER_TABLE) {
code = vnodeGetAllTableList(pHandle->vnode, pScanNode->uid, pTableListInfo->pTableList);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = terrno;
return NULL;
}
} else { // Create one table group.
STableKeyInfo info = {.lastKey = 0, .uid = pScanNode->uid, .groupId = 0};
taosArrayPush(pTableListInfo->pTableList, &info);
}
return createLastrowScanOperator(pScanNode, pHandle, pTableListInfo->pTableList, pTaskInfo);
} else {
ASSERT(0);
}
@ -4504,16 +4538,15 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
return pList;
}
tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo);
STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) {
int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
code = 0;
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
qDebug("no table qualified for query, %s", idstr);
goto _error;
}
@ -4523,7 +4556,12 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
goto _error;
}
tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
STsdbReader* pReader;
code = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
cleanupQueryTableDataCond(&cond);
return pReader;
@ -4586,10 +4624,10 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
ASSERT(0);
}
tsdbCleanupReadHandle(pTableScanInfo->dataReader);
tsdbReaderClose(pTableScanInfo->dataReader);
STableListInfo info = {0};
pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, 0, 0);
pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, NULL);
if (pTableScanInfo->dataReader == NULL) {
ASSERT(0);
qError("failed to create data reader");
@ -4744,6 +4782,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
&(*pTaskInfo)->tableqinfoList, pPlan->user);
if (NULL == (*pTaskInfo)->pRoot) {
code = (*pTaskInfo)->code;
goto _complete;

172
source/libs/executor/src/scanoperator.c
View File

@ -13,15 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <vnode.h>
#include "filter.h"
#include "executorimpl.h"
#include "function.h"
#include "functionMgt.h"
#include "os.h"
#include "querynodes.h"
#include "systable.h"
#include "tglobal.h"
#include "tname.h"
#include "ttime.h"
@ -35,8 +32,6 @@
#include "ttypes.h"
#include "vnode.h"
#include "executorInt.h"
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
@ -445,7 +440,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
}
pTableScanInfo->curTWinIdx += 1;
if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
}
}
@ -460,7 +455,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
// do prepare for the next round table scan operation
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
}
}
@ -469,7 +464,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0);
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
}
@ -487,7 +482,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
}
pTableScanInfo->curTWinIdx += 1;
if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
}
}
@ -503,7 +498,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
}
}
@ -531,7 +526,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
}
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
pInfo->scanTimes = 0;
pInfo->curTWinIdx = 0;
}
@ -543,11 +538,12 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbCleanupReadHandle(pInfo->dataReader);
tsdbReaderT* pReader =
tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
pInfo->dataReader = pReader;
tsdbReaderClose(pInfo->dataReader);
int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
GET_TASKID(pTaskInfo));
}
SSDataBlock* result = doTableScanGroup(pOperator);
@ -562,9 +558,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
}
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbSetTableList(pInfo->dataReader, tableList);
// tsdbSetTableList(pInfo->dataReader, tableList);
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
pInfo->curTWinIdx = 0;
pInfo->scanTimes = 0;
@ -591,7 +587,7 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
blockDataDestroy(pTableScanInfo->pResBlock);
cleanupQueryTableDataCond(&pTableScanInfo->cond);
tsdbCleanupReadHandle(pTableScanInfo->dataReader);
tsdbReaderClose(pTableScanInfo->dataReader);
if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo);
@ -599,15 +595,13 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
}
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) {
SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
// taosSsleep(20);
SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
int32_t numOfCols = 0;
SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
@ -637,8 +631,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColList;
pInfo->curTWinIdx = 0;
pInfo->queryId = queryId;
pInfo->taskId = taskId;
pInfo->currentGroupId = -1;
pOperator->name = "TableScanOperator"; // for debug purpose
@ -889,7 +881,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
pTableScanInfo->cond.twindows[0] = win;
pTableScanInfo->curTWinIdx = 0;
// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
// tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
// if (!pTableScanInfo->dataReader) {
// return false;
// }
@ -1295,13 +1287,13 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
}
if (pHandle) {
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
if (pHandle->tqReader) {
pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
pSTInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0);
tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, 0);
}
if (pSTInfo->interval.interval > 0) {
@ -2158,7 +2150,7 @@ typedef struct STableMergeScanInfo {
int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo);
int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -2177,13 +2169,13 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
}
int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, const char* idstr) {
for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId);
STsdbReader* pReader = NULL;
tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, &pReader, idstr);
taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(subTableList);
@ -2234,7 +2226,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL;
tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) {
@ -2275,7 +2267,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
SArray* pCols = tsdbRetrieveDataBlock(reader, NULL);
if (pCols == NULL) {
return terrno;
@ -2321,7 +2313,7 @@ static SSDataBlock* getTableDataBlock(void* param) {
blockDataCleanup(pBlock);
tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
while (tsdbNextDataBlock(reader)) {
if (isTaskKilled(pOperator->pTaskInfo)) {
longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
@ -2399,7 +2391,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableListInfo* tableListInfo = pInfo->tableListInfo;
createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
pInfo->dataReaders, GET_TASKID(pTaskInfo));
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
@ -2443,11 +2435,12 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
taosArrayClear(pInfo->sortSourceParams);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->dataReaders); ++i) {
tsdbReaderT* reader = taosArrayGetP(pInfo->dataReaders, i);
tsdbCleanupReadHandle(reader);
STsdbReader* reader = taosArrayGetP(pInfo->dataReaders, i);
tsdbReaderClose(reader);
}
taosArrayDestroy(pInfo->dataReaders);
taosArrayDestroy(pInfo->dataReaders);
pInfo->dataReaders = NULL;
return TSDB_CODE_SUCCESS;
}
@ -2529,6 +2522,12 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) {
STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
cleanupQueryTableDataCond(&pTableScanInfo->cond);
for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) {
STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, i);
tsdbReaderClose(reader);
}
taosArrayDestroy(pTableScanInfo->dataReaders);
if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo);
}
@ -2637,3 +2636,100 @@ _error:
taosMemoryFree(pOperator);
return NULL;
}
static SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SLastrowScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t size = taosArrayGetSize(pInfo->pTableList);
if (size == 0) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
// check if it is a group by tbname
if (size == taosArrayGetSize(pInfo->pTableList)) {
blockDataCleanup(pInfo->pRes);
tsdbRetrieveLastRow(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds);
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
} else {
// todo fetch the result for each group
}
return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
}
static void destroyLastrowScanOperator(void* param, int32_t numOfOutput) {
SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param;
blockDataDestroy(pInfo->pRes);
tsdbLastrowReaderClose(pInfo->pLastrowReader);
}
SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SArray* pTableList,
SExecTaskInfo* pTaskInfo) {
SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->pTableList = pTableList;
pInfo->readHandle = *readHandle;
pInfo->pRes = createResDataBlock(pScanNode->node.pOutputDataBlockDesc);
int32_t numOfCols = 0;
pInfo->pColMatchInfo = extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfCols,
COL_MATCH_FROM_COL_ID);
int32_t* pCols = taosMemoryMalloc(numOfCols * sizeof(int32_t));
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pColMatch = taosArrayGet(pInfo->pColMatchInfo, i);
pCols[i] = pColMatch->colId;
}
pInfo->pSlotIds = taosMemoryMalloc(numOfCols * sizeof(pInfo->pSlotIds[0]));
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pColMatch = taosArrayGet(pInfo->pColMatchInfo, i);
for (int32_t j = 0; j < pTaskInfo->schemaVer.sw->nCols; ++j) {
if (pColMatch->colId == pTaskInfo->schemaVer.sw->pSchema[j].colId &&
pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
pInfo->pSlotIds[pColMatch->targetSlotId] = -1;
break;
}
if (pColMatch->colId == pTaskInfo->schemaVer.sw->pSchema[j].colId) {
pInfo->pSlotIds[pColMatch->targetSlotId] = j;
break;
}
}
}
tsdbLastRowReaderOpen(readHandle->vnode, LASTROW_RETRIEVE_TYPE_ALL, pTableList, pCols, numOfCols,
&pInfo->pLastrowReader);
taosMemoryFree(pCols);
pOperator->name = "LastrowScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
initResultSizeInfo(pOperator, 1024);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doScanLastrow, NULL, NULL, destroyLastrowScanOperator, NULL, NULL, NULL);
pOperator->cost.openCost = 0;
return pOperator;
_error:
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
return NULL;
}

16
source/libs/executor/src/timewindowoperator.c
View File

@ -1208,10 +1208,17 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
}
blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBlock->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBlock);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBlock->info.rows > 0) {
break;
}
}
size_t rows = pBlock->info.rows;
@ -1662,6 +1669,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
pInfo->execModel = pTaskInfo->execModel;
pInfo->twAggSup = *pTwAggSupp;
pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
pInfo->pCondition = pPhyNode->window.node.pConditions;
if (pPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;

1
source/libs/executor/src/tsort.c
View File

@ -545,6 +545,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
return 0;
}
// TODO consider the page meta size
int32_t getProperSortPageSize(size_t rowSize) {
uint32_t defaultPageSize = 4096;

2
source/libs/function/inc/builtinsimpl.h
View File

@ -106,6 +106,8 @@ bool irateFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
int32_t irateFunction(SqlFunctionCtx *pCtx);
int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t lastrowFunction(SqlFunctionCtx* pCtx);
bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t firstFunction(SqlFunctionCtx *pCtx);
int32_t firstFunctionMerge(SqlFunctionCtx *pCtx);

16
source/libs/function/src/builtins.c
View File

@ -2211,22 +2211,12 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "last_row",
.type = FUNCTION_TYPE_LAST_ROW,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastRowFunction,
.finalizeFunc = lastRowFinalize,
},
{
.name = "_cache_last_row",
.type = FUNCTION_TYPE_CACHE_LAST_ROW,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
.translateFunc = translateLastRow,
.getEnvFunc = getMinmaxFuncEnv,
.initFunc = minmaxFunctionSetup,
.processFunc = maxFunction,
.finalizeFunc = functionFinalize
.processFunc = lastrowFunction,
.finalizeFunc = firstLastFinalize
},
{
.name = "first",

40
source/libs/function/src/builtinsimpl.c
View File

@ -5937,3 +5937,43 @@ int32_t interpFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t lastrowFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = 0;
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t type = pInputCol->info.type;
int32_t bytes = pInputCol->info.bytes;
pInfo->bytes = bytes;
for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
if (pInputCol->hasNull && colDataIsNull_s(pInputCol, i)) {
continue;
}
numOfElems++;
char* data = colDataGetData(pInputCol, i);
TSKEY cts = getRowPTs(pInput->pPTS, i);
if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) {
if (IS_VAR_DATA_TYPE(type)) {
bytes = varDataTLen(data);
pInfo->bytes = bytes;
}
memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts;
pInfo->hasResult = true;
pResInfo->numOfRes = 1;
}
}
SET_VAL(pResInfo, numOfElems, 1);
return TSDB_CODE_SUCCESS;
}

2
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -394,10 +394,12 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi
COPY_SCALAR_FIELD(msgType);
CLONE_NODE_FIELD(pAffectedRows);
COPY_SCALAR_FIELD(tableId);
COPY_SCALAR_FIELD(stableId);
COPY_SCALAR_FIELD(tableType);
COPY_CHAR_ARRAY_FIELD(tableFName);
COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow));
CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
CLONE_NODE_LIST_FIELD(pInsertCols);
return TSDB_CODE_SUCCESS;
}

14
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -2214,6 +2214,8 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return
static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); }
static const char* jkQueryInsertPhysiPlanInsertCols = "InsertCols";
static const char* jkQueryInsertPhysiPlanStableId = "StableId";
static const char* jkQueryInsertPhysiPlanTableId = "TableId";
static const char* jkQueryInsertPhysiPlanTableType = "TableType";
static const char* jkQueryInsertPhysiPlanTableFName = "TableFName";
@ -2224,6 +2226,12 @@ static int32_t physiQueryInsertNodeToJson(const void* pObj, SJson* pJson) {
const SQueryInserterNode* pNode = (const SQueryInserterNode*)pObj;
int32_t code = physicDataSinkNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkQueryInsertPhysiPlanInsertCols, pNode->pCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanStableId, pNode->stableId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanTableId, pNode->tableId);
}
@ -2247,6 +2255,12 @@ static int32_t jsonToPhysiQueryInsertNode(const SJson* pJson, void* pObj) {
SQueryInserterNode* pNode = (SQueryInserterNode*)pObj;
int32_t code = jsonToPhysicDataSinkNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkQueryInsertPhysiPlanInsertCols, &pNode->pCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUBigIntValue(pJson, jkQueryInsertPhysiPlanStableId, &pNode->stableId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUBigIntValue(pJson, jkQueryInsertPhysiPlanTableId, &pNode->tableId);
}

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -103,6 +103,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SDropDatabaseStmt));
case QUERY_NODE_ALTER_DATABASE_STMT:
return makeNode(type, sizeof(SAlterDatabaseStmt));
case QUERY_NODE_FLUSH_DATABASE_STMT:
return makeNode(type, sizeof(SFlushDatabaseStmt));
case QUERY_NODE_CREATE_TABLE_STMT:
return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -545,6 +547,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_ALTER_DATABASE_STMT:
nodesDestroyNode((SNode*)((SAlterDatabaseStmt*)pNode)->pOptions);
break;
case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field
break;
case QUERY_NODE_CREATE_TABLE_STMT: {
SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode;
nodesDestroyList(pStmt->pCols);

1
source/libs/parser/inc/parAst.h
View File

@ -135,6 +135,7 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal);

1
source/libs/parser/inc/sql.y
View File

@ -157,6 +157,7 @@ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C).
cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); }
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
%type not_exists_opt { bool }
%destructor not_exists_opt { }

16
source/libs/parser/src/parAstCreater.c
View File

@ -912,6 +912,17 @@ SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode*
return (SNode*)pStmt;
}
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) {
return NULL;
}
SAlterDatabaseStmt* pStmt = (SAlterDatabaseStmt*)nodesMakeNode(QUERY_NODE_FLUSH_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
return (SNode*)pStmt;
}
SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt);
STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
@ -1670,5 +1681,10 @@ SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols
pStmt->pTable = pTable;
pStmt->pCols = pCols;
pStmt->pQuery = pQuery;
if (QUERY_NODE_SELECT_STMT == nodeType(pQuery)) {
strcpy(((SSelectStmt*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pQuery)) {
strcpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias);
}
return (SNode*)pStmt;
}

6
source/libs/parser/src/parAstParser.c
View File

@ -195,6 +195,10 @@ static int32_t collectMetaKeyFromAlterDatabase(SCollectMetaKeyCxt* pCxt, SAlterD
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromFlushDatabase(SCollectMetaKeyCxt* pCxt, SFlushDatabaseStmt* pStmt) {
return reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) {
int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) {
@ -487,6 +491,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromSelect(pCxt, (SSelectStmt*)pStmt);
case QUERY_NODE_ALTER_DATABASE_STMT:
return collectMetaKeyFromAlterDatabase(pCxt, (SAlterDatabaseStmt*)pStmt);
case QUERY_NODE_FLUSH_DATABASE_STMT:
return collectMetaKeyFromFlushDatabase(pCxt, (SFlushDatabaseStmt*)pStmt);
case QUERY_NODE_CREATE_TABLE_STMT:
return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLE_STMT:

1
source/libs/parser/src/parTokenizer.c
View File

@ -91,6 +91,7 @@ static SKeyword keywordTable[] = {
{"FILL", TK_FILL},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT},
{"FLUSH", TK_FLUSH},
{"FROM", TK_FROM},
{"FSYNC", TK_FSYNC},
{"FUNCTION", TK_FUNCTION},

140
source/libs/parser/src/parTranslater.c
View File

@ -2839,17 +2839,87 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
return code;
}
static int32_t translateInsertCols(STranslateContext* pCxt, SInsertStmt* pInsert) {
if (NULL == pInsert->pCols) {
return createAllColumns(pCxt, false, &pInsert->pCols);
}
return translateExprList(pCxt, pInsert->pCols);
}
static int32_t translateInsertQuery(STranslateContext* pCxt, SInsertStmt* pInsert) {
int32_t code = resetTranslateNamespace(pCxt);
if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(pCxt, pInsert->pQuery);
}
return code;
}
static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* pPrimaryKeyExpr,
SNodeList** pOrderByList) {
SOrderByExprNode* pOrderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pOrderByExpr) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pOrderByExpr->nullOrder = NULL_ORDER_FIRST;
pOrderByExpr->order = ORDER_ASC;
pOrderByExpr->pExpr = nodesCloneNode(pPrimaryKeyExpr);
if (NULL == pOrderByExpr->pExpr) {
nodesDestroyNode((SNode*)pOrderByExpr);
return TSDB_CODE_OUT_OF_MEMORY;
}
((SExprNode*)pOrderByExpr->pExpr)->orderAlias = true;
NODES_DESTORY_LIST(*pOrderByList);
return nodesListMakeStrictAppend(pOrderByList, (SNode*)pOrderByExpr);
}
static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList);
}
return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSetOperator*)pStmt)->pOrderByList);
}
static int32_t translateInsertProject(STranslateContext* pCxt, SInsertStmt* pInsert) {
SNodeList* pProjects = getProjectList(pInsert->pQuery);
if (LIST_LENGTH(pInsert->pCols) != LIST_LENGTH(pProjects)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns");
}
SNode* pPrimaryKeyExpr = NULL;
SNode* pBoundCol = NULL;
SNode* pProj = NULL;
FORBOTH(pBoundCol, pInsert->pCols, pProj, pProjects) {
SColumnNode* pCol = (SColumnNode*)pBoundCol;
SExprNode* pExpr = (SExprNode*)pProj;
if (!dataTypeEqual(&pCol->node.resType, &pExpr->resType)) {
SNode* pFunc = NULL;
int32_t code = createCastFunc(pCxt, pProj, pCol->node.resType, &pFunc);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
REPLACE_LIST2_NODE(pFunc);
pExpr = (SExprNode*)pFunc;
}
snprintf(pExpr->aliasName, sizeof(pExpr->aliasName), "%s", pCol->colName);
if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
pPrimaryKeyExpr = pProj;
}
}
return addOrderByPrimaryKeyToQuery(pCxt, pPrimaryKeyExpr, pInsert->pQuery);
}
static int32_t translateInsert(STranslateContext* pCxt, SInsertStmt* pInsert) {
pCxt->pCurrStmt = (SNode*)pInsert;
int32_t code = translateFrom(pCxt, pInsert->pTable);
if (TSDB_CODE_SUCCESS == code) {
code = translateExprList(pCxt, pInsert->pCols);
code = translateInsertCols(pCxt, pInsert);
}
if (TSDB_CODE_SUCCESS == code) {
code = resetTranslateNamespace(pCxt);
code = translateInsertQuery(pCxt, pInsert);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(pCxt, pInsert->pQuery);
code = translateInsertProject(pCxt, pInsert);
}
return code;
}
@ -6129,6 +6199,67 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
return code;
}
static int32_t serializeFlushVgroup(SVgroupInfo* pVg, SArray* pBufArray) {
int32_t len = sizeof(SMsgHead);
void* buf = taosMemoryMalloc(len);
if (NULL == buf) {
return TSDB_CODE_OUT_OF_MEMORY;
}
((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
((SMsgHead*)buf)->contLen = htonl(len);
SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
if (NULL == pVgData) {
taosMemoryFree(buf);
return TSDB_CODE_OUT_OF_MEMORY;
}
pVgData->vg = *pVg;
pVgData->pData = buf;
pVgData->size = len;
taosArrayPush(pBufArray, &pVgData);
return TSDB_CODE_SUCCESS;
}
static int32_t serializeFlushDb(SArray* pVgs, SArray** pOutput) {
int32_t numOfVgs = taosArrayGetSize(pVgs);
SArray* pBufArray = taosArrayInit(numOfVgs, sizeof(void*));
if (NULL == pBufArray) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < numOfVgs; ++i) {
int32_t code = serializeFlushVgroup((SVgroupInfo*)taosArrayGet(pVgs, i), pBufArray);
if (TSDB_CODE_SUCCESS != code) {
taosArrayDestroy(pBufArray);
return code;
}
}
*pOutput = pBufArray;
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) {
SFlushDatabaseStmt* pStmt = (SFlushDatabaseStmt*)pQuery->pRoot;
SArray* pBufArray = NULL;
SArray* pVgs = NULL;
int32_t code = getDBVgInfo(pCxt, pStmt->dbName, &pVgs);
if (TSDB_CODE_SUCCESS == code) {
code = serializeFlushDb(pVgs, &pBufArray);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
}
if (TSDB_CODE_SUCCESS != code) {
taosArrayDestroy(pBufArray);
}
taosArrayDestroy(pVgs);
return code;
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pQuery->pRoot)) {
@ -6177,6 +6308,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_ALTER_TABLE_STMT:
code = rewriteAlterTable(pCxt, pQuery);
break;
case QUERY_NODE_FLUSH_DATABASE_STMT:
code = rewriteFlushDatabase(pCxt, pQuery);
break;
default:
break;
}

5893
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

8
source/libs/planner/src/planLogicCreater.c
View File

@ -1138,6 +1138,8 @@ static int32_t getMsgType(ENodeType sqlType) {
return TDMT_VND_DROP_TABLE;
case QUERY_NODE_ALTER_TABLE_STMT:
return TDMT_VND_ALTER_TABLE;
case QUERY_NODE_FLUSH_DATABASE_STMT:
return TDMT_VND_COMMIT;
default:
break;
}
@ -1279,10 +1281,16 @@ static int32_t createVnodeModifLogicNodeByInsert(SLogicPlanContext* pCxt, SInser
pModify->modifyType = MODIFY_TABLE_TYPE_INSERT;
pModify->tableId = pRealTable->pMeta->uid;
pModify->stableId = pRealTable->pMeta->suid;
pModify->tableType = pRealTable->pMeta->tableType;
snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId,
pRealTable->table.dbName, pRealTable->table.tableName);
TSWAP(pModify->pVgroupList, pRealTable->pVgroupList);
pModify->pInsertCols = nodesCloneList(pInsert->pCols);
if (NULL == pModify->pInsertCols) {
nodesDestroyNode((SNode*)pModify);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pLogicNode = (SLogicNode*)pModify;
return TSDB_CODE_SUCCESS;

18
source/libs/planner/src/planOptimizer.c
View File

@ -890,6 +890,20 @@ static int32_t pushDownCondOptDealProject(SOptimizeContext* pCxt, SProjectLogicN
return code;
}
static int32_t pushDownCondOptDealLogicNode(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
if (NULL == pLogicNode->pConditions ||
OPTIMIZE_FLAG_TEST_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS;
}
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pLogicNode->pChildren, 0);
int32_t code = pushDownCondOptPushCondToChild(pCxt, pChild, &pLogicNode->pConditions);
if (TSDB_CODE_SUCCESS == code) {
OPTIMIZE_FLAG_SET_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
pCxt->optimized = true;
}
return code;
}
static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pLogicNode)) {
@ -905,6 +919,10 @@ static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLog
case QUERY_NODE_LOGIC_PLAN_PROJECT:
code = pushDownCondOptDealProject(pCxt, (SProjectLogicNode*)pLogicNode);
break;
case QUERY_NODE_LOGIC_PLAN_SORT:
case QUERY_NODE_LOGIC_PLAN_PARTITION:
code = pushDownCondOptDealLogicNode(pCxt, pLogicNode);
break;
default:
break;
}

17
source/libs/planner/src/planPhysiCreater.c
View File

@ -1510,18 +1510,21 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod
}
pInserter->tableId = pModify->tableId;
pInserter->stableId = pModify->stableId;
pInserter->tableType = pModify->tableType;
strcpy(pInserter->tableFName, pModify->tableFName);
pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId;
pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet;
vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode);
int32_t code = TSDB_CODE_SUCCESS;
pInserter->sink.pInputDataBlockDesc =
(SDataBlockDescNode*)nodesCloneNode((SNode*)pSubplan->pNode->pOutputDataBlockDesc);
if (NULL == pInserter->sink.pInputDataBlockDesc) {
code = TSDB_CODE_OUT_OF_MEMORY;
int32_t code = setListSlotId(pCxt, pSubplan->pNode->pOutputDataBlockDesc->dataBlockId, -1, pModify->pInsertCols,
&pInserter->pCols);
if (TSDB_CODE_SUCCESS == code) {
pInserter->sink.pInputDataBlockDesc =
(SDataBlockDescNode*)nodesCloneNode((SNode*)pSubplan->pNode->pOutputDataBlockDesc);
if (NULL == pInserter->sink.pInputDataBlockDesc) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
@ -1530,7 +1533,7 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod
nodesDestroyNode((SNode*)pInserter);
}
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t buildInsertSelectSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {

4
source/libs/planner/test/planOtherTest.cpp
View File

@ -96,4 +96,8 @@ TEST_F(PlanOtherTest, insert) {
useDb("root", "test");
run("INSERT INTO t1 SELECT * FROM t1");
run("INSERT INTO t1 (ts, c1, c2) SELECT ts, c1, c2 FROM st1");
run("INSERT INTO t1 (ts, c1, c2) SELECT ts, c1, c2 FROM st1s1 UNION ALL SELECT ts, c1, c2 FROM st2");
}

2
source/libs/qworker/inc/qwMsg.h
View File

@ -31,7 +31,7 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req);
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SRpcMsg *pRsp, SDeleteRes *pRes);
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes);
int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code);
int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code);

6
source/libs/qworker/src/qwMsg.c
View File

@ -583,8 +583,8 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_
}
int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SRpcMsg *pRsp, SDeleteRes *pRes) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg || NULL == pRsp) {
int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SDeleteRes *pRes) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
@ -606,7 +606,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SR
QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql);
taosMemoryFreeClear(req.sql);
QW_ERR_JRET(qwProcessDelete(QW_FPARAMS(), &qwMsg, pRsp, pRes));
QW_ERR_JRET(qwProcessDelete(QW_FPARAMS(), &qwMsg, pRes));
QW_SCH_TASK_DLOG("processDelete end, node:%p", node);

4
source/libs/qworker/src/qwUtil.c
View File

@ -433,8 +433,8 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) {
}
void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN] = {0};
char tbName[TSDB_TABLE_NAME_LEN] = {0};
qGetQueriedTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion);

21
source/libs/qworker/src/qworker.c
View File

@ -242,9 +242,8 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
return TSDB_CODE_SUCCESS;
}
int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SDeleteRes *pRes) {
int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes) {
int32_t len = 0;
SVDeleteRsp rsp = {0};
bool queryEnd = false;
int32_t code = 0;
SOutputData output = {0};
@ -270,21 +269,11 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen
SDeleterRes* pDelRes = (SDeleterRes*)output.pData;
rsp.affectedRows = pDelRes->affectedRows;
pRes->suid = pDelRes->suid;
pRes->uidList = pDelRes->uidList;
pRes->skey = pDelRes->skey;
pRes->ekey = pDelRes->ekey;
SEncoder coder = {0};
tEncodeSize(tEncodeSVDeleteRsp, &rsp, len, code);
void *msg = rpcMallocCont(len);
tEncoderInit(&coder, msg, len);
tEncodeSVDeleteRsp(&coder, &rsp);
tEncoderClear(&coder);
*rspMsg = msg;
*dataLen = len;
pRes->affectedRows = pDelRes->affectedRows;
return TSDB_CODE_SUCCESS;
}
@ -926,7 +915,7 @@ _return:
qwRelease(refId);
}
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SRpcMsg *pRsp, SDeleteRes *pRes) {
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
int32_t code = 0;
SSubplan *plan = NULL;
qTaskInfo_t pTaskInfo = NULL;
@ -941,7 +930,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SRpcMsg *pRsp, SDeleteRes
}
ctx.plan = plan;
code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH);
if (code) {
QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
@ -958,7 +947,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SRpcMsg *pRsp, SDeleteRes
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx, NULL));
QW_ERR_JRET(qwGetDeleteResFromSink(QW_FPARAMS(), &ctx, &pRsp->contLen, &pRsp->pCont, pRes));
QW_ERR_JRET(qwGetDeleteResFromSink(QW_FPARAMS(), &ctx, pRes));
_return:

112
source/libs/scheduler/src/schRemote.c
View File

@ -21,7 +21,6 @@
#include "tref.h"
#include "trpc.h"
int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
int32_t lastMsgType = pTask->lastMsgType;
int32_t taskStatus = SCH_GET_TASK_STATUS(pTask);
@ -42,7 +41,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
TMSG_INFO(msgType));
}
//SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS;
case TDMT_SCH_FETCH_RSP:
if (lastMsgType != reqMsgType && -1 != lastMsgType) {
@ -57,13 +56,14 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
//SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS;
case TDMT_VND_CREATE_TABLE_RSP:
case TDMT_VND_DROP_TABLE_RSP:
case TDMT_VND_ALTER_TABLE_RSP:
case TDMT_VND_SUBMIT_RSP:
case TDMT_VND_DELETE_RSP:
case TDMT_VND_COMMIT_RSP:
break;
default:
SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%s", TMSG_INFO(msgType), jobTaskStatusStr(taskStatus));
@ -82,7 +82,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
//SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
// SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
return TSDB_CODE_SUCCESS;
}
@ -105,6 +105,11 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
}
switch (msgType) {
case TDMT_VND_COMMIT_RSP: {
SCH_ERR_JRET(rspCode);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
case TDMT_VND_CREATE_TABLE_RSP: {
SVCreateTbBatchRsp batchRsp = {0};
if (msg) {
@ -126,8 +131,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
}
SCH_ERR_JRET(rspCode);
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
@ -152,8 +157,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
}
SCH_ERR_JRET(rspCode);
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
@ -166,7 +171,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
tDecoderClear(&coder);
SCH_ERR_JRET(code);
SCH_ERR_JRET(rsp.code);
pJob->execRes.res = rsp.pMeta;
pJob->execRes.msgType = TDMT_VND_ALTER_TABLE;
}
@ -177,8 +182,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
@ -226,7 +231,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
}
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
@ -236,7 +241,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_ERR_JRET(rspCode);
if (msg) {
SDecoder coder = {0};
SDecoder coder = {0};
SVDeleteRsp rsp = {0};
tDecoderInit(&coder, msg, msgSize);
tDecodeSVDeleteRsp(&coder, &rsp);
@ -250,7 +255,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
}
}
case TDMT_SCH_QUERY_RSP:
case TDMT_SCH_MERGE_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
@ -263,8 +268,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp));
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
break;
@ -315,14 +320,14 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp));
}
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
return TSDB_CODE_SUCCESS;
}
SCH_ERR_JRET(schLaunchFetchTask(pJob));
taosMemoryFreeClear(msg);
taosMemoryFreeClear(msg);
return TSDB_CODE_SUCCESS;
}
@ -342,7 +347,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed);
msg = NULL;
msg = NULL;
schProcessOnDataFetched(pJob);
break;
@ -371,7 +376,6 @@ _return:
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
}
int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
@ -381,7 +385,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
qDebug("begin to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode));
SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId));
code = schHandleResponseMsg(pJob, pTask, pParam->execId, pMsg, rspCode);
pMsg->pData = NULL;
@ -397,7 +401,8 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code);
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId,
code);
taosMemoryFreeClear(param);
return TSDB_CODE_SUCCESS;
}
@ -421,6 +426,9 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) {
return TSDB_CODE_SUCCESS;
}
int32_t schHandleCommitCallback(void *param, SDataBuf *pMsg, int32_t code) {
return schHandleCallback(param, pMsg, code);
}
int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchedulerHbRsp rsp = {0};
@ -460,7 +468,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo
SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
param->queryId = pJob->queryId;
param->refId = pJob->refId;
param->taskId = SCH_TASK_ID(pTask);
@ -477,19 +485,19 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo
SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
param->head.isHbParam = true;
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
param->nodeEpId.nodeId = addr->nodeId;
SEp* pEp = SCH_GET_CUR_EP(addr);
SEp *pEp = SCH_GET_CUR_EP(addr);
strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn);
param->nodeEpId.ep.port = pEp->port;
param->pTrans = trans->pTrans;
*pParam = param;
return TSDB_CODE_SUCCESS;
}
}
// hb msg
SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam));
@ -497,14 +505,15 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo
qError("calloc SSchTaskCallbackParam failed");
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
param->pTrans = trans->pTrans;
*pParam = param;
return TSDB_CODE_SUCCESS;
}
int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void* msg, uint32_t msgSize, int32_t msgType, SSchTrans *trans, bool isHb, SMsgSendInfo **pMsgSendInfo) {
int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint32_t msgSize, int32_t msgType,
SSchTrans *trans, bool isHb, SMsgSendInfo **pMsgSendInfo) {
int32_t code = 0;
SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == msgSendInfo) {
@ -519,7 +528,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void* msg, uint3
msgSendInfo->requestId = pJob->conn.requestId;
msgSendInfo->requestObjRefId = pJob->conn.requestObjRefId;
}
if (TDMT_SCH_LINK_BROKEN != msgType) {
msgSendInfo->msgInfo.pData = msg;
msgSendInfo->msgInfo.len = msgSize;
@ -538,7 +547,6 @@ _return:
SCH_RET(code);
}
int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
switch (msgType) {
case TDMT_VND_CREATE_TABLE:
@ -558,6 +566,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
case TDMT_SCH_QUERY_HEARTBEAT:
*fp = schHandleHbCallback;
break;
case TDMT_VND_COMMIT:
*fp = schHandleCommitCallback;
break;
case TDMT_SCH_LINK_BROKEN:
*fp = schHandleLinkBrokenCallback;
break;
@ -635,7 +646,6 @@ _return:
SCH_RET(code);
}
int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) {
int32_t code = 0;
SSchHbCallbackParam *param = NULL;
@ -695,9 +705,9 @@ _return:
}
int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) {
int32_t code = 0;
int32_t msgType = TDMT_SCH_LINK_BROKEN;
SSchTrans trans = {.pTrans = pJob->conn.pTrans};
int32_t code = 0;
int32_t msgType = TDMT_SCH_LINK_BROKEN;
SSchTrans trans = {.pTrans = pJob->conn.pTrans};
SMsgSendInfo *pMsgSendInfo = NULL;
SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, NULL, 0, msgType, &trans, isHb, &pMsgSendInfo));
@ -818,19 +828,18 @@ int32_t schUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, SQueryNodeAddr *addr
return TSDB_CODE_SUCCESS;
}
int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQueryNodeAddr *addr, int32_t msgType, void *msg,
uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) {
int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQueryNodeAddr *addr, int32_t msgType,
void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) {
int32_t code = 0;
SEpSet *epSet = &addr->epSet;
SMsgSendInfo *pMsgSendInfo = NULL;
bool isHb = (TDMT_SCH_QUERY_HEARTBEAT == msgType);
bool isHb = (TDMT_SCH_QUERY_HEARTBEAT == msgType);
SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo));
SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask));
SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask));
qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType),
addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port,
trans->pTrans, trans->pHandle);
qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId,
epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle);
if (pTask) {
pTask->lastMsgType = msgType;
@ -865,8 +874,7 @@ _return:
SCH_RET(code);
}
int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction) {
int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) {
SSchedulerHbReq req = {0};
int32_t code = 0;
SRpcCtx rpcCtx = {0};
@ -910,7 +918,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction) {
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
int64_t transporterId = 0;
int64_t transporterId = 0;
SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId};
addr.epSet.inUse = 0;
addr.epSet.numOfEps = 1;
@ -945,7 +953,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
case TDMT_VND_CREATE_TABLE:
case TDMT_VND_DROP_TABLE:
case TDMT_VND_ALTER_TABLE:
case TDMT_VND_SUBMIT: {
case TDMT_VND_SUBMIT:
case TDMT_VND_COMMIT: {
msgSize = pTask->msgLen;
msg = taosMemoryCalloc(1, msgSize);
if (NULL == msg) {
@ -965,7 +974,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
req.taskId = pTask->taskId;
req.phyLen = pTask->msgLen;
req.sqlLen = strlen(pJob->sql);
req.sql = (char*)pJob->sql;
req.sql = (char *)pJob->sql;
req.msg = pTask->msg;
msgSize = tSerializeSVDeleteReq(NULL, 0, &req);
msg = taosMemoryCalloc(1, msgSize);
@ -977,7 +986,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
tSerializeSVDeleteReq(msg, msgSize, &req);
break;
}
case TDMT_SCH_QUERY:
case TDMT_SCH_QUERY:
case TDMT_SCH_MERGE_QUERY: {
SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx));
@ -1079,8 +1088,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
}
SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle,
(rpcCtx.args ? &rpcCtx : NULL)));
SCH_ERR_JRET(
schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)));
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) {
SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId));
@ -1096,6 +1105,3 @@ _return:
taosMemoryFreeClear(msg);
SCH_RET(code);
}

1
source/util/src/terror.c
View File

@ -77,6 +77,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp format")
TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error")
TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk")
TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")

110
source/util/src/tmallocator.c
View File

@ -1,110 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tmallocator.h"
/* ------------------------ HEAP ALLOCATOR ------------------------ */
#if 0
typedef struct {
size_t tusage;
} SHeapAllocator;
static void * haMalloc(SMemAllocator *pma, size_t size);
static void * haCalloc(SMemAllocator *pma, size_t nmemb, size_t size);
static void * haRealloc(SMemAllocator *pma, void *ptr, size_t size);
static void haFree(SMemAllocator *pma, void *ptr);
static size_t haUsage(SMemAllocator *pma);
SMemAllocator *tdCreateHeapAllocator() {
SMemAllocator *pma = NULL;
pma = taosMemoryCalloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator));
if (pma) {
pma->impl = POINTER_SHIFT(pma, sizeof(SMemAllocator));
pma->malloc = haMalloc;
pma->calloc = haCalloc;
pma->realloc = haRealloc;
pma->free = haFree;
pma->usage = haUsage;
}
return pma;
}
void tdDestroyHeapAllocator(SMemAllocator *pMemAllocator) {
// TODO
}
static void *haMalloc(SMemAllocator *pma, size_t size) {
void * ptr;
size_t tsize = size + sizeof(size_t);
SHeapAllocator *pha = (SHeapAllocator *)(pma->impl);
ptr = taosMemoryMalloc(tsize);
if (ptr) {
*(size_t *)ptr = size;
ptr = POINTER_SHIFT(ptr, sizeof(size_t));
atomic_fetch_add_64(&(pha->tusage), tsize);
}
return ptr;
}
static void *haCalloc(SMemAllocator *pma, size_t nmemb, size_t size) {
void * ptr;
size_t tsize = nmemb * size;
ptr = haMalloc(pma, tsize);
if (ptr) {
memset(ptr, 0, tsize);
}
return ptr;
}
static void *haRealloc(SMemAllocator *pma, void *ptr, size_t size) {
size_t psize;
size_t tsize = size + sizeof(size_t);
if (ptr == NULL) {
psize = 0;
} else {
psize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
}
if (psize < size) {
// TODO
} else {
return ptr;
}
}
static void haFree(SMemAllocator *pma, void *ptr) { /* TODO */
SHeapAllocator *pha = (SHeapAllocator *)(pma->impl);
if (ptr) {
size_t tsize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)) + sizeof(size_t);
atomic_fetch_sub_64(&(pha->tusage), tsize);
taosMemoryFree(POINTER_SHIFT(ptr, -sizeof(size_t)));
}
}
static size_t haUsage(SMemAllocator *pma) { return ((SHeapAllocator *)(pma->impl))->tusage; }
/* ------------------------ ARENA ALLOCATOR ------------------------ */
typedef struct {
size_t usage;
} SArenaAllocator;
#endif

2
tests/pytest/util/constant.py
View File

@ -93,7 +93,7 @@ TS_FUNC = [
"CSUM", "DERIVATIVE", "DIFF", "IRATE", "MAVG", "SAMPLE", "STATECOUNT", "STATEDURATION", "TWA"
]
SYSINFO_FUCN = [
SYSINFO_FUNC = [
"DATABASE", "CLIENT_VERSION", "SERVER_VERSION", "SERVER_STATUS", "CURRENT_USER", "USER"
]

4
tests/script/jenkins/basic.txt
View File

@ -163,8 +163,8 @@
# --- sma
./test.sh -f tsim/sma/drop_sma.sim
./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
# --- valgrind
./test.sh -f tsim/valgrind/checkError1.sim

335
tests/script/tsim/query/read.sim Normal file
View File

@ -0,0 +1,335 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
print ========= start dnode1 as leader
system sh/exec.sh -n dnode1 -s start
sleep 2000
sql connect
sql create database abc1 vgroups 2;
sql use abc1;
sql create table st1 (ts timestamp, k int, x int, y int, z binary(12), u nchar(12)) tags(a int, b nchar(12), c varchar(24), d bool) sma(x);
sql create table tu using st1 tags(1, 'abc', 'binary1', true);
sql create table tu1 using st1 tags(2, '水木', 'binary2', false);
sql create table tu2 using st1 tags(3, '水木1', 'binary3', true);
sql create table tu3 using st1 tags(4, 'abc', '12', false);
sql insert into tu values('2022-01-01 1:1:1', 1, 10, 9, 'a', '水3木') ('2022-07-02 22:46:53.294', 2, 10, 8, 'a', '水1木') ('2022-07-02 22:47:53.294', 1, 10, 7, 'b', '水2木')('2022-07-02 22:48:53.294', 1, 10, null, 'd', '3')('2022-07-02 22:50:53.294', 1, 10, null, null, '322');
sql insert into tu1 values('2022-01-01 1:1:1', 11, 101, 91, 'aa', '3水木');
sql insert into tu2 values('2022-01-01 1:1:1', 111, 1010, 919, 'aaa', '3水木3');
sql select * from tu;
if $rows != 5 then
return -1
endi
sql select * from tu order by ts desc;
if $rows != 5 then
return -1
endi
sql create table st2 (ts timestamp, k int, x int, y int, z binary(12), u nchar(12)) tags(a int) sma(x);
sql create table tuu1 using st2 tags(2);
sql insert into tuu1 values('2022-01-01 1:1:1', 11, 101, 911, 'aa', '3水木33');
sql insert into tuu1 values('2022-01-01 1:1:2', NULL, 101, 911, 'aa', '3水木33');
sql insert into tu values('2022-01-01 1:1:1', NULL, NULL, NULL, NULL, '水3木');
sql insert into tu values('2022-01-01 1:1:1', NULL, 911, NULL, NULL, '');
sql flush database abc1;
sql insert into tu values('2021-12-1 1:1:1', 1,1,1,'a', 12);
sql insert into tu values('2022-6-1 1:1:1', 1,1,1,'a', 12);
sql insert into tu values('2022-6-1 1:1:2', 1,1,1,'a', 12);
sql insert into tu values('2022-6-1 1:1:3', 1,1,1,'a', 12);
sql select * from tu order by ts desc;
if $rows != 9 then
return -1
endi
sql select * from tu order by ts asc;
if $rows != 9 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:1:0' order by ts asc;
if $rows != 0 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:1:0' order by ts desc;
if $rows != 0 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:1:2' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:1:2' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:1:2' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:10:2' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-1 1:10:2' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-9 1:10:2' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<'2022-1-9 1:10:2' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-1-9 1:10:2' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-1-9 1:10:2' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-9 1:10:2' order by ts asc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-9 1:10:2' order by ts desc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:10:2' order by ts asc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:10:2' order by ts desc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:1' order by ts asc;
if $row != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:1' order by ts desc;
if $row != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' order by ts asc;
if $rows != 8 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' order by ts desc;
if $rows != 8 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' order by ts asc;
if $rows != 9 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' order by ts desc;
if $rows != 9 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:2' order by ts asc;
if $rows != 3 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:2' order by ts desc;
if $rows != 3 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:1' order by ts asc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:1' order by ts desc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:0' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-31 1:1:1' and ts<='2022-6-1 1:1:0' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-6-1 1:1:0' order by ts asc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-6-1 1:1:0' order by ts desc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-1 1:1:0' order by ts asc;
if $rows != 5 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-1 1:1:0' order by ts desc;
if $rows != 5 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-7 1:1:0' order by ts desc;
if $rows != 9 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-7 1:1:0' order by ts asc;
if $rows != 9 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-2 1:1:0' order by ts desc;
if $rows != 5 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-2 1:1:0' order by ts asc;
if $rows != 5 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-2 22:47:0' order by ts desc;
if $rows != 6 then
return -1
endi
sql select * from tu where ts>='2021-12-1 1:1:1' and ts<='2022-7-2 22:47:0' order by ts asc;
if $rows != 6 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:47:0' order by ts asc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:47:0' order by ts desc;
if $rows != 1 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-8-2 22:47:0' order by ts asc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-8-2 22:47:0' order by ts desc;
if $rows != 4 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.299' order by ts asc;
if $rows != 3 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.299' order by ts desc;
if $rows != 3 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.293';
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.293' order by ts desc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.292' order by ts asc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.292' order by ts desc;
if $rows != 2 then
return -1
endi
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<='2022-7-2 22:48:53.294';
sql select * from tu where ts>='2022-7-1 1:1:1' and ts<'2022-7-2 22:48:53.294';
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:48:53.294';
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:47:53.294';
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:47:53.293' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:47:53.293' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts desc;
sql select * from tu where ts>='2022-7-1 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts desc;
sql select * from tu where ts>='2022-7-1 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts asc;
sql select * from tu where ts>='2022-7-1 22:46:55' and ts<='2022-7-2 22:48:53.293' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:47:53.294';
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:48:53.294';
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:48:53.294' order by ts desc;
sql select * from tu where ts>='2022-7-1 22:46:55' and ts<'2022-7-2 22:48:53.294' order by ts desc;
sql select * from tu where ts>='2021-12-2 22:46:55' and ts<'2022-7-2 22:48:53.294' order by ts desc;
sql select * from tu where ts>='2021-12-2 22:46:55' and ts<'2022-7-2 22:48:53.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:59.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:59.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:59.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:59.294' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:59.294' order by ts desc;
sql select * from tu where ts>='2021-7-2 22:46:55' and ts<'2022-7-2 22:46:54.294' order by ts desc;
sql select * from tu where ts>='2021-12-2 22:46:55' and ts<'2022-7-2 22:46:54.294' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:55' and ts<'2022-7-2 22:46:54.294' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:51' and ts<'2022-7-2 22:48:54.294' order by ts desc;
sql select * from tu where ts>='2022-7-2 22:46:51' and ts<'2022-7-2 22:48:54.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:51' and ts<'2022-7-2 22:58:54.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:51' and ts<'2022-7-2 22:58:54.294' order by ts asc;
sql select * from tu where ts>='2022-7-2 22:46:51' and ts<'2022-7-2 22:58:54.294' order by ts desc;

1
tests/script/tsim/stable/alter_count.sim
View File

@ -150,6 +150,7 @@ endi
sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb
if $data01 != 21 then
print expect 21, actual $data01
return -1
endi

10
tests/script/tsim/stream/distributeInterval0.sim
View File

@ -97,23 +97,23 @@ if $data01 != 8 then
goto loop1
endi
if $data02 != 4 then
if $data02 != 6 then
print =====data02=$data02
goto loop1
endi
if $data03 != 4 then
print ======$data03
if $data03 != 52 then
print ======data03=$data03
goto loop1
endi
if $data04 != 52 then
print ======$data04
print ======data04=$data04
goto loop1
endi
if $data05 != 13 then
print ======$data05
print ======data05=$data05
goto loop1
endi

49
tests/script/tsim/testsuit.sim
View File

@ -1,9 +1,7 @@
run tsim/user/pass_alter.sim
run tsim/user/basic1.sim
run tsim/user/privilege2.sim
run tsim/user/user_len.sim
run tsim/user/privilege1.sim
run tsim/user/pass_len.sim
run tsim/user/password.sim
run tsim/user/privilege_db.sim
run tsim/user/privilege_sysinfo.sim
run tsim/user/basic.sim
run tsim/table/basic1.sim
run tsim/trans/lossdata1.sim
run tsim/trans/create_db.sim
@ -26,18 +24,23 @@ run tsim/stable/values.sim
run tsim/stable/dnode3.sim
run tsim/stable/alter_insert1.sim
run tsim/stable/refcount.sim
run tsim/stable/tag_filter.sim
run tsim/stable/disk.sim
run tsim/db/basic1.sim
run tsim/db/basic3.sim
run tsim/db/basic7.sim
run tsim/db/basic6.sim
run tsim/db/alter_replica_13.sim
run tsim/db/create_all_options.sim
run tsim/db/basic2.sim
run tsim/db/error1.sim
run tsim/db/alter_replica_31.sim
run tsim/db/taosdlog.sim
run tsim/db/alter_option.sim
run tsim/mnode/basic1.sim
#run tsim/mnode/basic3.sim
run tsim/mnode/basic4.sim
run tsim/mnode/basic3.sim
run tsim/mnode/basic5.sim
run tsim/mnode/basic2.sim
run tsim/parser/fourArithmetic-basic.sim
run tsim/parser/groupby-basic.sim
@ -57,11 +60,12 @@ run tsim/query/complex_group.sim
run tsim/query/interval.sim
run tsim/query/session.sim
run tsim/query/scalarFunction.sim
#run tsim/query/scalarNull.sim
run tsim/query/scalarNull.sim
run tsim/query/complex_where.sim
run tsim/tmq/basic1.sim
run tsim/tmq/basic4.sim
run tsim/tmq/basic1Of2Cons.sim
run tsim/tmq/snapshot.sim
run tsim/tmq/prepareBasicEnv-1vgrp.sim
run tsim/tmq/topic.sim
run tsim/tmq/basic4Of2Cons.sim
@ -73,14 +77,36 @@ run tsim/tmq/basic3Of2Cons.sim
run tsim/tmq/basic2Of2ConsOverlap.sim
run tsim/tmq/clearConsume.sim
run tsim/qnode/basic1.sim
run tsim/dnode/basic1.sim
run tsim/dnode/redistribute_vgroup_replica3_v3.sim
run tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
run tsim/dnode/redistribute_vgroup_replica3_v2.sim
run tsim/dnode/drop_dnode_has_mnode.sim
run tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
run tsim/dnode/drop_dnode_has_vnode_replica1.sim
run tsim/dnode/balance_replica3.sim
run tsim/dnode/redistribute_vgroup_replica1.sim
run tsim/dnode/drop_dnode_has_vnode_replica3.sim
run tsim/dnode/balance_replica1.sim
run tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
run tsim/dnode/drop_dnode_has_qnode_snode.sim
run tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
run tsim/dnode/create_dnode.sim
run tsim/testsuit.sim
run tsim/show/basic.sim
run tsim/stream/basic1.sim
run tsim/stream/windowClose.sim
run tsim/stream/partitionby1.sim
run tsim/stream/triggerInterval0.sim
run tsim/stream/triggerSession0.sim
run tsim/stream/distributeIntervalRetrive0.sim
run tsim/stream/basic0.sim
run tsim/stream/session0.sim
run tsim/stream/schedSnode.sim
run tsim/stream/partitionby.sim
run tsim/stream/session1.sim
run tsim/stream/distributeInterval0.sim
run tsim/stream/distributeSession0.sim
run tsim/stream/state0.sim
run tsim/stream/basic2.sim
run tsim/insert/basic1.sim
run tsim/insert/commit-merge0.sim
@ -88,15 +114,18 @@ run tsim/insert/basic0.sim
run tsim/insert/update0.sim
run tsim/insert/backquote.sim
run tsim/insert/null.sim
run tsim/catalog/alterInCurrent.sim
run tsim/sync/oneReplica1VgElectWithInsert.sim
run tsim/sync/threeReplica1VgElect.sim
run tsim/sync/oneReplica1VgElect.sim
run tsim/sync/3Replica5VgElect.sim
run tsim/sync/3Replica5VgElect3mnodedrop.sim
run tsim/sync/3Replica5VgElect3mnode.sim
run tsim/sync/insertDataByRunBack.sim
run tsim/sync/oneReplica5VgElect.sim
run tsim/sync/3Replica1VgElect.sim
run tsim/sync/threeReplica1VgElectWihtInsert.sim
run tsim/sma/tsmaCreateInsertData.sim
run tsim/sma/tsmaCreateInsertQuery.sim
run tsim/sma/rsmaCreateInsertQuery.sim
run tsim/valgrind/basic.sim
run tsim/valgrind/checkError.sim

1
tests/script/tsim/tmq/basic1.sim
View File

@ -131,6 +131,7 @@ if $data[0][1] != $consumerId then
return -1
endi
if $data[0][2] != $expectmsgcnt then
print expect $expectmsgcnt , actual $data02
return -1
endi
if $data[0][3] != $expectmsgcnt then

16
tests/system-test/0-others/udfTest.py
View File

@ -313,11 +313,11 @@ class TDTestCase:
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select udf1(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
tdSql.query("select round(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select round(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
# stable
@ -342,10 +342,10 @@ class TDTestCase:
tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
# tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# regular table with compute functions

16
tests/system-test/0-others/udf_create.py
View File

@ -315,11 +315,11 @@ class TDTestCase:
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select udf1(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
tdSql.query("select round(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select round(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
# stable
@ -344,10 +344,10 @@ class TDTestCase:
tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
# tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# regular table with compute functions

16
tests/system-test/0-others/udf_restart_taosd.py
View File

@ -312,11 +312,11 @@ class TDTestCase:
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select udf1(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
tdSql.query("select round(num1) , last_row(num1) from tb;")
tdSql.checkRows(1)
# tdSql.query("select round(num1) , last_row(num1) from tb;")
# tdSql.checkRows(1)
# stable
@ -341,10 +341,10 @@ class TDTestCase:
tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
tdSql.checkRows(1)
# tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
# tdSql.checkRows(1)
# regular table with compute functions

49
tests/system-test/1-insert/create_retentions.py
View File

@ -87,29 +87,29 @@ class TDTestCase:
@property
def create_stable_sql_err(self):
return [
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m",
# f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) tags (tag1 int) " ,
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " ,
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " ,
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " ,
f"create stable stb11 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m",
f"create stable stb12 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min",
f"create stable stb13 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s",
f"create stable stb14 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m",
f"create stable stb15 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
f"create stable stb16 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ",
f"create stable stb21 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s",
f"create stable stb22 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m",
f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s",
f"create stable stb23 ({PRIMARY_COL} timestamp, {INT_COL} int, {NCHAR_COL} nchar(16)) tags (tag1 int) " ,
f"create stable stb24 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " ,
f"create stable stb25 ({PRIMARY_COL} timestamp, {INT_COL} int) " ,
f"create stable stb26 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " ,
# watermark, max_delay: [0, 900000], [ms, s, m, ?]
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d",
f"create stable stb17 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u",
f"create stable stb18 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b",
f"create stable stb19 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms",
f"create stable stb20 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m",
f"create stable stb27 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s",
f"create stable stb28 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h",
f"create stable stb29 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h",
f"create stable stb30 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d",
]
@ -125,8 +125,8 @@ class TDTestCase:
f"create stable stb7 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL})",
]
def test_create_stb(self):
tdSql.execute("use db2")
def test_create_stb(self, db="db2"):
tdSql.execute(f"use {db}")
for err_sql in self.create_stable_sql_err:
tdSql.error(err_sql)
for cur_sql in self.create_stable_sql_current:
@ -136,7 +136,7 @@ class TDTestCase:
tdSql.checkRows(len(self.create_stable_sql_current))
tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup stable
# tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m")
tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m")
def test_create_databases(self):
@ -255,6 +255,7 @@ class TDTestCase:
tdSql.execute("drop database if exists db2 ")
tdSql.execute("use db3")
self.test_create_stb(db="db3")
# self.__create_tb()
# self.__insert_data()
self.all_test()

14
tests/system-test/1-insert/test_stmt_set_tbname_tag.py
View File

@ -218,13 +218,13 @@ class TDTestCase:
tdLog.debug("assert 8th case %s"%rows)
assert rows[0][0] == 3, ' 8th case is failed'
# #query: selector Functions 9
# queryparam=new_bind_params(1)
# queryparam[0].int(2)
# rows=self.stmtExe(conn,"select bottom(bu,?) from log group by bu ; ",queryparam)
# tdLog.debug("assert 9th case %s"%rows)
# assert rows[0][0] == 4, ' 9 case is failed'
# assert rows[1][0] == 3, ' 9 case is failed'
#query: selector Functions 9
queryparam=new_bind_params(1)
queryparam[0].int(2)
rows=self.stmtExe(conn,"select bottom(bu,?) from log group by bu order by bu desc ; ",queryparam)
tdLog.debug("assert 9th case %s"%rows)
assert rows[1][0] == 4, ' 9 case is failed'
assert rows[2][0] == 3, ' 9 case is failed'
# #query: time-series specific Functions 10

545
tests/system-test/2-query/and_or_for_byte.py Normal file
View File

@ -0,0 +1,545 @@
import taos
import sys
import datetime
import inspect
from util.log import *
from util.sql import *
from util.cases import *
import random
class TDTestCase:
updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143,
"jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143,
"wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.tb_nums = 10
self.row_nums = 20
self.ts = 1434938400000
self.time_step = 1000
def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ):
tdLog.info(" prepare datas for auto check abs function ")
tdSql.execute(" create database test ")
tdSql.execute(" use test ")
tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\
c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)")
for tbnum in range(tbnums):
tbname = "sub_tb_%d"%tbnum
tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum))
ts = self.ts
for row in range(rownums):
ts = self.ts + time_step*row
c1 = random.randint(0,10000)
c2 = random.randint(0,100000)
c3 = random.randint(0,125)
c4 = random.randint(0,125)
c5 = random.random()/1.0
c6 = random.random()/1.0
c7 = "'true'"
c8 = "'binary_val'"
c9 = "'nchar_val'"
c10 = ts
tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})")
tdSql.execute("use test")
tbnames = ["stb", "sub_tb_1"]
support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
for tbname in tbnames:
tdSql.query("desc {}".format(tbname))
coltypes = tdSql.queryResult
colnames = []
for coltype in coltypes:
colname = coltype[0]
if coltype[1] in support_types:
colnames.append(colname)
cols = random.sample(colnames,3)
self.check_function("&",False,tbname,cols[0],cols[1],cols[2])
self.check_function("|",False,tbname,cols[0],cols[1],cols[2])
def prepare_datas(self):
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def prepare_tag_datas(self):
# prepare datas
tdSql.execute(
"create database if not exists testdb keep 3650 duration 1000")
tdSql.execute(" use testdb ")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32))
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(
f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def check_result_auto(self, origin_query, abs_query):
abs_result = tdSql.getResult(abs_query)
origin_result = tdSql.getResult(origin_query)
auto_result = []
for row in origin_result:
row_check = []
for elem in row:
if elem == None:
elem = None
elif elem >= 0:
elem = elem
else:
elem = -elem
row_check.append(elem)
auto_result.append(row_check)
check_status = True
for row_index, row in enumerate(abs_result):
for col_index, elem in enumerate(row):
if auto_result[row_index][col_index] != elem:
check_status = False
if not check_status:
tdLog.notice(
"abs function value has not as expected , sql is \"%s\" " % abs_query)
sys.exit(1)
else:
tdLog.info(
"abs value check pass , it work as expected ,sql is \"%s\" " % abs_query)
def check_function(self, opera ,agg, tbname , *args):
if opera =="&":
pass
elif opera =="|":
pass
else:
pass
work_sql = " select "
for ind , arg in enumerate(args):
if ind ==len(args)-1:
work_sql += f"cast({arg} as bigint) "
else:
work_sql += f"cast({arg} as bigint){opera}"
if not agg:
work_sql+= f" from {tbname} order by ts"
else:
work_sql+= f" from {tbname} "
tdSql.query(work_sql)
work_result = tdSql.queryResult
origin_sql = " select "
for ind , arg in enumerate(args):
if ind ==len(args)-1:
origin_sql += f"cast({arg} as bigint) "
else:
origin_sql += f"cast({arg} as bigint),"
if not agg:
origin_sql+= f" from {tbname} order by ts"
else:
origin_sql+= f" from {tbname} "
tdSql.query(origin_sql)
origin_result = tdSql.queryResult
# compute and or with byte in binary data
compute_result = []
for row in origin_result:
if None in row:
compute_result.append(None)
else:
if opera == "&":
result = row[0]
for elem in row:
result = result&elem
elif opera == "|":
result = row[0]
for elem in row:
result = result|elem
compute_result.append(result)
tdSql.query(work_sql)
for ind , result in enumerate(compute_result):
tdSql.checkData(ind,0,result)
def test_errors(self):
tdSql.execute("use testdb")
error_sql_lists = [
"select c1&&c2 from t1",
"select c1&|c2 from t1",
"select c1&(c1=c2) from t1",
"select c1&* from t1",
"select 123&, from t1",
"select 123&\" from t1",
"select c1&- from t1;",
"select c1&&= from t1)",
"select c1&! from t1",
"select c1&@ from stb1",
"select c1&# from stb1",
"select c1&$ from stb1",
"select c1&% from stb1",
"select c1&() from stb1",
]
for error_sql in error_sql_lists:
tdSql.error(error_sql)
def basic_query(self):
# basic query
tdSql.query("select c1&c2|c3 from ct1")
tdSql.checkRows(13)
tdSql.query("select c1 ,c2&c3, c1&c2&c3 from t1")
tdSql.checkRows(12)
tdSql.query("select c1 ,c1&c1&c1|c1 from stb1")
tdSql.checkRows(25)
# used for empty table , ct3 is empty
tdSql.query("select abs(c1)&c2&c3 from ct3")
tdSql.checkRows(0)
tdSql.query("select abs(c2&c1&c3) from ct3")
tdSql.checkRows(0)
tdSql.query("select abs(c3)+c1&c3+c2 from ct3")
tdSql.checkRows(0)
tdSql.query("select abs(c1)&c2&c3 from ct4")
tdSql.checkRows(12)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,8)
tdSql.checkData(10,0,0)
tdSql.query("select abs(c2&c1&c3) from ct4")
tdSql.checkRows(12)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,8)
tdSql.checkData(10,0,0)
tdSql.query("select (abs(c3)+c1)&(c3+c2) from ct4")
tdSql.checkRows(12)
tdSql.checkData(0,0,None)
tdSql.checkData(1,0,640)
tdSql.checkData(10,0,0)
# used for regular table
tdSql.query("select abs(c1)&c3&c3 from t1")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, 1)
tdSql.checkData(3, 0, 1)
tdSql.checkData(5, 0, None)
tdSql.query("select abs(c1)&c2|ceil(c3)&c4|floor(c5) from t1")
tdSql.checkData(1, 0, 11)
tdSql.checkData(3, 0, 3)
tdSql.checkData(5, 0, None)
tdSql.query("select ts,c1, c2, c3&c4|c5 from t1")
tdSql.checkData(1, 3, 11)
tdSql.checkData(3, 3, 3)
tdSql.checkData(5, 3, None)
self.check_function("&",False,"stb1","c1","ceil(c2)","abs(c3)","c4+1")
self.check_function("|",False,"stb1","c1","ceil(c2)","abs(c3)","c4+1")
self.check_function("&",False,"stb1","c1+c2","ceil(c2)","abs(c3+c2)","c4+1")
self.check_function("&",False,"ct4","123","ceil(c2)","abs(c3+c2)","c4+1")
self.check_function("&",False,"ct4","123","ceil(t1)","abs(c3+c2)","c4+1")
self.check_function("&",False,"ct4","t1+c1","-ceil(t1)","abs(c3+c2)","c4+1")
self.check_function("&",False,"stb1","c1","floor(t1)","abs(c1+c2)","t1+1")
self.check_function("&",True,"stb1","max(c1)","min(floor(t1))","sum(abs(c1+c2))","last(t1)+1")
self.check_function("&",False,"stb1","abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))))","floor(t1)","abs(c1+c2)","t1+1")
# mix with common col
tdSql.query("select c1&abs(c1)&c2&c3 ,c1,c2, t1 from ct1")
tdSql.checkData(0, 0, 8)
tdSql.checkData(1, 0, 1)
tdSql.checkData(4, 0, 0)
tdSql.checkData(4, 3, 0)
tdSql.checkData(3, 2, 55555)
# mix with common functions
tdSql.query(" select c1&abs(c1)&c2&c3, abs(c1), c5, floor(c5) from ct4 ")
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
tdSql.checkData(0, 3, None)
tdSql.checkData(3, 0, 2)
tdSql.checkData(3, 1, 6)
tdSql.checkData(3, 2, 6.66000)
tdSql.checkData(3, 3, 6.00000)
tdSql.query("select c1&abs(c1)&c2&c3, abs(c1),c5, floor(c5) from stb1 order by ts ")
tdSql.checkData(3, 0, 2)
tdSql.checkData(3, 1, 6)
tdSql.checkData(3, 2, 6.66000)
tdSql.checkData(3, 3, 6.00000)
# mix with agg functions , not support
tdSql.error("select c1&abs(c1)&c2&c3, abs(c1),c5, count(c5) from stb1 ")
tdSql.error("select c1&abs(c1)&c2&c3, abs(c1),c5, count(c5) from ct1 ")
tdSql.error("select c1&abs(c1)&c2&c3, count(c5) from stb1 ")
tdSql.error("select c1&abs(c1)&c2&c3, count(c5) from ct1 ")
tdSql.error("select c1&abs(c1)&c2&c3, count(c5) from ct1 ")
tdSql.error("select c1&abs(c1)&c2&c3, count(c5) from stb1 ")
# agg functions mix with agg functions
tdSql.query("select sum(c1&abs(c1)&c2&c3) ,max(c5), count(c5) from stb1")
tdSql.query("select max(c1)&max(c2)|first(ts), count(c5) from ct1")
# bug fix for compute
tdSql.query("select c1&abs(c1)&c2&c3, abs(c1&abs(c1)&c2&c3) -0 ,ceil(c1&abs(c1)&c2&c3)-0 from ct4 ")
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 8)
tdSql.checkData(1, 1, 8.000000000)
tdSql.checkData(1, 2, 8.000000000)
tdSql.query(" select c1&c2|c3, abs(c1&c2|c3) -0 ,ceil(c1&c2|c3-0.1)-0.1 from ct4")
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 888)
tdSql.checkData(1, 1, 888.000000000)
tdSql.checkData(1, 2, 894.900000000)
def check_boundary_values(self):
tdSql.execute("drop database if exists bound_test")
tdSql.execute("create database if not exists bound_test")
time.sleep(3)
tdSql.execute("use bound_test")
tdSql.execute(
"create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);"
)
tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )')
tdSql.execute(
f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.execute(
f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
tdSql.error(
f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
)
self.check_function("&", False , "sub1_bound" ,"c1","c2","c3","c4","c5","c6" )
self.check_function("&", False ,"sub1_bound","abs(c1)","abs(c2)","abs(c3)","abs(c4)","abs(c5)","abs(c6)" )
self.check_function("&", False ,"stb_bound","123","abs(c2)","t1","abs(c4)","abs(c5)","abs(c6)" )
# check basic elem for table per row
tdSql.query(
"select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ")
tdSql.checkData(0, 0, 2147483647)
tdSql.checkData(0, 1, 9223372036854775807)
tdSql.checkData(0, 2, 32767)
tdSql.checkData(0, 3, 127)
tdSql.checkData(0, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(1, 0, 2147483647)
tdSql.checkData(1, 1, 9223372036854775807)
tdSql.checkData(1, 2, 32767)
tdSql.checkData(1, 3, 127)
tdSql.checkData(1, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(1, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(3, 0, 2147483646)
tdSql.checkData(3, 1, 9223372036854775806)
tdSql.checkData(3, 2, 32766)
tdSql.checkData(3, 3, 126)
tdSql.checkData(3, 4, 339999995214436424907732413799364296704.00000)
tdSql.checkData(3, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
# check + - * / in functions
self.check_function("&", False ,"stb_bound","abs(c1+1)","abs(c2)","t1","abs(c3*1)","abs(c5)/2","abs(c6)" )
tdSql.query(
"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
tdSql.checkData(0, 0, 2147483648.000000000)
tdSql.checkData(0, 1, 9223372036854775807)
tdSql.checkData(0, 2, 32767.000000000)
tdSql.checkData(0, 3, 63.500000000)
tdSql.checkData(
0, 4, 169999997607218212453866206899682148352.000000000)
tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000)
tdSql.checkData(1, 0, 2147483646.000000000)
tdSql.checkData(1, 1, 9223372036854775808.000000000)
tdSql.checkData(1, 2, 32767.000000000)
tdSql.checkData(1, 3, 63.500000000)
tdSql.checkData(
1, 4, 169999997607218212453866206899682148352.000000000)
def test_tag_compute_for_scalar_function(self):
tdSql.execute("use testdb")
self.check_function("&", False ,"ct4","123","abs(c1)","t1","abs(t2)","abs(t3)","abs(t4)","t5")
self.check_function("&", False ,"ct4","c1+2","abs(t2+2)","t3","abs(t4)","abs(t5)","abs(c1)","t5")
tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ")
tdSql.query("select c1 ,t1 from stb1 where t1 =0 ")
tdSql.checkRows(13)
self.check_function("&", False ,"t1","c1+2","abs(c2)")
tdSql.query("select t1 from stb1 where t1 >0 ")
tdSql.checkRows(3)
tdSql.query("select t1 from stb1 where t1 =3 ")
tdSql.checkRows(1)
# tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)")
# tdSql.checkData(0,0,61)
# tdSql.query("select distinct(c1) ,t1 from stb1")
# tdSql.checkRows(20)
tdSql.query("select max(c1) , t1&c2&t2 from stb1;")
tdSql.checkData(0,1,0)
# tag filter with abs function
tdSql.query("select t1 from stb1 where abs(t1)=1")
tdSql.checkRows(1)
tdSql.query("select t1 from stb1 where abs(c1+t1)=1")
tdSql.checkRows(1)
tdSql.checkData(0,0,0)
tdSql.query(
"select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
def support_super_table_test(self):
tdSql.execute(" use testdb ")
self.check_function("|", False , "stb1" , "c1","c2","c3","c4" )
self.check_function("|", False , "stb1" , "c1","c2","abs(c3)","c4","ceil(t1)" )
self.check_function("&", False , "stb1" , "c1","c2","abs(c3)","floor(c4)","ceil(t1)" )
self.check_function("&", True , "stb1" , "max(c1)","max(c2)","sum(abs(c3))","max(floor(c4))","min(ceil(t1))" )
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
self.prepare_datas()
self.prepare_tag_datas()
self.test_errors()
self.basic_query()
self.check_boundary_values()
self.test_tag_compute_for_scalar_function()
self.support_super_table_test()
self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

54
tests/system-test/2-query/csum.py
View File

@ -419,12 +419,66 @@ class TDTestCase:
tdSql.checkData(3,0,4)
tdSql.query("select csum(abs(c1))+2 from t1 ")
tdSql.checkRows(4)
def csum_support_stable(self):
tdSql.query(" select csum(1) from stb1 ")
tdSql.checkRows(70)
tdSql.query("select csum(c1) from stb1 partition by tbname ")
tdSql.checkRows(40)
# tdSql.query("select csum(st1) from stb1 partition by tbname")
# tdSql.checkRows(70)
tdSql.query("select csum(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(40)
tdSql.query("select csum(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(40)
tdSql.query("select csum(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(40)
# # bug need fix
# tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
# tdSql.checkRows(4)
# tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ")
# bug need fix
tdSql.query("select csum(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(40)
# bug need fix
# tdSql.query("select tbname , csum(c1) from stb1 partition by tbname")
# tdSql.checkRows(40)
# tdSql.query("select tbname , csum(st1) from stb1 partition by tbname")
# tdSql.checkRows(70)
# tdSql.query("select tbname , csum(st1) from stb1 partition by tbname slimit 1")
# tdSql.checkRows(7)
# partition by tags
# tdSql.query("select st1 , csum(c1) from stb1 partition by st1")
# tdSql.checkRows(40)
# tdSql.query("select csum(c1) from stb1 partition by st1")
# tdSql.checkRows(40)
# tdSql.query("select st1 , csum(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(4)
# tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(4)
# partition by col
# tdSql.query("select c1 , csum(c1) from stb1 partition by c1")
# tdSql.checkRows(41)
# tdSql.query("select csum(c1) from stb1 partition by c1")
# tdSql.checkRows(41)
# tdSql.query("select c1 , csum(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(4)
# tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(4)
def run(self):
import traceback
try:
# run in develop branch
self.csum_test_run()
self.csum_support_stable()
pass
except Exception as e:
traceback.print_exc()

885
tests/system-test/2-query/elapsed.py
View File

File diff suppressed because it is too large Load Diff

57
tests/system-test/2-query/function_diff.py
View File

@ -355,9 +355,63 @@ class TDTestCase:
tdSql.execute(f"create table tt{i} using stb2 tags({i})")
pass
def diff_support_stable(self):
tdSql.query(" select diff(1) from stb1 ")
tdSql.checkRows(229)
tdSql.checkData(0,0,0)
tdSql.query("select diff(c1) from stb1 partition by tbname ")
tdSql.checkRows(199)
# tdSql.query("select diff(st1) from stb1 partition by tbname")
# tdSql.checkRows(229)
tdSql.query("select diff(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(199)
tdSql.query("select diff(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(199)
tdSql.query("select diff(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(199)
# # bug need fix
# tdSql.query("select diff(st1+c1) from stb1 partition by tbname slimit 1 ")
# tdSql.checkRows(19)
# tdSql.error("select diff(st1+c1) from stb1 partition by tbname limit 1 ")
# bug need fix
tdSql.query("select diff(st1+c1) from stb1 partition by tbname")
tdSql.checkRows(199)
# bug need fix
# tdSql.query("select tbname , diff(c1) from stb1 partition by tbname")
# tdSql.checkRows(199)
# tdSql.query("select tbname , diff(st1) from stb1 partition by tbname")
# tdSql.checkRows(199)
# tdSql.query("select tbname , diff(st1) from stb1 partition by tbname slimit 1")
# tdSql.checkRows(19)
# partition by tags
# tdSql.query("select st1 , diff(c1) from stb1 partition by st1")
# tdSql.checkRows(199)
# tdSql.query("select diff(c1) from stb1 partition by st1")
# tdSql.checkRows(199)
# tdSql.query("select st1 , diff(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(19)
# tdSql.query("select diff(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(19)
# partition by col
# tdSql.query("select c1 , diff(c1) from stb1 partition by c1")
# tdSql.checkRows(199)
# tdSql.query("select diff(c1) from stb1 partition by c1")
# tdSql.checkRows(41)
# tdSql.query("select c1 , diff(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(19)
# tdSql.query("select diff(c1) from stb1 partition by st1 slimit 1")
# tdSql.checkRows(19)
def diff_test_run(self) :
tdLog.printNoPrefix("==========TD-10594==========")
tdLog.printNoPrefix("==========run test case for diff function==========")
tbnum = 10
nowtime = int(round(time.time() * 1000))
per_table_rows = 10
@ -422,6 +476,7 @@ class TDTestCase:
try:
# run in develop branch
self.diff_test_run()
self.diff_support_stable()
pass
except Exception as e:
traceback.print_exc()

Some files were not shown because too many files have changed in this diff Show More