xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

This commit is contained in:
Minghao Li 2022-06-29 17:08:22 +08:00
parent 3440822ebf 1642cf0e50
commit 108530c1b6
372 changed files with 8971 additions and 41457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmake/cmake.install
View File

@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg)
INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include)
INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include)
INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)

2
docs/examples/java/src/main/java/com/taos/example/LineProtocolExample.java
View File

@ -26,7 +26,7 @@ public class LineProtocolExample {
private static void createDatabase(Connection conn) throws SQLException {
try (Statement stmt = conn.createStatement()) {
// the default precision is ms (microsecond), but we use us(microsecond) here.
// the default precision is ms (millisecond), but we use us(microsecond) here.
stmt.execute("CREATE DATABASE IF NOT EXISTS test PRECISION 'us'");
stmt.execute("USE test");
}

494
include/common/ttokendef.h
View File

@ -16,251 +16,255 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_ENABLE 34
#define TK_NK_INTEGER 35
#define TK_SYSINFO 36
#define TK_DROP 37
#define TK_GRANT 38
#define TK_ON 39
#define TK_TO 40
#define TK_REVOKE 41
#define TK_FROM 42
#define TK_NK_COMMA 43
#define TK_READ 44
#define TK_WRITE 45
#define TK_NK_DOT 46
#define TK_DNODE 47
#define TK_PORT 48
#define TK_DNODES 49
#define TK_NK_IPTOKEN 50
#define TK_LOCAL 51
#define TK_QNODE 52
#define TK_BNODE 53
#define TK_SNODE 54
#define TK_MNODE 55
#define TK_DATABASE 56
#define TK_USE 57
#define TK_IF 58
#define TK_NOT 59
#define TK_EXISTS 60
#define TK_BUFFER 61
#define TK_CACHELAST 62
#define TK_COMP 63
#define TK_DURATION 64
#define TK_NK_VARIABLE 65
#define TK_FSYNC 66
#define TK_MAXROWS 67
#define TK_MINROWS 68
#define TK_KEEP 69
#define TK_PAGES 70
#define TK_PAGESIZE 71
#define TK_PRECISION 72
#define TK_REPLICA 73
#define TK_STRICT 74
#define TK_WAL 75
#define TK_VGROUPS 76
#define TK_SINGLE_STABLE 77
#define TK_RETENTIONS 78
#define TK_SCHEMALESS 79
#define TK_NK_COLON 80
#define TK_TABLE 81
#define TK_NK_LP 82
#define TK_NK_RP 83
#define TK_STABLE 84
#define TK_ADD 85
#define TK_COLUMN 86
#define TK_MODIFY 87
#define TK_RENAME 88
#define TK_TAG 89
#define TK_SET 90
#define TK_NK_EQ 91
#define TK_USING 92
#define TK_TAGS 93
#define TK_COMMENT 94
#define TK_BOOL 95
#define TK_TINYINT 96
#define TK_SMALLINT 97
#define TK_INT 98
#define TK_INTEGER 99
#define TK_BIGINT 100
#define TK_FLOAT 101
#define TK_DOUBLE 102
#define TK_BINARY 103
#define TK_TIMESTAMP 104
#define TK_NCHAR 105
#define TK_UNSIGNED 106
#define TK_JSON 107
#define TK_VARCHAR 108
#define TK_MEDIUMBLOB 109
#define TK_BLOB 110
#define TK_VARBINARY 111
#define TK_DECIMAL 112
#define TK_MAX_DELAY 113
#define TK_WATERMARK 114
#define TK_ROLLUP 115
#define TK_TTL 116
#define TK_SMA 117
#define TK_FIRST 118
#define TK_LAST 119
#define TK_SHOW 120
#define TK_DATABASES 121
#define TK_TABLES 122
#define TK_STABLES 123
#define TK_MNODES 124
#define TK_MODULES 125
#define TK_QNODES 126
#define TK_FUNCTIONS 127
#define TK_INDEXES 128
#define TK_ACCOUNTS 129
#define TK_APPS 130
#define TK_CONNECTIONS 131
#define TK_LICENCE 132
#define TK_GRANTS 133
#define TK_QUERIES 134
#define TK_SCORES 135
#define TK_TOPICS 136
#define TK_VARIABLES 137
#define TK_BNODES 138
#define TK_SNODES 139
#define TK_CLUSTER 140
#define TK_TRANSACTIONS 141
#define TK_DISTRIBUTED 142
#define TK_CONSUMERS 143
#define TK_SUBSCRIPTIONS 144
#define TK_LIKE 145
#define TK_INDEX 146
#define TK_FUNCTION 147
#define TK_INTERVAL 148
#define TK_TOPIC 149
#define TK_AS 150
#define TK_WITH 151
#define TK_META 152
#define TK_CONSUMER 153
#define TK_GROUP 154
#define TK_DESC 155
#define TK_DESCRIBE 156
#define TK_RESET 157
#define TK_QUERY 158
#define TK_CACHE 159
#define TK_EXPLAIN 160
#define TK_ANALYZE 161
#define TK_VERBOSE 162
#define TK_NK_BOOL 163
#define TK_RATIO 164
#define TK_NK_FLOAT 165
#define TK_COMPACT 166
#define TK_VNODES 167
#define TK_IN 168
#define TK_OUTPUTTYPE 169
#define TK_AGGREGATE 170
#define TK_BUFSIZE 171
#define TK_STREAM 172
#define TK_INTO 173
#define TK_TRIGGER 174
#define TK_AT_ONCE 175
#define TK_WINDOW_CLOSE 176
#define TK_KILL 177
#define TK_CONNECTION 178
#define TK_TRANSACTION 179
#define TK_BALANCE 180
#define TK_VGROUP 181
#define TK_MERGE 182
#define TK_REDISTRIBUTE 183
#define TK_SPLIT 184
#define TK_SYNCDB 185
#define TK_DELETE 186
#define TK_NULL 187
#define TK_NK_QUESTION 188
#define TK_NK_ARROW 189
#define TK_ROWTS 190
#define TK_TBNAME 191
#define TK_QSTARTTS 192
#define TK_QENDTS 193
#define TK_WSTARTTS 194
#define TK_WENDTS 195
#define TK_WDURATION 196
#define TK_CAST 197
#define TK_NOW 198
#define TK_TODAY 199
#define TK_TIMEZONE 200
#define TK_COUNT 201
#define TK_LAST_ROW 202
#define TK_BETWEEN 203
#define TK_IS 204
#define TK_NK_LT 205
#define TK_NK_GT 206
#define TK_NK_LE 207
#define TK_NK_GE 208
#define TK_NK_NE 209
#define TK_MATCH 210
#define TK_NMATCH 211
#define TK_CONTAINS 212
#define TK_JOIN 213
#define TK_INNER 214
#define TK_SELECT 215
#define TK_DISTINCT 216
#define TK_WHERE 217
#define TK_PARTITION 218
#define TK_BY 219
#define TK_SESSION 220
#define TK_STATE_WINDOW 221
#define TK_SLIDING 222
#define TK_FILL 223
#define TK_VALUE 224
#define TK_NONE 225
#define TK_PREV 226
#define TK_LINEAR 227
#define TK_NEXT 228
#define TK_HAVING 229
#define TK_RANGE 230
#define TK_EVERY 231
#define TK_ORDER 232
#define TK_SLIMIT 233
#define TK_SOFFSET 234
#define TK_LIMIT 235
#define TK_OFFSET 236
#define TK_ASC 237
#define TK_NULLS 238
#define TK_ID 239
#define TK_NK_BITNOT 240
#define TK_INSERT 241
#define TK_VALUES 242
#define TK_IMPORT 243
#define TK_NK_SEMI 244
#define TK_FILE 245
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_ENABLE 34
#define TK_NK_INTEGER 35
#define TK_SYSINFO 36
#define TK_DROP 37
#define TK_GRANT 38
#define TK_ON 39
#define TK_TO 40
#define TK_REVOKE 41
#define TK_FROM 42
#define TK_NK_COMMA 43
#define TK_READ 44
#define TK_WRITE 45
#define TK_NK_DOT 46
#define TK_DNODE 47
#define TK_PORT 48
#define TK_DNODES 49
#define TK_NK_IPTOKEN 50
#define TK_LOCAL 51
#define TK_QNODE 52
#define TK_BNODE 53
#define TK_SNODE 54
#define TK_MNODE 55
#define TK_DATABASE 56
#define TK_USE 57
#define TK_IF 58
#define TK_NOT 59
#define TK_EXISTS 60
#define TK_BUFFER 61
#define TK_CACHELAST 62
#define TK_COMP 63
#define TK_DURATION 64
#define TK_NK_VARIABLE 65
#define TK_FSYNC 66
#define TK_MAXROWS 67
#define TK_MINROWS 68
#define TK_KEEP 69
#define TK_PAGES 70
#define TK_PAGESIZE 71
#define TK_PRECISION 72
#define TK_REPLICA 73
#define TK_STRICT 74
#define TK_WAL 75
#define TK_VGROUPS 76
#define TK_SINGLE_STABLE 77
#define TK_RETENTIONS 78
#define TK_SCHEMALESS 79
#define TK_NK_COLON 80
#define TK_TABLE 81
#define TK_NK_LP 82
#define TK_NK_RP 83
#define TK_STABLE 84
#define TK_ADD 85
#define TK_COLUMN 86
#define TK_MODIFY 87
#define TK_RENAME 88
#define TK_TAG 89
#define TK_SET 90
#define TK_NK_EQ 91
#define TK_USING 92
#define TK_TAGS 93
#define TK_COMMENT 94
#define TK_BOOL 95
#define TK_TINYINT 96
#define TK_SMALLINT 97
#define TK_INT 98
#define TK_INTEGER 99
#define TK_BIGINT 100
#define TK_FLOAT 101
#define TK_DOUBLE 102
#define TK_BINARY 103
#define TK_TIMESTAMP 104
#define TK_NCHAR 105
#define TK_UNSIGNED 106
#define TK_JSON 107
#define TK_VARCHAR 108
#define TK_MEDIUMBLOB 109
#define TK_BLOB 110
#define TK_VARBINARY 111
#define TK_DECIMAL 112
#define TK_MAX_DELAY 113
#define TK_WATERMARK 114
#define TK_ROLLUP 115
#define TK_TTL 116
#define TK_SMA 117
#define TK_FIRST 118
#define TK_LAST 119
#define TK_SHOW 120
#define TK_DATABASES 121
#define TK_TABLES 122
#define TK_STABLES 123
#define TK_MNODES 124
#define TK_MODULES 125
#define TK_QNODES 126
#define TK_FUNCTIONS 127
#define TK_INDEXES 128
#define TK_ACCOUNTS 129
#define TK_APPS 130
#define TK_CONNECTIONS 131
#define TK_LICENCE 132
#define TK_GRANTS 133
#define TK_QUERIES 134
#define TK_SCORES 135
#define TK_TOPICS 136
#define TK_VARIABLES 137
#define TK_BNODES 138
#define TK_SNODES 139
#define TK_CLUSTER 140
#define TK_TRANSACTIONS 141
#define TK_DISTRIBUTED 142
#define TK_CONSUMERS 143
#define TK_SUBSCRIPTIONS 144
#define TK_LIKE 145
#define TK_INDEX 146
#define TK_FUNCTION 147
#define TK_INTERVAL 148
#define TK_TOPIC 149
#define TK_AS 150
#define TK_WITH 151
#define TK_META 152
#define TK_CONSUMER 153
#define TK_GROUP 154
#define TK_DESC 155
#define TK_DESCRIBE 156
#define TK_RESET 157
#define TK_QUERY 158
#define TK_CACHE 159
#define TK_EXPLAIN 160
#define TK_ANALYZE 161
#define TK_VERBOSE 162
#define TK_NK_BOOL 163
#define TK_RATIO 164
#define TK_NK_FLOAT 165
#define TK_COMPACT 166
#define TK_VNODES 167
#define TK_IN 168
#define TK_OUTPUTTYPE 169
#define TK_AGGREGATE 170
#define TK_BUFSIZE 171
#define TK_STREAM 172
#define TK_INTO 173
#define TK_TRIGGER 174
#define TK_AT_ONCE 175
#define TK_WINDOW_CLOSE 176
#define TK_KILL 177
#define TK_CONNECTION 178
#define TK_TRANSACTION 179
#define TK_BALANCE 180
#define TK_VGROUP 181
#define TK_MERGE 182
#define TK_REDISTRIBUTE 183
#define TK_SPLIT 184
#define TK_SYNCDB 185
#define TK_DELETE 186
#define TK_NULL 187
#define TK_NK_QUESTION 188
#define TK_NK_ARROW 189
#define TK_ROWTS 190
#define TK_TBNAME 191
#define TK_QSTARTTS 192
#define TK_QENDTS 193
#define TK_WSTARTTS 194
#define TK_WENDTS 195
#define TK_WDURATION 196
#define TK_CAST 197
#define TK_NOW 198
#define TK_TODAY 199
#define TK_TIMEZONE 200
#define TK_CLIENT_VERSION 201
#define TK_SERVER_VERSION 202
#define TK_SERVER_STATUS 203
#define TK_CURRENT_USER 204
#define TK_COUNT 205
#define TK_LAST_ROW 206
#define TK_BETWEEN 207
#define TK_IS 208
#define TK_NK_LT 209
#define TK_NK_GT 210
#define TK_NK_LE 211
#define TK_NK_GE 212
#define TK_NK_NE 213
#define TK_MATCH 214
#define TK_NMATCH 215
#define TK_CONTAINS 216
#define TK_JOIN 217
#define TK_INNER 218
#define TK_SELECT 219
#define TK_DISTINCT 220
#define TK_WHERE 221
#define TK_PARTITION 222
#define TK_BY 223
#define TK_SESSION 224
#define TK_STATE_WINDOW 225
#define TK_SLIDING 226
#define TK_FILL 227
#define TK_VALUE 228
#define TK_NONE 229
#define TK_PREV 230
#define TK_LINEAR 231
#define TK_NEXT 232
#define TK_HAVING 233
#define TK_RANGE 234
#define TK_EVERY 235
#define TK_ORDER 236
#define TK_SLIMIT 237
#define TK_SOFFSET 238
#define TK_LIMIT 239
#define TK_OFFSET 240
#define TK_ASC 241
#define TK_NULLS 242
#define TK_ID 243
#define TK_NK_BITNOT 244
#define TK_INSERT 245
#define TK_VALUES 246
#define TK_IMPORT 247
#define TK_NK_SEMI 248
#define TK_FILE 249
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

8
include/libs/catalog/catalog.h
View File

@ -147,14 +147,6 @@ int32_t catalogInit(SCatalogCfg* cfg);
*/
int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
/**
* Free a cluster's all catalog info, usually it's not necessary, until the application is closing.
* no current or future usage should be guaranteed by application
* @param pCatalog (input, NO more usage)
* @return error code
*/
void catalogFreeHandle(SCatalog* pCatalog);
int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum);
/**

16
include/libs/executor/executor.h
View File

@ -36,7 +36,6 @@ typedef struct SReadHandle {
void* vnode;
void* mnd;
SMsgCb* pMsgCb;
// int8_t initTsdbReader;
} SReadHandle;
enum {
@ -140,12 +139,6 @@ int32_t qKillTask(qTaskInfo_t tinfo);
*/
int32_t qAsyncKillTask(qTaskInfo_t tinfo);
/**
* return whether query is completed or not
* @param tinfo
* @return
*/
int32_t qIsTaskCompleted(qTaskInfo_t tinfo);
/**
* destroy query info structure
@ -176,6 +169,15 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
/**
* return the scan info, in the form of tuple of two items, including table uid and current timestamp
* @param tinfo
* @param uid
* @param ts
* @return
*/
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
#ifdef __cplusplus
}
#endif

3
include/libs/function/functionMgt.h
View File

@ -105,7 +105,7 @@ typedef enum EFunctionType {
// system function
FUNCTION_TYPE_DATABASE = 3000,
FUNCTION_TYPE_CLIENT_VERSION,
FUNCTION_TYPE_SERVER_SERSION,
FUNCTION_TYPE_SERVER_VERSION,
FUNCTION_TYPE_SERVER_STATUS,
FUNCTION_TYPE_CURRENT_USER,
FUNCTION_TYPE_USER,
@ -193,6 +193,7 @@ bool fmIsForbidGroupByFunc(int32_t funcId);
bool fmIsIntervalInterpoFunc(int32_t funcId);
bool fmIsInterpFunc(int32_t funcId);
bool fmIsLastRowFunc(int32_t funcId);
bool fmIsSystemInfoFunc(int32_t funcId);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);

266
include/libs/function/taosudf.h Normal file
View File

@ -0,0 +1,266 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TAOSUDF_H
#define TDENGINE_TAOSUDF_H
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <taos.h>
#include <taoserror.h>
#ifdef __cplusplus
extern "C" {
#endif
#if defined(__GNUC__)
#define FORCE_INLINE inline __attribute__((always_inline))
#else
#define FORCE_INLINE
#endif
typedef struct SUdfColumnMeta {
int16_t type;
int32_t bytes;
uint8_t precision;
uint8_t scale;
} SUdfColumnMeta;
typedef struct SUdfColumnData {
int32_t numOfRows;
int32_t rowsAlloc;
union {
struct {
int32_t nullBitmapLen;
char *nullBitmap;
int32_t dataLen;
char *data;
} fixLenCol;
struct {
int32_t varOffsetsLen;
int32_t *varOffsets;
int32_t payloadLen;
char *payload;
int32_t payloadAllocLen;
} varLenCol;
};
} SUdfColumnData;
typedef struct SUdfColumn {
SUdfColumnMeta colMeta;
bool hasNull;
SUdfColumnData colData;
} SUdfColumn;
typedef struct SUdfDataBlock {
int32_t numOfRows;
int32_t numOfCols;
SUdfColumn **udfCols;
} SUdfDataBlock;
typedef struct SUdfInterBuf {
int32_t bufLen;
char* buf;
int8_t numOfResult; //zero or one
} SUdfInterBuf;
typedef void *UdfcFuncHandle;
// dynamic lib init and destroy
typedef int32_t (*TUdfInitFunc)();
typedef int32_t (*TUdfDestroyFunc)();
#define UDF_MEMORY_EXP_GROWTH 1.5
#define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1)
#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row))))
#define udfColDataSetNull_f(pColumn, row) \
do { \
BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \
} while (0)
#define udfColDataSetNotNull_f(pColumn, r_) \
do { \
BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \
} while (0)
#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1)
typedef uint16_t VarDataLenT; // maxVarDataLen: 32767
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) \
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) {
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row];
} else {
return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row;
}
}
static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) {
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) {
if (udfColDataIsNull_var(pColumn, row)) {
return true;
}
char* data = udfColDataGetData(pColumn, row);
return (*data == TSDB_DATA_TYPE_NULL);
} else {
return udfColDataIsNull_var(pColumn, row);
}
} else {
return udfColDataIsNull_f(pColumn, row);
}
}
static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) {
SUdfColumnMeta *meta = &pColumn->colMeta;
SUdfColumnData *data = &pColumn->colData;
if (newCapacity== 0 || newCapacity <= data->rowsAlloc) {
return TSDB_CODE_SUCCESS;
}
int allocCapacity = (data->rowsAlloc< 8) ? 8 : data->rowsAlloc;
while (allocCapacity < newCapacity) {
allocCapacity *= UDF_MEMORY_EXP_GROWTH;
}
if (IS_VAR_DATA_TYPE(meta->type)) {
char* tmp = (char*)realloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->varLenCol.varOffsets = (int32_t*)tmp;
data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity;
// for payload, add data in udfColDataAppend
} else {
char* tmp = (char*)realloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->fixLenCol.nullBitmap = tmp;
data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity);
if (meta->type == TSDB_DATA_TYPE_NULL) {
return TSDB_CODE_SUCCESS;
}
tmp = (char*)realloc(data->fixLenCol.data, allocCapacity* meta->bytes);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->fixLenCol.data = tmp;
data->fixLenCol.dataLen = allocCapacity* meta->bytes;
}
data->rowsAlloc = allocCapacity;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) {
udfColEnsureCapacity(pColumn, row+1);
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
udfColDataSetNull_var(pColumn, row);
} else {
udfColDataSetNull_f(pColumn, row);
}
pColumn->hasNull = true;
}
static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) {
SUdfColumnMeta *meta = &pColumn->colMeta;
SUdfColumnData *data = &pColumn->colData;
udfColEnsureCapacity(pColumn, currentRow+1);
bool isVarCol = IS_VAR_DATA_TYPE(meta->type);
if (isNull) {
udfColDataSetNull(pColumn, currentRow);
} else {
if (!isVarCol) {
udfColDataSetNotNull_f(pColumn, currentRow);
memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes);
} else {
int32_t dataLen = varDataTLen(pData);
if (meta->type == TSDB_DATA_TYPE_JSON) {
if (*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0;
} else if (*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData + sizeof(char));
} else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = sizeof(int64_t);
} else if (*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = sizeof(char);
}
dataLen += sizeof(char);
}
if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) {
uint32_t newSize = data->varLenCol.payloadAllocLen;
if (newSize <= 1) {
newSize = 8;
}
while (newSize < data->varLenCol.payloadLen + dataLen) {
newSize = newSize * UDF_MEMORY_EXP_GROWTH;
}
char *buf = (char*)realloc(data->varLenCol.payload, newSize);
if (buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->varLenCol.payload = buf;
data->varLenCol.payloadAllocLen = newSize;
}
uint32_t len = data->varLenCol.payloadLen;
data->varLenCol.varOffsets[currentRow] = len;
memcpy(data->varLenCol.payload + len, pData, dataLen);
data->varLenCol.payloadLen += dataLen;
}
}
data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow+1) : data->numOfRows;
return 0;
}
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TAOSUDF_H

228
include/libs/function/tudf.h
View File

@ -16,6 +16,13 @@
#ifndef TDENGINE_TUDF_H
#define TDENGINE_TUDF_H
#undef malloc
#define malloc malloc
#undef free
#define free free
#undef realloc
#define alloc alloc
#include <taosudf.h>
#include <stdint.h>
#include <stdbool.h>
@ -36,56 +43,6 @@ extern "C" {
#endif
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
//======================================================================================
//begin API to taosd and qworker
typedef struct SUdfColumnMeta {
int16_t type;
int32_t bytes;
uint8_t precision;
uint8_t scale;
} SUdfColumnMeta;
typedef struct SUdfColumnData {
int32_t numOfRows;
int32_t rowsAlloc;
union {
struct {
int32_t nullBitmapLen;
char *nullBitmap;
int32_t dataLen;
char *data;
} fixLenCol;
struct {
int32_t varOffsetsLen;
int32_t *varOffsets;
int32_t payloadLen;
char *payload;
int32_t payloadAllocLen;
} varLenCol;
};
} SUdfColumnData;
typedef struct SUdfColumn {
SUdfColumnMeta colMeta;
bool hasNull;
SUdfColumnData colData;
} SUdfColumn;
typedef struct SUdfDataBlock {
int32_t numOfRows;
int32_t numOfCols;
SUdfColumn **udfCols;
} SUdfDataBlock;
typedef struct SUdfInterBuf {
int32_t bufLen;
char* buf;
int8_t numOfResult; //zero or one
} SUdfInterBuf;
typedef void *UdfcFuncHandle;
//low level APIs
/**
@ -127,177 +84,6 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
int32_t cleanUpUdfs();
// end API to taosd and qworker
//=============================================================================================================================
// begin API to UDF writer.
// dynamic lib init and destroy
typedef int32_t (*TUdfInitFunc)();
typedef int32_t (*TUdfDestroyFunc)();
//TODO: add API to check function arguments type, number etc.
#define UDF_MEMORY_EXP_GROWTH 1.5
#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1)
#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row))))
#define udfColDataSetNull_f(pColumn, row) \
do { \
BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \
} while (0)
#define udfColDataSetNotNull_f(pColumn, r_) \
do { \
BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \
} while (0)
#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1)
static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) {
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row];
} else {
return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row;
}
}
static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) {
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) {
if (udfColDataIsNull_var(pColumn, row)) {
return true;
}
char* data = udfColDataGetData(pColumn, row);
return (*data == TSDB_DATA_TYPE_NULL);
} else {
return udfColDataIsNull_var(pColumn, row);
}
} else {
return udfColDataIsNull_f(pColumn, row);
}
}
static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) {
SUdfColumnMeta *meta = &pColumn->colMeta;
SUdfColumnData *data = &pColumn->colData;
if (newCapacity== 0 || newCapacity <= data->rowsAlloc) {
return TSDB_CODE_SUCCESS;
}
int allocCapacity = TMAX(data->rowsAlloc, 8);
while (allocCapacity < newCapacity) {
allocCapacity *= UDF_MEMORY_EXP_GROWTH;
}
if (IS_VAR_DATA_TYPE(meta->type)) {
char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->varLenCol.varOffsets = (int32_t*)tmp;
data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity;
// for payload, add data in udfColDataAppend
} else {
char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->fixLenCol.nullBitmap = tmp;
data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity);
if (meta->type == TSDB_DATA_TYPE_NULL) {
return TSDB_CODE_SUCCESS;
}
tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->fixLenCol.data = tmp;
data->fixLenCol.dataLen = allocCapacity* meta->bytes;
}
data->rowsAlloc = allocCapacity;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) {
udfColEnsureCapacity(pColumn, row+1);
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
udfColDataSetNull_var(pColumn, row);
} else {
udfColDataSetNull_f(pColumn, row);
}
pColumn->hasNull = true;
}
static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) {
SUdfColumnMeta *meta = &pColumn->colMeta;
SUdfColumnData *data = &pColumn->colData;
udfColEnsureCapacity(pColumn, currentRow+1);
bool isVarCol = IS_VAR_DATA_TYPE(meta->type);
if (isNull) {
udfColDataSetNull(pColumn, currentRow);
} else {
if (!isVarCol) {
colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow);
memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes);
} else {
int32_t dataLen = varDataTLen(pData);
if (meta->type == TSDB_DATA_TYPE_JSON) {
if (*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0;
} else if (*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData + CHAR_BYTES);
} else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES;
} else if (*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES;
}
dataLen += CHAR_BYTES;
}
if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) {
uint32_t newSize = data->varLenCol.payloadAllocLen;
if (newSize <= 1) {
newSize = 8;
}
while (newSize < data->varLenCol.payloadLen + dataLen) {
newSize = newSize * UDF_MEMORY_EXP_GROWTH;
}
char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize);
if (buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
data->varLenCol.payload = buf;
data->varLenCol.payloadAllocLen = newSize;
}
uint32_t len = data->varLenCol.payloadLen;
data->varLenCol.varOffsets[currentRow] = len;
memcpy(data->varLenCol.payload + len, pData, dataLen);
data->varLenCol.payloadLen += dataLen;
}
}
data->numOfRows = TMAX(currentRow + 1, data->numOfRows);
return 0;
}
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
// end API to UDF writer
//=======================================================================================================================
#ifdef __cplusplus
}

1
include/libs/nodes/plannodes.h
View File

@ -103,6 +103,7 @@ typedef struct SIndefRowsFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
bool isTailFunc;
bool isUniqueFunc;
} SIndefRowsFuncLogicNode;
typedef struct SInterpFuncLogicNode {

2
include/libs/nodes/querynodes.h
View File

@ -50,6 +50,7 @@ typedef struct SExprNode {
char aliasName[TSDB_COL_NAME_LEN];
char userAlias[TSDB_COL_NAME_LEN];
SArray* pAssociation;
bool orderAlias;
} SExprNode;
typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG, COLUMN_TYPE_TBNAME } EColumnType;
@ -88,6 +89,7 @@ typedef struct SValueNode {
bool isDuration;
bool translate;
bool notReserved;
bool isNull;
int16_t placeholderNo;
union {
bool b;

2
include/libs/planner/planner.h
View File

@ -48,6 +48,8 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
// @pSource one execution location of this group of datasource subplans
int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
void qClearSubplanExecutionNode(SSubplan* pSubplan);
// Convert to subplan to string for the scheduler to send to the executor
int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);

6
include/libs/qcom/query.h
View File

@ -271,19 +271,19 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define qDebug(...) \
do { \
if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qTrace(...) \
do { \
if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qDebugL(...) \
do { \
if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \
} while (0)

15
include/util/taoserror.h
View File

@ -25,7 +25,7 @@ extern "C" {
// clang-format off
#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
#define TAOS_SUCCEEDED(err) ((err) >= 0)
#define TAOS_FAILED(err) ((err) < 0)
@ -35,7 +35,7 @@ const char* terrstr();
int32_t* taosGetErrno();
#define terrno (*taosGetErrno())
#define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
@ -466,11 +466,11 @@ int32_t* taosGetErrno();
#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400)
#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401)
#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402)
#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403)
#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404)
#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405)
#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406)
#define TSDB_CODE_CTG_VG_META_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x2407)
#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403)
#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2404)
#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2405)
#define TSDB_CODE_CTG_VG_META_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x2406)
#define TSDB_CODE_CTG_EXIT TAOS_DEF_ERROR_CODE(0, 0x2407)
//scheduler&qworker
#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501)
@ -615,6 +615,7 @@ int32_t* taosGetErrno();
//rsma
#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150)
#define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151)
#define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152)
//index
#define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200)

2
packaging/check_package.sh
View File

@ -170,7 +170,7 @@ function check_lib_path() {
function check_header_path() {
# check all header
header_dir=("taos.h" "taosdef.h" "taoserror.h")
header_dir=("taos.h" "taosdef.h" "taoserror.h" "taosudf.h")
for i in "${header_dir[@]}";do
check_link ${inc_link_dir}/$i
done

1
packaging/deb/DEBIAN/prerm
View File

@ -29,6 +29,7 @@ else
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${log_link_dir} || :

1
packaging/deb/makedeb.sh
View File

@ -70,6 +70,7 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat
cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include
cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include
cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples
#cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
#cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector

2
packaging/rpm/tdengine.spec
View File

@ -77,6 +77,7 @@ cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driv
cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include
#cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
#cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
#cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector
@ -201,6 +202,7 @@ if [ $1 -eq 0 ];then
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${log_link_dir} || :

3
packaging/tools/install.sh
View File

@ -314,11 +314,12 @@ function install_jemalloc() {
}
function install_header() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
}
function add_newHostname_to_hosts() {

3
packaging/tools/install_arbi.sh
View File

@ -115,11 +115,12 @@ function install_bin() {
}
function install_header() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
}
function install_jemalloc() {

3
packaging/tools/install_client.sh
View File

@ -148,11 +148,12 @@ function install_lib() {
}
function install_header() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
}
function install_jemalloc() {

9
packaging/tools/make_install.sh
View File

@ -349,16 +349,17 @@ function install_lib() {
function install_header() {
if [ "$osType" != "Darwin" ]; then
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
else
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
${install_main_dir}/include ||
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
${install_main_2_dir}/include &&
${csudo}chmod 644 ${install_main_dir}/include/* ||
${csudo}chmod 644 ${install_main_2_dir}/include/*

2
packaging/tools/makearbi.sh
View File

@ -36,7 +36,7 @@ fi
bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh"
install_files="${script_dir}/install_arbi.sh"
#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h"
#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord
init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord

2
packaging/tools/makeclient.sh
View File

@ -62,7 +62,7 @@ else
lib_files="${build_dir}/lib/libtaos.${version}.dylib"
fi
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h"
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
if [ "$dbName" != "taos" ]; then
cfg_dir="${top_dir}/../enterprise/packaging/cfg"
else

2
packaging/tools/makepkg.sh
View File

@ -93,7 +93,7 @@ else
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h"
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
if [ "$dbName" != "taos" ]; then
cfg_dir="${top_dir}/../enterprise/packaging/cfg"

3
packaging/tools/post.sh
View File

@ -81,10 +81,11 @@ function kill_taosd() {
}
function install_include() {
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h|| :
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h
${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h
${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h
${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h
}
function install_lib() {

1
packaging/tools/preun.sh
View File

@ -128,6 +128,7 @@ ${csudo}rm -f ${cfg_link_dir}/*.new || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :

1
packaging/tools/remove.sh
View File

@ -84,6 +84,7 @@ function clean_header() {
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taosdef.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
${csudo} rm -f ${inc_link_dir}/taosudf.h || :
}
function clean_config() {

2
packaging/tools/remove_arbi.sh
View File

@ -59,6 +59,8 @@ function clean_header() {
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
}
function clean_log() {

1
packaging/tools/remove_client.sh
View File

@ -54,6 +54,7 @@ function clean_header() {
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
}
function clean_config() {

2
source/client/src/clientMain.c
View File

@ -727,6 +727,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
.schemalessType = pTscObj->schemalessType,
.isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
.async = true,
.svrVer = pTscObj->sVer,
.nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)
};
return TSDB_CODE_SUCCESS;
}

21
source/common/src/tmsg.c
View File

@ -621,8 +621,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (NULL == pReq->pFuncs) return -1;
}
for (int32_t i = 0; i < numOfFuncs; ++i) {
char *pFunc = NULL;
if (tDecodeCStrAlloc(&decoder, &pFunc) < 0) return -1;
char pFunc[TSDB_FUNC_NAME_LEN] = {0};
if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
if (taosArrayPush(pReq->pFuncs, pFunc) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -2300,7 +2300,6 @@ int32_t tDeserializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp
return 0;
}
int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -2387,7 +2386,6 @@ int32_t tDeserializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp
void tFreeSDnodeListRsp(SDnodeListRsp *pRsp) { taosArrayDestroy(pRsp->dnodeList); }
int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -2909,20 +2907,19 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR
return 0;
}
int32_t tEncodeSVariablesInfo(SEncoder* pEncoder, SVariablesInfo* pInfo) {
int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) {
if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1;
return 0;
}
int32_t tDecodeSVariablesInfo(SDecoder* pDecoder, SVariablesInfo* pInfo) {
int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) {
if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1;
return 0;
}
int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) {
int32_t tSerializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -2930,7 +2927,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp
int32_t varNum = taosArrayGetSize(pRsp->variables);
if (tEncodeI32(&encoder, varNum) < 0) return -1;
for (int32_t i = 0; i < varNum; ++i) {
SVariablesInfo* pInfo = taosArrayGet(pRsp->variables, i);
SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i);
if (tEncodeSVariablesInfo(&encoder, pInfo) < 0) return -1;
}
tEndEncode(&encoder);
@ -2940,7 +2937,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp
return tlen;
}
int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) {
int32_t tDeserializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
@ -2962,11 +2959,11 @@ int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesR
return 0;
}
void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp) {
void tFreeSShowVariablesRsp(SShowVariablesRsp *pRsp) {
if (NULL == pRsp) {
return;
}
taosArrayDestroy(pRsp->variables);
}

20
source/dnode/vnode/src/inc/sma.h
View File

@ -205,16 +205,16 @@ struct STFile {
uint8_t state;
};
#define TD_FILE_F(tf) (&((tf)->f))
#define TD_FILE_PFILE(tf) ((tf)->pFile)
#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL)
#define TD_FILE_FULL_NAME(tf) (TD_FILE_F(tf)->aname)
#define TD_FILE_REL_NAME(tf) (TD_FILE_F(tf)->rname)
#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL)
#define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf))
#define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL)
#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TD_FILE_DID(tf) (TD_FILE_F(tf)->did)
#define TD_TFILE_F(tf) (&((tf)->f))
#define TD_TFILE_PFILE(tf) ((tf)->pFile)
#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL)
#define TD_TFILE_FULL_NAME(tf) (TD_TFILE_F(tf)->aname)
#define TD_TFILE_REL_NAME(tf) (TD_TFILE_F(tf)->rname)
#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL)
#define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf))
#define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL)
#define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TD_TFILE_DID(tf) (TD_TFILE_F(tf)->did)
int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -64,6 +64,7 @@ typedef struct STsdbSnapshotReader STsdbSnapshotReader;
#define VNODE_TQ_DIR "tq"
#define VNODE_WAL_DIR "wal"
#define VNODE_TSMA_DIR "tsma"
#define VNODE_RSMA_DIR "rsma"
#define VNODE_RSMA0_DIR "tsdb"
#define VNODE_RSMA1_DIR "rsma1"
#define VNODE_RSMA2_DIR "rsma2"
@ -161,7 +162,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
// sma
int32_t smaOpen(SVnode* pVnode);
int32_t smaClose(SSma* pSma);
int32_t smaCloseEnv(SSma* pSma);
int32_t smaCloseEx(SSma* pSma);

8
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -123,7 +123,7 @@ int32_t smaOpen(SVnode *pVnode) {
}
// restore the rsma
#if 0
#if 1
if (rsmaRestore(pSma) < 0) {
goto _err;
}
@ -154,12 +154,6 @@ int32_t smaCloseEx(SSma *pSma) {
return 0;
}
int32_t smaClose(SSma *pSma) {
smaCloseEnv(pSma);
smaCloseEx(pSma);
return 0;
}
/**
* @brief rsma env restore
*

687
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -17,11 +17,12 @@
#define RSMA_QTASKINFO_PERSIST_MS 7200000
#define RSMA_QTASKINFO_BUFSIZE 32768
#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid
typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
typedef struct SRSmaQTaskFIter SRSmaQTaskFIter;
typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
@ -32,13 +33,17 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
static void tdRSmaFetchTrigger(void *param, void *tmrId);
static void tdRSmaPersistTrigger(void *param, void *tmrId);
static void *tdRSmaPersistExec(void *param);
static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName);
static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName);
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile);
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish);
static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd);
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter);
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma);
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma);
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
struct SRSmaInfoItem {
SRSmaInfo *pRsmaInfo;
void *taskInfo; // qTaskInfo_t
@ -63,22 +68,23 @@ struct SRSmaQTaskInfoItem {
void *qTaskInfo;
};
struct SRSmaQTaskFIter {
struct SRSmaQTaskInfoIter {
STFile *pTFile;
int64_t offset;
int64_t fsize;
int32_t nBytes;
int32_t nAlloc;
char *buf;
char *pBuf;
// ------------
char *qBuf; // for iterator
int32_t nBufPos;
};
static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t);
return lenWithHead - RSMA_QTASKINFO_HEAD_LEN;
}
static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); }
static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); }
static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) {
// Note: free/kill may in RC
@ -244,6 +250,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
pItem->pRsmaInfo = pRSmaInfo;
pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
if (!pItem->taskInfo) {
terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
goto _err;
}
pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE;
@ -294,7 +301,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
if (pRSmaInfo) {
ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally
ASSERT(0); // TODO: free original pRSmaInfo if exists abnormally
smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid);
return TSDB_CODE_SUCCESS;
}
@ -338,10 +345,10 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
goto _err;
} else {
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
}
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
// start the persist timer
if (TASK_TRIGGER_STAT_INIT ==
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
@ -356,10 +363,9 @@ _err:
}
/**
* @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
* @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam currently
*
* @param pTsdb
* @param pMeta
* @param pVnode
* @param pReq
* @return int32_t
*/
@ -695,8 +701,310 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
return TSDB_CODE_SUCCESS;
}
static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName);
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
SVnode *pVnode = pSma->pVnode;
SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
taosArrayDestroy(suidList);
smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr());
return TSDB_CODE_FAILED;
}
int32_t arrSize = taosArrayGetSize(suidList);
if (arrSize == 0) {
taosArrayDestroy(suidList);
smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode));
return TSDB_CODE_SUCCESS;
}
SMetaReader mr = {0};
metaReaderInit(&mr, SMA_META(pSma), 0);
for (int32_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("vgId:%d, rsma restore, suid[%d] is %" PRIi64, TD_VID(pVnode), i, suid);
if (metaGetTableEntryByUid(&mr, suid) < 0) {
smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
terrstr());
goto _err;
}
ASSERT(mr.me.type == TSDB_SUPER_TABLE);
ASSERT(mr.me.uid == suid);
if (TABLE_IS_ROLLUP(mr.me.flags)) {
SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
for (int i = 0; i < TSDB_RETENTION_L2; ++i) {
smaDebug("vgId:%d, rsma restore, table:%" PRIi64 " level:%d, maxdelay:%" PRIi64 " watermark:%" PRIi64
" qmsgLen:%" PRIi32,
TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]);
}
if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) {
smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
goto _err;
}
smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid);
}
}
metaReaderClear(&mr);
taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS;
_err:
metaReaderClear(&mr);
taosArrayDestroy(suidList);
return TSDB_CODE_FAILED;
}
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
SVnode *pVnode = pSma->pVnode;
STFile tFile = {0};
char qTaskInfoFName[TSDB_FILENAME_LEN];
tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
goto _err;
}
if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
return TSDB_CODE_SUCCESS;
}
if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) {
goto _err;
}
SRSmaQTaskInfoIter fIter = {0};
if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
tdRSmaQTaskInfoIterDestroy(&fIter);
tdCloseTFile(&tFile);
goto _err;
}
if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
tdRSmaQTaskInfoIterDestroy(&fIter);
tdCloseTFile(&tFile);
goto _err;
}
tdRSmaQTaskInfoIterDestroy(&fIter);
tdCloseTFile(&tFile);
return TSDB_CODE_SUCCESS;
_err:
smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
return TSDB_CODE_FAILED;
}
/**
* @brief reload ts data from checkpoint
*
* @param pSma
* @return int32_t
*/
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
// TODO
return TSDB_CODE_SUCCESS;
_err:
smaError("rsma restore, ts data reload failed since %s", terrstr());
return TSDB_CODE_FAILED;
}
int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
// step 1: iterate all stables to restore the rsma env
if (tdRSmaRestoreQTaskInfoInit(pSma) < 0) {
goto _err;
}
// step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
if (tdRSmaRestoreQTaskInfoReload(pSma) < 0) {
goto _err;
}
// step 3: reload ts data from checkpoint
if (tdRSmaRestoreTSDataReload(pSma) < 0) {
goto _err;
}
return TSDB_CODE_SUCCESS;
_err:
smaError("failed to restore rsma task since %s", terrstr());
return TSDB_CODE_FAILED;
}
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
SRSmaInfo *pRSmaInfo = NULL;
void *qTaskInfo = NULL;
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
return TSDB_CODE_SUCCESS;
}
if (pItem->type == 1) {
qTaskInfo = pRSmaInfo->items[0].taskInfo;
} else if (pItem->type == 2) {
qTaskInfo = pRSmaInfo->items[1].taskInfo;
} else {
ASSERT(0);
}
if (!qTaskInfo) {
smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
return TSDB_CODE_SUCCESS;
}
if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) {
smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid,
pItem->type, terrstr(terrno));
return TSDB_CODE_FAILED;
}
smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type);
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile) {
memset(pIter, 0, sizeof(*pIter));
pIter->pTFile = pTFile;
pIter->offset = TD_FILE_HEAD_SIZE;
if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) {
return TSDB_CODE_FAILED;
}
if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) {
pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE;
} else {
pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE;
}
if (pIter->nAlloc < TD_FILE_HEAD_SIZE) {
pIter->nAlloc = TD_FILE_HEAD_SIZE;
}
pIter->pBuf = taosMemoryMalloc(pIter->nAlloc);
if (!pIter->pBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
pIter->qBuf = pIter->pBuf;
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish) {
STFile *pTFile = pIter->pTFile;
int64_t nBytes = RSMA_QTASKINFO_BUFSIZE;
if (pIter->offset >= pIter->fsize) {
*isFinish = true;
return TSDB_CODE_SUCCESS;
}
if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) {
nBytes = pIter->fsize - pIter->offset;
}
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
int32_t infoLen = 0;
taosDecodeFixedI32(pIter->qBuf, &infoLen);
if (infoLen > nBytes) {
ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
pIter->nAlloc = infoLen;
void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
if (!pBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
pIter->pBuf = pBuf;
pIter->qBuf = pIter->pBuf;
nBytes = infoLen;
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
}
pIter->offset += nBytes;
pIter->nBytes = nBytes;
pIter->nBufPos = 0;
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
while (1) {
// block iter
bool isFinish = false;
if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (isFinish) {
return TSDB_CODE_SUCCESS;
}
// consume the block
int32_t qTaskInfoLenWithHead = 0;
pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return TSDB_CODE_FAILED;
}
while (1) {
if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) {
SRSmaQTaskInfoItem infoItem = {0};
pIter->qBuf = taosDecodeFixedI8(pIter->qBuf, &infoItem.type);
pIter->qBuf = taosDecodeFixedI64(pIter->qBuf, &infoItem.suid);
infoItem.qTaskInfo = pIter->qBuf;
infoItem.len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead);
// do the restore job
smaDebug("vgId:%d, restore the qtask info %s offset:%" PRIi64 "\n", SMA_VID(pSma),
TD_TFILE_FULL_NAME(pIter->pTFile), pIter->offset - pIter->nBytes + pIter->nBufPos);
tdRSmaQTaskInfoItemRestore(pSma, &infoItem);
pIter->qBuf = POINTER_SHIFT(pIter->qBuf, infoItem.len);
pIter->nBufPos += qTaskInfoLenWithHead;
if ((pIter->nBufPos + RSMA_QTASKINFO_HEAD_LEN) >= pIter->nBytes) {
// prepare and load next block in the file
pIter->offset -= (pIter->nBytes - pIter->nBufPos);
break;
}
pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
continue;
}
// prepare and load next block in the file
pIter->offset -= (pIter->nBytes - pIter->nBufPos);
break;
}
}
return TSDB_CODE_SUCCESS;
}
static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName) {
tdGetVndFileName(vid, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
}
static void *tdRSmaPersistExec(void *param) {
@ -704,6 +1012,7 @@ static void *tdRSmaPersistExec(void *param) {
SRSmaStat *pRSmaStat = param;
SSma *pSma = pRSmaStat->pSma;
STfs *pTfs = pSma->pVnode->pTfs;
int32_t vid = SMA_VID(pSma);
int64_t toffset = 0;
bool isFileCreated = false;
@ -716,26 +1025,16 @@ static void *tdRSmaPersistExec(void *param) {
goto _end;
}
STFile tFile = {0};
int32_t vid = SMA_VID(pSma);
STFile tFile = {0};
while (infoHash) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
#if 0
smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid);
for (int32_t i = 15; i > 0; --i) {
taosSsleep(1);
smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i);
}
smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid);
#endif
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo;
if (!taskInfo) {
smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
continue;
}
char *pOutput = NULL;
int32_t len = 0;
int8_t type = (int8_t)(i + 1);
@ -743,69 +1042,77 @@ static void *tdRSmaPersistExec(void *param) {
smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
terrstr(terrno));
goto _err;
} else {
if (!pOutput) {
smaDebug("vgId:%d, table %" PRIi64
" level %d serialize rsma task success but no output(len %d) and no need to persist",
vid, pRSmaInfo->suid, i + 1, len);
continue;
} else if (len <= 0) {
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist",
vid, pRSmaInfo->suid, i + 1, len);
taosMemoryFree(pOutput);
}
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid,
pRSmaInfo->suid, i + 1, len);
#if 1
if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) {
smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid,
i + 1, terrstr(terrno));
} else {
smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1);
}
#endif
}
if (!pOutput || len <= 0) {
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success but no output(len %d), not persist",
vid, pRSmaInfo->suid, i + 1, len);
taosMemoryFreeClear(pOutput);
continue;
}
smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d, need persist", vid,
pRSmaInfo->suid, i + 1, len);
#if 0
if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) {
smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid,
i + 1, terrstr(terrno));
} else {
smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1);
}
#endif
if (!isFileCreated) {
char qTaskInfoFName[TSDB_FILENAME_LEN];
tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName);
tdInitTFile(&tFile, pTfs, qTaskInfoFName);
tdCreateTFile(&tFile, pTfs, true, -1);
isFileCreated = true;
}
len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid));
tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
tdAppendTFile(&tFile, &type, sizeof(type), &toffset);
tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset);
char tmpBuf[RSMA_QTASKINFO_HEAD_LEN] = {0};
void *pTmpBuf = &tmpBuf;
int32_t headLen = 0;
headLen += taosEncodeFixedI32(&pTmpBuf, len + RSMA_QTASKINFO_HEAD_LEN);
headLen += taosEncodeFixedI8(&pTmpBuf, type);
headLen += taosEncodeFixedI64(&pTmpBuf, pRSmaInfo->suid);
ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN);
tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset);
smaDebug("vgId:%d, table %" PRIi64 " level %d head part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
i + 1, headLen, toffset);
tdAppendTFile(&tFile, pOutput, len, &toffset);
smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid,
i + 1, len, toffset);
taosMemoryFree(pOutput);
}
infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
}
_normal:
if (isFileCreated) {
if (tdUpdateTFileHeader(&tFile) < 0) {
smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno));
smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
tstrerror(terrno));
tdCloseTFile(&tFile);
tdRemoveTFile(&tFile);
goto _err;
} else {
smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile));
smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
}
tdCloseTFile(&tFile);
char newFName[TSDB_FILENAME_LEN];
strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]);
strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) {
smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName);
if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
goto _err;
} else {
smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName);
smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
}
}
goto _end;
@ -841,13 +1148,14 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
TASK_TRIGGER_STAT_INACTIVE,
TASK_TRIGGER_STAT_ACTIVE)) {
smaDebug("persist task is active again");
smaDebug("vgId:%d, persist task is active again", SMA_VID(pRSmaStat->pSma));
} else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
TASK_TRIGGER_STAT_CANCELLED,
TASK_TRIGGER_STAT_FINISHED)) {
smaDebug(" persist task is cancelled and set finished");
smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma));
} else {
smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)),
SMA_VID(pRSmaStat->pSma));
ASSERT(0);
}
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
@ -864,7 +1172,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
*/
static void tdRSmaPersistTrigger(void *param, void *tmrId) {
SRSmaStat *pRSmaStat = param;
int8_t tmrStat =
int8_t tmrStat =
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
switch (tmrStat) {
case TASK_TRIGGER_STAT_ACTIVE: {
@ -872,7 +1181,7 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
TASK_TRIGGER_STAT_CANCELLED,
TASK_TRIGGER_STAT_FINISHED)) {
smaDebug("rsma persistence start since active");
smaDebug("vgId:%d, rsma persistence start since active", SMA_VID(pRSmaStat->pSma));
// start persist task
tdRSmaPersistTask(pRSmaStat);
@ -895,252 +1204,6 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
} break;
default: {
smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
ASSERT(0);
} break;
}
}
int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
SVnode *pVnode = pSma->pVnode;
// step 1: iterate all stables to restore the rsma env
SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr());
return TSDB_CODE_FAILED;
}
if (taosArrayGetSize(suidList) == 0) {
smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode));
return TSDB_CODE_SUCCESS;
}
SMetaReader mr = {0};
metaReaderInit(&mr, SMA_META(pSma), 0);
for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("suid [%d] is %" PRIi64, i, suid);
if (metaGetTableEntryByUid(&mr, suid) < 0) {
smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
goto _err;
}
ASSERT(mr.me.type == TSDB_SUPER_TABLE);
ASSERT(mr.me.uid == suid);
if (TABLE_IS_ROLLUP(mr.me.flags)) {
SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
for (int i = 0; i < 2; ++i) {
smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode),
suid, i, param->maxdelay[i], i, param->watermark[i]);
}
if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) {
smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
goto _err;
}
}
}
// step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore
STFile tFile = {0};
char qTaskInfoFName[TSDB_FILENAME_LEN];
tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
goto _err;
}
if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) {
goto _err;
}
SRSmaQTaskFIter fIter = {0};
if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
goto _err;
}
SRSmaQTaskInfoItem infoItem = {0};
bool isEnd = false;
int32_t code = 0;
while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) {
if (isEnd) {
break;
}
if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) {
break;
}
}
tdRSmaQTaskInfoIterDestroy(&fIter);
if (code < 0) {
goto _err;
}
metaReaderClear(&mr);
taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS;
_err:
ASSERT(0);
metaReaderClear(&mr);
taosArrayDestroy(suidList);
smaError("failed to restore rsma info since %s", terrstr());
return TSDB_CODE_FAILED;
}
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) {
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
SRSmaInfo *pRSmaInfo = NULL;
void *qTaskInfo = NULL;
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
return TSDB_CODE_SUCCESS;
}
if (infoItem->type == 1) {
qTaskInfo = pRSmaInfo->items[0].taskInfo;
} else if (infoItem->type == 2) {
qTaskInfo = pRSmaInfo->items[1].taskInfo;
} else {
ASSERT(0);
}
if (!qTaskInfo) {
smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
return TSDB_CODE_SUCCESS;
}
if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) {
smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid,
infoItem->type, terrstr(terrno));
return TSDB_CODE_FAILED;
}
smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid,
infoItem->type);
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) {
memset(pIter, 0, sizeof(*pIter));
pIter->pTFile = pTFile;
pIter->offset = TD_FILE_HEAD_SIZE;
if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) {
return TSDB_CODE_FAILED;
}
if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) {
pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE;
} else {
pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE;
}
if (pIter->nAlloc < TD_FILE_HEAD_SIZE) {
pIter->nAlloc = TD_FILE_HEAD_SIZE;
}
pIter->buf = taosMemoryMalloc(pIter->nAlloc);
if (!pIter->buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) {
STFile *pTFile = pIter->pTFile;
int64_t nBytes = RSMA_QTASKINFO_BUFSIZE;
if (pIter->offset >= pIter->fsize) {
*isFinish = true;
return TSDB_CODE_SUCCESS;
}
if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) {
nBytes = pIter->fsize - pIter->offset;
}
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
int32_t infoLen = 0;
taosDecodeFixedI32(pIter->buf, &infoLen);
if (infoLen > nBytes) {
ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
pIter->nAlloc = infoLen;
void *pBuf = taosMemoryRealloc(pIter->buf, infoLen);
if (!pBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
pIter->buf = pBuf;
nBytes = infoLen;
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
}
pIter->offset += nBytes;
pIter->nBytes = nBytes;
pIter->nBufPos = 0;
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) {
while (1) {
// block iter
bool isFinish = false;
if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (isFinish) {
*isEnd = true;
return TSDB_CODE_SUCCESS;
}
// consume the block
int32_t qTaskInfoLenWithHead = 0;
pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
if (qTaskInfoLenWithHead < 0) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return TSDB_CODE_FAILED;
}
while (1) {
if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) {
pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type);
pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid);
pItem->qTaskInfo = pIter->buf;
pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead);
// do the restore job
printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset);
pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len);
pIter->nBufPos += qTaskInfoLenWithHead;
pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
continue;
}
// prepare and load next block in the file
pIter->offset -= (pIter->nBytes - pIter->nBufPos);
break;
}
}
return TSDB_CODE_SUCCESS;
}

42
source/dnode/vnode/src/sma/smaUtil.c
View File

@ -22,7 +22,6 @@
#define TD_FILE_INIT_MAGIC 0xFFFFFFFF
static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo);
static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo);
@ -46,7 +45,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) {
}
int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) {
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte);
if (nwrite < nbyte) {
@ -58,9 +57,9 @@ int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) {
}
int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) {
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
int64_t loffset = taosLSeekFile(TD_FILE_PFILE(pTFile), offset, whence);
int64_t loffset = taosLSeekFile(TD_TFILE_PFILE(pTFile), offset, whence);
if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -70,12 +69,12 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) {
}
int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) {
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
return taosFStatFile(pTFile->pFile, size, NULL);
}
int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) {
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte);
if (nread < 0) {
@ -108,7 +107,7 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) {
char buf[TD_FILE_HEAD_SIZE] = "\0";
uint32_t _version;
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) {
return -1;
@ -133,7 +132,7 @@ void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) {
}
int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) {
ASSERT(TD_FILE_OPENED(pTFile));
ASSERT(TD_TFILE_OPENED(pTFile));
int64_t toffset;
@ -141,6 +140,11 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
return -1;
}
#if 1
smaDebug("append to file %s, offset:%" PRIi64 " + nbyte:%" PRIi64 " =%" PRIi64, TD_TFILE_FULL_NAME(pTFile), toffset,
nbyte, toffset + nbyte);
#endif
ASSERT(pTFile->info.fsize == toffset);
if (offset) {
@ -157,9 +161,9 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
}
int32_t tdOpenTFile(STFile *pTFile, int flags) {
ASSERT(!TD_FILE_OPENED(pTFile));
ASSERT(!TD_TFILE_OPENED(pTFile));
pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), flags);
pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), flags);
if (pTFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -169,9 +173,9 @@ int32_t tdOpenTFile(STFile *pTFile, int flags) {
}
void tdCloseTFile(STFile *pTFile) {
if (TD_FILE_OPENED(pTFile)) {
if (TD_TFILE_OPENED(pTFile)) {
taosCloseFile(&pTFile->pFile);
TD_FILE_SET_CLOSED(pTFile);
TD_TFILE_SET_CLOSED(pTFile);
}
}
@ -183,8 +187,8 @@ int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
char fullname[TSDB_FILENAME_LEN];
SDiskID did = {0};
TD_FILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
TD_FILE_SET_CLOSED(pTFile);
TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
TD_TFILE_SET_CLOSED(pTFile);
memset(&(pTFile->info), 0, sizeof(pTFile->info));
pTFile->info.magic = TD_FILE_INIT_MAGIC;
@ -202,18 +206,18 @@ int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pTFile->pFile == NULL) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TD_FILE_REL_NAME(pTFile));
if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_FILE_DID(pTFile)) < 0) {
char *s = strdup(TD_TFILE_REL_NAME(pTFile));
if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) {
taosMemoryFreeClear(s);
return -1;
}
taosMemoryFreeClear(s);
pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pTFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -240,7 +244,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
return 0;
}
int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); }
int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); }
// smaXXXUtil ================
// ...

4
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -152,14 +152,14 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
return pVnode;
_err:
if (pVnode->pSma) smaClose(pVnode->pSma);
if (pVnode->pSma) smaCloseEnv(pVnode->pSma);
if (pVnode->pQuery) vnodeQueryClose(pVnode);
if (pVnode->pTq) tqClose(pVnode->pTq);
if (pVnode->pWal) walClose(pVnode->pWal);
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
if (pVnode->pSma) smaCloseEx(pVnode->pSma);
if (pVnode->pMeta) metaClose(pVnode->pMeta);
tsem_destroy(&(pVnode->canCommit));
taosMemoryFree(pVnode);
return NULL;

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -136,7 +136,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_TTL_TABLE:
if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
//if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA: {
if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;

76
source/libs/catalog/inc/catalogInt.h
View File

@ -205,6 +205,7 @@ typedef struct SCtgJob {
SArray* pTasks;
int32_t taskDone;
SMetaData jobRes;
int32_t jobResCode;
int32_t taskIdx;
SRWLatch taskLock;
@ -284,24 +285,26 @@ typedef struct SCtgApiStat {
} SCtgApiStat;
typedef struct SCtgRuntimeStat {
uint64_t qNum;
uint64_t qDoneNum;
uint64_t numOfOpAbort;
uint64_t numOfOpEnqueue;
uint64_t numOfOpDequeue;
} SCtgRuntimeStat;
typedef struct SCtgCacheStat {
uint64_t clusterNum;
uint64_t dbNum;
uint64_t tblNum;
uint64_t stblNum;
uint64_t userNum;
uint64_t vgHitNum;
uint64_t vgMissNum;
uint64_t tbMetaHitNum;
uint64_t tbMetaMissNum;
uint64_t tbIndexHitNum;
uint64_t tbIndexMissNum;
uint64_t userHitNum;
uint64_t userMissNum;
uint64_t numOfCluster;
uint64_t numOfDb;
uint64_t numOfTbl;
uint64_t numOfStb;
uint64_t numOfUser;
uint64_t numOfVgHit;
uint64_t numOfVgMiss;
uint64_t numOfMetaHit;
uint64_t numOfMetaMiss;
uint64_t numOfIndexHit;
uint64_t numOfIndexMiss;
uint64_t numOfUserHit;
uint64_t numOfUserMiss;
uint64_t numOfClear;
} SCtgCacheStat;
typedef struct SCatalogStat {
@ -371,6 +374,7 @@ typedef struct SCtgDropTbIndexMsg {
typedef struct SCtgClearCacheMsg {
SCatalog* pCtg;
bool freeCtg;
} SCtgClearCacheMsg;
typedef struct SCtgUpdateEpsetMsg {
@ -385,6 +389,7 @@ typedef struct SCtgCacheOperation {
void *data;
bool syncOp;
tsem_t rspSem;
bool stopQueue;
} SCtgCacheOperation;
typedef struct SCtgQNode {
@ -394,6 +399,7 @@ typedef struct SCtgQNode {
typedef struct SCtgQueue {
SRWLatch qlock;
bool stopQueue;
SCtgQNode *head;
SCtgQNode *tail;
tsem_t reqSem;
@ -513,8 +519,35 @@ typedef struct SCtgOperation {
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0)
#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0)
#define CTG_API_LEAVE(c) do { \
int32_t __code = c; \
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
} while (0)
#define CTG_API_ENTER() do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_API_LEAVE_NOLOCK(c) do { \
int32_t __code = c; \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
} while (0)
#define CTG_API_ENTER_NOLOCK() do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE_NOLOCK(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
void ctgdShowClusterCache(SCatalog* pCtg);
@ -547,7 +580,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy
int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq);
int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet);
int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp);
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool syncOp);
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool syncOp);
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type);
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size);
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size);
@ -582,17 +615,18 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S
int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask);
int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask);
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum);
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param);
int32_t ctgLaunchJob(SCtgJob *pJob);
int32_t ctgMakeAsyncRes(SCtgJob *pJob);
int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param);
int32_t ctgGetTbCfgCb(SCtgTask *pTask);
void ctgFreeHandle(SCatalog* pCatalog);
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst);
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput);
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList);
void ctgFreeJob(void* job);
void ctgFreeHandle(SCatalog* pCtg);
void ctgFreeHandleImpl(SCatalog* pCtg);
void ctgFreeVgInfo(SDBVgInfo *vgInfo);
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup);
void ctgResetTbMetaTask(SCtgTask* pTask);
@ -608,6 +642,8 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCt
int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes);
void ctgFreeSTableIndex(void *info);
void ctgClearSubTaskRes(SCtgSubRes *pRes);
void ctgFreeQNode(SCtgQNode *node);
void ctgClearHandle(SCatalog* pCtg);
extern SCatalogMgmt gCtgMgmt;

97
source/libs/catalog/src/catalog.c
View File

@ -99,7 +99,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx*
STableMetaOutput* output = taosMemoryCalloc(1, sizeof(STableMetaOutput));
if (NULL == output) {
ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
@ -264,7 +264,7 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) {
STableMetaOutput* output = taosMemoryCalloc(1, sizeof(STableMetaOutput));
if (NULL == output) {
ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t code = 0;
@ -442,7 +442,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl
vgList = taosArrayInit(1, sizeof(SVgroupInfo));
if (NULL == vgList) {
ctgError("taosArrayInit %d failed", (int32_t)sizeof(SVgroupInfo));
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == taosArrayPush(vgList, &vgroupInfo)) {
@ -548,9 +548,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_API_ENTER();
if (NULL == gCtgMgmt.pCluster) {
qError("catalog cluster cache are not ready, clusterId:0x%" PRIx64, clusterId);
CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY);
CTG_API_LEAVE(TSDB_CODE_CTG_NOT_READY);
}
int32_t code = 0;
@ -562,13 +564,13 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
if (ctg && (*ctg)) {
*catalogHandle = *ctg;
qDebug("got catalog handle from cache, clusterId:0x%" PRIx64 ", CTG:%p", clusterId, *ctg);
return TSDB_CODE_SUCCESS;
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
clusterCtg = taosMemoryCalloc(1, sizeof(SCatalog));
if (NULL == clusterCtg) {
qError("calloc %d failed", (int32_t)sizeof(SCatalog));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
}
clusterCtg->clusterId = clusterId;
@ -580,13 +582,19 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
false, HASH_ENTRY_LOCK);
if (NULL == clusterCtg->dbCache) {
qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
clusterCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == clusterCtg->userCache) {
qError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES);
if (code) {
if (HASH_NODE_EXIST(code)) {
ctgFreeHandle(clusterCtg);
ctgFreeHandleImpl(clusterCtg);
continue;
}
@ -601,34 +609,15 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
*catalogHandle = clusterCtg;
CTG_CACHE_STAT_INC(clusterNum, 1);
CTG_CACHE_STAT_INC(numOfCluster, 1);
return TSDB_CODE_SUCCESS;
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
_return:
ctgFreeHandle(clusterCtg);
ctgFreeHandleImpl(clusterCtg);
CTG_RET(code);
}
void catalogFreeHandle(SCatalog* pCtg) {
if (NULL == pCtg) {
return;
}
if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) {
ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:0x%" PRIx64, pCtg->clusterId);
return;
}
CTG_CACHE_STAT_DEC(clusterNum, 1);
uint64_t clusterId = pCtg->clusterId;
ctgFreeHandle(pCtg);
ctgInfo("handle freed, culsterId:0x%" PRIx64, clusterId);
CTG_API_LEAVE(code);
}
int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) {
@ -1008,7 +997,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalo
pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);
if (NULL == pRsp->pTableMeta) {
ctgError("taosArrayInit %d failed", tbNum);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
for (int32_t i = 0; i < tbNum; ++i) {
@ -1023,7 +1012,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalo
if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
ctgError("taosArrayPush failed, idx:%d", i);
taosMemoryFreeClear(pTableMeta);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
}
@ -1058,14 +1047,9 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SC
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0, taskNum = 0;
int32_t code = 0;
SCtgJob *pJob = NULL;
CTG_ERR_JRET(ctgInitJob(pCtg, pConn, &pJob, pReq, fp, param, &taskNum));
if (taskNum <= 0) {
SMetaData* pMetaData = taosMemoryCalloc(1, sizeof(SMetaData));
fp(pMetaData, param, TSDB_CODE_SUCCESS);
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
CTG_ERR_JRET(ctgInitJob(pCtg, pConn, &pJob, pReq, fp, param));
CTG_ERR_JRET(ctgLaunchJob(pJob));
@ -1073,6 +1057,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SC
// *jobId = pJob->refId;
_return:
if (pJob) {
taosReleaseRef(gCtgMgmt.jobPool, pJob->refId);
@ -1274,19 +1259,19 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) {
}
int32_t catalogClearCache(void) {
CTG_API_ENTER();
CTG_API_ENTER_NOLOCK();
qInfo("start to clear catalog cache");
if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
CTG_API_LEAVE_NOLOCK(TSDB_CODE_SUCCESS);
}
int32_t code = ctgClearCacheEnqueue(NULL, true);
int32_t code = ctgClearCacheEnqueue(NULL, false, false, true);
qInfo("clear catalog cache end, code: %s", tstrerror(code));
CTG_API_LEAVE(code);
CTG_API_LEAVE_NOLOCK(code);
}
@ -1299,32 +1284,12 @@ void catalogDestroy(void) {
atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
if (tsem_post(&gCtgMgmt.queue.reqSem)) {
qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
while (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
taosUsleep(1);
}
CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock);
SCatalog* pCtg = NULL;
void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog**)pIter;
if (pCtg) {
catalogFreeHandle(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
if (!taosCheckCurrentInDll()) {
ctgClearCacheEnqueue(NULL, true, true, true);
}
taosHashCleanup(gCtgMgmt.pCluster);
gCtgMgmt.pCluster = NULL;
if (CTG_IS_LOCKED(&gCtgMgmt.lock) == TD_RWLATCH_WRITE_FLAG_COPY) CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock);
qInfo("catalog destroyed");
}

44
source/libs/catalog/src/ctgAsync.c
View File

@ -427,7 +427,7 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) {
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param) {
int32_t code = 0;
int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta);
int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup);
@ -443,11 +443,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
*taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum;
if (*taskNum <= 0) {
ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, pConn->requestId);
return TSDB_CODE_SUCCESS;
}
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum;
*job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) {
@ -477,15 +473,15 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
pJob->tbCfgNum = tbCfgNum;
pJob->svrVerNum = svrVerNum;
pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask));
pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask));
if (NULL == pJob->pTasks) {
ctgError("taosArrayInit %d tasks failed", *taskNum);
ctgError("taosArrayInit %d tasks failed", taskNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (pReq->forceUpdate) {
CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, *taskNum, pJob, pReq));
if (pReq->forceUpdate && taskNum) {
CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, taskNum, pJob, pReq));
}
for (int32_t i = 0; i < dbVgNum; ++i) {
@ -558,11 +554,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
taosAcquireRef(gCtgMgmt.jobPool, pJob->refId);
qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId, *taskNum, pReq->forceUpdate);
qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate);
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFreeClear(*job);
CTG_RET(code);
}
@ -763,7 +760,7 @@ int32_t ctgDumpSvrVer(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgInvokeSubCb(SCtgTask *pTask) {
int32_t ctgCallSubCb(SCtgTask *pTask) {
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &pTask->lock);
@ -790,6 +787,15 @@ _return:
CTG_RET(code);
}
int32_t ctgCallUserCb(void* param) {
SCtgJob* pJob = (SCtgJob*)param;
(*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode);
taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
SCtgJob* pJob = pTask->pJob;
@ -804,7 +810,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
pTask->code = rspCode;
pTask->status = CTG_TASK_DONE;
ctgInvokeSubCb(pTask);
ctgCallSubCb(pTask);
int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
if (taskDone < taosArrayGetSize(pJob->pTasks)) {
@ -818,9 +824,9 @@ _return:
qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code));
(*pJob->userFp)(&pJob->jobRes, pJob->userParam, code);
taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);
pJob->jobResCode = code;
taosAsyncExec(ctgCallUserCb, pJob, NULL);
CTG_RET(code);
}
@ -1697,6 +1703,12 @@ int32_t ctgLaunchJob(SCtgJob *pJob) {
pTask->status = CTG_TASK_LAUNCHED;
}
if (taskNum <= 0) {
qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode));
taosAsyncExec(ctgCallUserCb, pJob, NULL);
}
return TSDB_CODE_SUCCESS;
}

209
source/libs/catalog/src/ctgCache.c
View File

@ -214,7 +214,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac
*pCache = dbCache;
CTG_CACHE_STAT_INC(vgHitNum, 1);
CTG_CACHE_STAT_INC(numOfVgHit, 1);
ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
@ -228,7 +228,7 @@ _return:
*pCache = NULL;
CTG_CACHE_STAT_INC(vgMissNum, 1);
CTG_CACHE_STAT_INC(numOfVgMiss, 1);
return TSDB_CODE_SUCCESS;
}
@ -260,7 +260,7 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S
ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName);
CTG_CACHE_STAT_INC(tbMetaHitNum, 1);
CTG_CACHE_STAT_INC(numOfMetaHit, 1);
return TSDB_CODE_SUCCESS;
@ -268,7 +268,7 @@ _return:
ctgReleaseTbMetaToCache(pCtg, dbCache, pCache);
CTG_CACHE_STAT_INC(tbMetaMissNum, 1);
CTG_CACHE_STAT_INC(numOfMetaMiss, 1);
return TSDB_CODE_SUCCESS;
}
@ -307,7 +307,7 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid,
ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName);
CTG_CACHE_STAT_INC(tbMetaHitNum, 1);
CTG_CACHE_STAT_INC(numOfMetaHit, 1);
return TSDB_CODE_SUCCESS;
@ -315,7 +315,7 @@ _return:
ctgReleaseTbMetaToCache(pCtg, dbCache, pCache);
CTG_CACHE_STAT_INC(tbMetaMissNum, 1);
CTG_CACHE_STAT_INC(numOfMetaMiss, 1);
*pDb = NULL;
*pTb = NULL;
@ -351,7 +351,7 @@ int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName,
ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName);
CTG_CACHE_STAT_INC(tbIndexHitNum, 1);
CTG_CACHE_STAT_INC(numOfIndexHit, 1);
return TSDB_CODE_SUCCESS;
@ -359,7 +359,7 @@ _return:
ctgReleaseTbIndexToCache(pCtg, dbCache, pCache);
CTG_CACHE_STAT_INC(tbIndexMissNum, 1);
CTG_CACHE_STAT_INC(numOfIndexMiss, 1);
return TSDB_CODE_SUCCESS;
}
@ -455,7 +455,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta**
*pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize);
if (NULL == *pTableMeta) {
ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta));
@ -583,11 +583,6 @@ _return:
}
int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) {
if (NULL == pCtg->userCache) {
ctgDebug("empty user auth cache, user:%s", user);
goto _return;
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user));
if (NULL == pUser) {
ctgDebug("user not in cache, user:%s", user);
@ -597,7 +592,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE
*inCache = true;
ctgDebug("Got user from cache, user:%s", user);
CTG_CACHE_STAT_INC(userHitNum, 1);
CTG_CACHE_STAT_INC(numOfUserHit, 1);
if (pUser->superUser) {
*pass = true;
@ -626,7 +621,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE
_return:
*inCache = false;
CTG_CACHE_STAT_INC(userMissNum, 1);
CTG_CACHE_STAT_INC(numOfUserMiss, 1);
return TSDB_CODE_SUCCESS;
}
@ -649,7 +644,7 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) {
SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode));
if (NULL == node) {
qError("calloc %d failed", (int32_t)sizeof(SCtgQNode));
CTG_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_RET(TSDB_CODE_OUT_OF_MEMORY);
}
if (operation->syncOp) {
@ -659,12 +654,18 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) {
node->op = operation;
CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
if (gCtgMgmt.queue.stopQueue) {
ctgFreeQNode(node);
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_RET(TSDB_CODE_CTG_EXIT);
}
gCtgMgmt.queue.stopQueue = operation->stopQueue;
gCtgMgmt.queue.tail->next = node;
gCtgMgmt.queue.tail = node;
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
CTG_QUEUE_INC();
CTG_RT_STAT_INC(qNum, 1);
CTG_RT_STAT_INC(numOfOpEnqueue, 1);
tsem_post(&gCtgMgmt.queue.reqSem);
@ -687,7 +688,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId)
SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *p = strchr(dbFName, '.');
@ -720,7 +721,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncOp)
SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *p = strchr(dbFName, '.');
@ -754,7 +755,7 @@ int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId,
SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -786,7 +787,7 @@ int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId,
SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -816,7 +817,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg));
ctgFreeVgInfo(dbInfo);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *p = strchr(dbFName, '.');
@ -851,7 +852,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy
SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *p = strchr(output->dbFName, '.');
@ -883,7 +884,7 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEp
SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateEpsetMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -915,7 +916,7 @@ int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncOp
SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -944,7 +945,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncO
SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg));
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -975,7 +976,7 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) {
SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
@ -996,19 +997,21 @@ _return:
}
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool syncOp) {
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
op->opId = CTG_OP_CLEAR_CACHE;
op->syncOp = syncOp;
op->stopQueue = stopQueue;
SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
msg->freeCtg = freeCtg;
op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
@ -1033,7 +1036,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) {
mgmt->slots = taosMemoryCalloc(1, msgSize);
if (NULL == mgmt->slots) {
qError("calloc %d failed", (int32_t)msgSize);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum);
@ -1053,13 +1056,13 @@ int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size)
slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size);
if (NULL == slot->meta) {
qError("taosArrayInit %d failed, id:0x%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
if (NULL == taosArrayPush(slot->meta, meta)) {
qError("taosArrayPush meta to rent failed, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
slot->needSort = true;
@ -1177,7 +1180,7 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_
*res = taosMemoryMalloc(msize);
if (NULL == *res) {
qError("malloc %d failed", (int32_t)msize);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
void *meta = taosArrayGet(slot->meta, 0);
@ -1227,13 +1230,13 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache) {
ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.stbCache) {
ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
@ -1244,10 +1247,10 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
}
ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_CACHE_STAT_INC(dbNum, 1);
CTG_CACHE_STAT_INC(numOfDb, 1);
SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
@ -1308,7 +1311,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
CTG_CACHE_STAT_DEC(dbNum, 1);
CTG_CACHE_STAT_DEC(numOfDb, 1);
ctgInfo("db removed from cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId);
return TSDB_CODE_SUCCESS;
@ -1405,7 +1408,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid);
} else {
CTG_CACHE_STAT_DEC(stblNum, 1);
CTG_CACHE_STAT_DEC(numOfStb, 1);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid);
}
@ -1419,7 +1422,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) {
taosMemoryFree(meta);
ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName));
@ -1429,7 +1432,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
}
if (NULL == orig) {
CTG_CACHE_STAT_INC(tblNum, 1);
CTG_CACHE_STAT_INC(numOfTbl, 1);
}
ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
@ -1441,10 +1444,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam
if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) {
ctgError("taosHashPut to stable cache failed, suid:0x%"PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_CACHE_STAT_INC(stblNum, 1);
CTG_CACHE_STAT_INC(numOfStb, 1);
ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType);
@ -1472,7 +1475,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa
ctgFreeSTableIndex(*index);
taosMemoryFreeClear(*index);
ctgError("taosHashPut new tbCache failed, tbName:%s", tbName);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
*index = NULL;
@ -1520,6 +1523,41 @@ _return:
CTG_RET(code);
}
void ctgClearAllInstance(void) {
SCatalog* pCtg = NULL;
void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog**)pIter;
if (pCtg) {
ctgClearHandle(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
}
taosHashClear(gCtgMgmt.pCluster);
}
void ctgFreeAllInstance(void) {
SCatalog* pCtg = NULL;
void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog**)pIter;
if (pCtg) {
ctgFreeHandle(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
}
taosHashClear(gCtgMgmt.pCluster);
}
int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateVgMsg *msg = operation->data;
@ -1534,7 +1572,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d",
dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
bool newAdded = false;
@ -1714,13 +1752,13 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) {
ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else {
CTG_CACHE_STAT_DEC(stblNum, 1);
CTG_CACHE_STAT_DEC(numOfStb, 1);
}
if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) {
ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
} else {
CTG_CACHE_STAT_DEC(tblNum, 1);
CTG_CACHE_STAT_DEC(numOfTbl, 1);
}
ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
@ -1756,7 +1794,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} else {
CTG_CACHE_STAT_DEC(tblNum, 1);
CTG_CACHE_STAT_DEC(numOfTbl, 1);
}
ctgDebug("table %s removed from cache, dbFName:%s", msg->tbName, msg->dbFName);
@ -1772,14 +1810,6 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateUserMsg *msg = operation->data;
SCatalog* pCtg = msg->pCtg;
if (NULL == pCtg->userCache) {
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == pCtg->userCache) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user));
if (NULL == pUser) {
@ -1937,49 +1967,59 @@ int32_t ctgOpClearCache(SCtgCacheOperation *operation) {
SCtgClearCacheMsg *msg = operation->data;
SCatalog* pCtg = msg->pCtg;
CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock);
if (pCtg) {
catalogFreeHandle(pCtg);
if (msg->freeCtg) {
ctgFreeHandle(pCtg);
} else {
ctgClearHandle(pCtg);
}
goto _return;
}
void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog**)pIter;
if (pCtg) {
catalogFreeHandle(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
if (msg->freeCtg) {
ctgFreeAllInstance();
} else {
ctgClearAllInstance();
}
taosHashClear(gCtgMgmt.pCluster);
_return:
CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock);
taosMemoryFreeClear(msg);
CTG_RET(code);
}
void ctgUpdateThreadUnexpectedStopped(void) {
if (!atomic_load_8((int8_t*)&gCtgMgmt.exit) && CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
}
void ctgCleanupCacheQueue(void) {
SCtgQNode *node = NULL;
SCtgQNode *nodeNext = NULL;
SCtgCacheOperation *op = NULL;
bool stopQueue = false;
while (true) {
node = gCtgMgmt.queue.head->next;
while (node) {
if (node->op) {
taosMemoryFree(node->op->data);
if (node->op->syncOp) {
tsem_post(&node->op->rspSem);
op = node->op;
if (op->stopQueue) {
SCatalog *pCtg = ((SCtgUpdateMsgHeader *)op->data)->pCtg;
ctgDebug("process [%s] operation", gCtgCacheOperation[op->opId].name);
(*gCtgCacheOperation[op->opId].func)(op);
stopQueue = true;
CTG_RT_STAT_INC(numOfOpDequeue, 1);
} else {
taosMemoryFree(node->op);
taosMemoryFree(op->data);
CTG_RT_STAT_INC(numOfOpAbort, 1);
}
if (op->syncOp) {
tsem_post(&op->rspSem);
} else {
taosMemoryFree(op);
}
}
@ -1989,7 +2029,7 @@ void ctgCleanupCacheQueue(void) {
node = nodeNext;
}
if (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
if (!stopQueue) {
taosUsleep(1);
} else {
break;
@ -2002,22 +2042,15 @@ void ctgCleanupCacheQueue(void) {
void* ctgUpdateThreadFunc(void* param) {
setThreadName("catalog");
#ifdef WINDOWS
if (taosCheckCurrentInDll()) {
atexit(ctgUpdateThreadUnexpectedStopped);
}
#endif
qInfo("catalog update thread started");
CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
while (true) {
if (tsem_wait(&gCtgMgmt.queue.reqSem)) {
qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
}
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
ctgCleanupCacheQueue();
break;
}
@ -2034,7 +2067,7 @@ void* ctgUpdateThreadFunc(void* param) {
tsem_post(&operation->rspSem);
}
CTG_RT_STAT_INC(qDoneNum, 1);
CTG_RT_STAT_INC(numOfOpDequeue, 1);
ctgdShowCacheInfo();
ctgdShowClusterCache(pCtg);

6
source/libs/catalog/src/ctgDbg.c
View File

@ -19,7 +19,7 @@
#include "catalogInt.h"
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {.cacheEnable = true};
SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true};
void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
ASSERT(*(int32_t*)param == 1);
@ -255,8 +255,8 @@ int32_t ctgdEnableDebug(char *option) {
}
int32_t ctgdGetStatNum(char *option, void *res) {
if (0 == strcasecmp(option, "runtime.qDoneNum")) {
*(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
if (0 == strcasecmp(option, "runtime.numOfOpDequeue")) {
*(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.numOfOpDequeue);
return TSDB_CODE_SUCCESS;
}

149
source/libs/catalog/src/ctgUtil.c
View File

@ -52,6 +52,10 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
}
}
void ctgFreeQNode(SCtgQNode *node) {
//TODO
}
void ctgFreeSTableIndex(void *info) {
if (NULL == info) {
return;
@ -141,10 +145,10 @@ void ctgFreeStbMetaCache(SCtgDBCache *dbCache) {
return;
}
int32_t stblNum = taosHashGetSize(dbCache->stbCache);
int32_t stbNum = taosHashGetSize(dbCache->stbCache);
taosHashCleanup(dbCache->stbCache);
dbCache->stbCache = NULL;
CTG_CACHE_STAT_DEC(stblNum, stblNum);
CTG_CACHE_STAT_DEC(numOfStb, stbNum);
}
void ctgFreeTbCacheImpl(SCtgTbCache *pCache) {
@ -168,7 +172,7 @@ void ctgFreeTbCache(SCtgDBCache *dbCache) {
}
taosHashCleanup(dbCache->tbCache);
dbCache->tbCache = NULL;
CTG_CACHE_STAT_DEC(tblNum, tblNum);
CTG_CACHE_STAT_DEC(numOfTbl, tblNum);
}
void ctgFreeVgInfo(SDBVgInfo *vgInfo) {
@ -198,46 +202,109 @@ void ctgFreeDbCache(SCtgDBCache *dbCache) {
ctgFreeTbCache(dbCache);
}
void ctgFreeInstDbCache(SHashObj* pDbCache) {
if (NULL == pDbCache) {
return;
}
int32_t dbNum = taosHashGetSize(pDbCache);
void *pIter = taosHashIterate(pDbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
atomic_store_8(&dbCache->deleted, 1);
ctgFreeDbCache(dbCache);
pIter = taosHashIterate(pDbCache, pIter);
}
void ctgFreeHandle(SCatalog* pCtg) {
taosHashCleanup(pDbCache);
CTG_CACHE_STAT_DEC(numOfDb, dbNum);
}
void ctgFreeInstUserCache(SHashObj* pUserCache) {
if (NULL == pUserCache) {
return;
}
int32_t userNum = taosHashGetSize(pUserCache);
void *pIter = taosHashIterate(pUserCache, NULL);
while (pIter) {
SCtgUserAuth *userCache = pIter;
ctgFreeSCtgUserAuth(userCache);
pIter = taosHashIterate(pUserCache, pIter);
}
taosHashCleanup(pUserCache);
CTG_CACHE_STAT_DEC(numOfUser, userNum);
}
void ctgFreeHandleImpl(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
if (pCtg->dbCache) {
int32_t dbNum = taosHashGetSize(pCtg->dbCache);
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
atomic_store_8(&dbCache->deleted, 1);
ctgFreeDbCache(dbCache);
pIter = taosHashIterate(pCtg->dbCache, pIter);
}
taosHashCleanup(pCtg->dbCache);
CTG_CACHE_STAT_DEC(dbNum, dbNum);
}
if (pCtg->userCache) {
int32_t userNum = taosHashGetSize(pCtg->userCache);
void *pIter = taosHashIterate(pCtg->userCache, NULL);
while (pIter) {
SCtgUserAuth *userCache = pIter;
ctgFreeSCtgUserAuth(userCache);
pIter = taosHashIterate(pCtg->userCache, pIter);
}
taosHashCleanup(pCtg->userCache);
CTG_CACHE_STAT_DEC(userNum, userNum);
}
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
taosMemoryFree(pCtg);
}
void ctgFreeHandle(SCatalog* pCtg) {
if (NULL == pCtg) {
return;
}
uint64_t clusterId = pCtg->clusterId;
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
CTG_CACHE_STAT_DEC(numOfCluster, 1);
taosMemoryFree(pCtg);
ctgInfo("handle freed, culsterId:0x%" PRIx64, clusterId);
}
void ctgClearHandle(SCatalog* pCtg) {
if (NULL == pCtg) {
return;
}
uint64_t clusterId = pCtg->clusterId;
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB);
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE);
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == pCtg->dbCache) {
qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
}
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == pCtg->userCache) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
}
CTG_CACHE_STAT_INC(numOfClear, 1);
ctgInfo("handle cleared, culsterId:0x%" PRIx64, clusterId);
}
void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) {
if (NULL == pOutput) {
return;
@ -590,7 +657,7 @@ int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) {
vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo));
if (NULL == vgList) {
ctgError("taosArrayInit failed, num:%d", vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
void *pIter = taosHashIterate(vgHash, NULL);
@ -600,7 +667,7 @@ int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) {
if (NULL == taosArrayPush(vgList, vgInfo)) {
ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId);
taosHashCancelIterate(vgHash, pIter);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
pIter = taosHashIterate(vgHash, pIter);
@ -717,7 +784,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
*dst = taosMemoryMalloc(sizeof(SDBVgInfo));
if (NULL == *dst) {
qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*dst, src, sizeof(SDBVgInfo));
@ -727,7 +794,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
if (NULL == (*dst)->vgHash) {
qError("taosHashInit %d failed", (int32_t)hashSize);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t *vgId = NULL;
@ -740,7 +807,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
taosHashCancelIterate(src->vgHash, pIter);
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pIter = taosHashIterate(src->vgHash, pIter);
@ -756,7 +823,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput)
*pOutput = taosMemoryMalloc(sizeof(STableMetaOutput));
if (NULL == *pOutput) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*pOutput, output, sizeof(STableMetaOutput));
@ -767,7 +834,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput)
if (NULL == (*pOutput)->tbMeta) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
taosMemoryFreeClear(*pOutput);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize);

18
source/libs/catalog/test/catalogTests.cpp
View File

@ -1386,7 +1386,7 @@ TEST(tableMeta, updateStbMeta) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n != 3) {
taosMsleep(50);
} else {
@ -1456,7 +1456,7 @@ TEST(refreshGetMeta, normal2normal) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -1535,7 +1535,7 @@ TEST(refreshGetMeta, normal2notexist) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -1609,7 +1609,7 @@ TEST(refreshGetMeta, normal2child) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -1693,7 +1693,7 @@ TEST(refreshGetMeta, stable2child) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -1778,7 +1778,7 @@ TEST(refreshGetMeta, stable2stable) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -1866,7 +1866,7 @@ TEST(refreshGetMeta, child2stable) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -2083,7 +2083,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n > 0) {
break;
}
@ -2109,7 +2109,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
while (true) {
uint64_t n = 0;
ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
if (n != 3) {
taosMsleep(50);
} else {

262
source/libs/command/src/command.c
View File

@ -18,7 +18,31 @@
#include "tdatablock.h"
#include "tglobal.h"
extern SConfig *tsCfg;
extern SConfig* tsCfg;
static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(pBlock->info.rows);
(*pRsp)->numOfCols = htonl(numOfCols);
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, numOfCols, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
return TSDB_CODE_SUCCESS;
}
static int32_t getSchemaBytes(const SSchema* pSchema) {
switch (pSchema->type) {
case TSDB_DATA_TYPE_BINARY:
@ -89,33 +113,13 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = buildDescResultDataBlock();
setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(numOfRows);
(*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
return TSDB_CODE_SUCCESS;
return buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);
}
static int32_t execResetQueryCache() { return catalogClearCache(); }
static SSDataBlock* buildCreateDBResultDataBlock() {
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = createDataBlock();
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
blockDataAppendColInfo(pBlock, &infoData);
@ -149,14 +153,14 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
return v;
}
char *buildRetension(SArray *pRetension) {
char* buildRetension(SArray* pRetension) {
size_t size = taosArrayGetSize(pRetension);
if (size == 0) {
return NULL;
}
char *p1 = taosMemoryCalloc(1, 100);
SRetention *p = taosArrayGet(pRetension, 0);
char* p1 = taosMemoryCalloc(1, 100);
SRetention* p = taosArrayGet(pRetension, 0);
int32_t len = 0;
@ -185,8 +189,7 @@ char *buildRetension(SArray *pRetension) {
return p1;
}
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, SDbCfgInfo* pCfg) {
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1;
@ -198,7 +201,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0};
int32_t len = 0;
char *prec = NULL;
char* prec = NULL;
switch (pCfg->precision) {
case TSDB_TIME_PRECISION_MILLI:
prec = TSDB_TIME_PRECISION_MILLI_STR;
@ -214,15 +217,16 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
break;
}
char *retentions = buildRetension(pCfg->pRetensions);
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm "
"FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d",
dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile,
pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables);
char* retentions = buildRetension(pCfg->pRetensions);
len += sprintf(buf2 + VARSTR_HEADER_SIZE,
"CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm "
"FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d",
dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod,
pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages,
pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);
@ -230,35 +234,14 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S
}
(varDataLen(buf2)) = len;
colDataAppend(pCol2, 0, buf2, false);
}
static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = buildCreateDBResultDataBlock();
setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg);
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(1);
(*pRsp)->numOfCols = htonl(SHOW_CREATE_DB_RESULT_COLS);
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
return TSDB_CODE_SUCCESS;
return buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp);
}
static SSDataBlock* buildCreateTbResultDataBlock() {
@ -276,14 +259,14 @@ static SSDataBlock* buildCreateTbResultDataBlock() {
void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
SSchema* pSchema = pCfg->pSchemas + i;
char type[32];
char type[32];
sprintf(type, "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE));
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
}
}
@ -291,19 +274,18 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
char type[32];
char type[32];
sprintf(type, "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE));
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
}
}
void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
@ -311,13 +293,12 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
}
}
int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
SArray *pTagVals = NULL;
STag *pTag = (STag*)pCfg->pTags;
SArray* pTagVals = NULL;
STag* pTag = (STag*)pCfg->pTags;
if (pCfg->pTags && pTag->flags & TD_TAG_JSON) {
char *pJson = parseTagDatatoJson(pTag);
char* pJson = parseTagDatatoJson(pTag);
if (pJson) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
taosMemoryFree(pJson);
@ -325,8 +306,8 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
return TSDB_CODE_SUCCESS;
}
int32_t code = tTagToValArray((const STag *)pCfg->pTags, &pTagVals);
int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals);
if (code) {
return code;
}
@ -339,20 +320,20 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
if (i > 0) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", ");
}
if (j >= valueNum) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL");
continue;
}
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, j);
STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
if (pSchema->colId > pTagVal->cid) {
qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid);
taosArrayDestroy(pTagVals);
return TSDB_CODE_APP_ERROR;
} else if (pSchema->colId == pTagVal->cid) {
char type = pTagVal->type;
int32_t tlen = 0;
char type = pTagVal->type;
int32_t tlen = 0;
if (IS_VAR_DATA_TYPE(type)) {
dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen);
@ -364,7 +345,6 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
} else {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL");
}
/*
if (type == TSDB_DATA_TYPE_BINARY) {
@ -372,7 +352,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
if (num) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", ");
}
memcpy(buf + VARSTR_HEADER_SIZE + *len, pTagVal->pData, pTagVal->nData);
*len += pTagVal->nData;
}
@ -397,7 +377,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
taosArrayDestroy(pTagVals);
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) {
@ -426,7 +406,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) {
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP(");
for (int32_t i = 0; i < funcNum; ++i) {
char* pFunc = taosArrayGet(pCfg->pFuncs, i);
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc);
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc);
}
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")");
}
@ -436,7 +416,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) {
}
}
static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, STableCfg* pCfg) {
static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char* tbName, STableCfg* pCfg) {
int32_t code = 0;
blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1;
@ -454,7 +434,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName,
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName);
appendColumnFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS (");
appendTagFields(buf2, &len, pCfg);
appendTagFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
appendTableOptions(buf2, &len, pCfg);
} else if (TSDB_CHILD_TABLE == pCfg->tableType) {
@ -474,40 +454,19 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName,
}
varDataLen(buf2) = len;
colDataAppend(pCol2, 0, buf2, false);
return TSDB_CODE_SUCCESS;
}
static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = buildCreateTbResultDataBlock();
int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg);
int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg);
if (code) {
return code;
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(1);
(*pRsp)->numOfCols = htonl(SHOW_CREATE_TB_RESULT_COLS);
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
return TSDB_CODE_SUCCESS;
return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp);
}
static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
@ -516,17 +475,17 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR
terrno = TSDB_CODE_TSC_NOT_STABLE_ERROR;
return terrno;
}
return execShowCreateTable(pStmt, pRsp);
}
static int32_t execAlterLocal(SAlterLocalStmt* pStmt) {
static int32_t execAlterLocal(SAlterLocalStmt* pStmt) {
if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) {
return terrno;
return terrno;
}
if (taosSetCfg(tsCfg, pStmt->config)) {
return terrno;
return terrno;
}
return TSDB_CODE_SUCCESS;
@ -551,21 +510,20 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() {
return pBlock;
}
int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
int32_t numOfCfg = taosArrayGetSize(tsCfg->array);
int32_t numOfRows = 0;
blockDataEnsureCapacity(pBlock, numOfCfg);
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
SConfigItem *pItem = taosArrayGet(tsCfg->array, i);
SConfigItem* pItem = taosArrayGet(tsCfg->array, i);
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
colDataAppend(pColInfo, i, name, false);
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t valueLen = 0;
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
varDataSetLen(value, valueLen);
@ -575,42 +533,70 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
numOfRows++;
}
pBlock->info.rows = numOfRows;
return TSDB_CODE_SUCCESS;
}
static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = buildLocalVariablesResultDataBlock();
int32_t code = setLocalVariablesResultIntoDataBlock(pBlock);
int32_t code = setLocalVariablesResultIntoDataBlock(pBlock);
if (code) {
return code;
}
return buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp);
}
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
*pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) {
static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pRsp)->useconds = 0;
(*pRsp)->completed = 1;
(*pRsp)->precision = 0;
(*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htonl(pBlock->info.rows);
(*pRsp)->numOfCols = htonl(SHOW_LOCAL_VARIABLES_RESULT_COLS);
pBlock->pDataBlock = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SColumnInfoData));
int32_t len = 0;
blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
blockDataDestroy(pBlock);
SNode* pProj = NULL;
FOREACH(pProj, pProjects) {
SColumnInfoData infoData = {0};
infoData.info.type = ((SExprNode*)pProj)->resType.type;
infoData.info.bytes = ((SExprNode*)pProj)->resType.bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
}
*pOutput = pBlock;
return TSDB_CODE_SUCCESS;
}
int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
int32_t numOfCols = LIST_LENGTH(pProjects);
blockDataEnsureCapacity(pBlock, 1);
int32_t index = 0;
SNode* pProj = NULL;
FOREACH(pProj, pProjects) {
if (((SValueNode*)pProj)->isNull) {
colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true);
} else {
colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false);
}
}
pBlock->info.rows = 1;
return TSDB_CODE_SUCCESS;
}
static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = createSelectResultDataBlock(pSelect->pProjectionList, &pBlock);
if (TSDB_CODE_SUCCESS == code) {
code = buildSelectResultDataBlock(pSelect->pProjectionList, pBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp);
}
return code;
}
int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) {
switch (nodeType(pStmt)) {
case QUERY_NODE_DESCRIBE_STMT:
@ -627,6 +613,8 @@ int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) {
return execAlterLocal((SAlterLocalStmt*)pStmt);
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
return execShowLocalVariables(pRsp);
case QUERY_NODE_SELECT_STMT:
return execSelectWithoutFrom((SSelectStmt*)pStmt, pRsp);
default:
break;
}

44
source/libs/executor/inc/executorimpl.h
View File

@ -253,18 +253,15 @@ typedef struct STableScanInfo {
SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder;
int64_t numOfRows;
SScanInfo scanInfo;
int32_t scanTimes;
SNode* pFilterNode; // filter info, which is push down by optimizer
SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context
SResultRowInfo* pResultRowInfo;
int32_t* rowEntryInfoOffset;
SExprInfo* pExpr;
SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup
int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup
SExprInfo* pExpr;// todo: remove this by using SExprSup
SSDataBlock* pResBlock;
SArray* pColMatchInfo;
int32_t numOfOutput;
SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
@ -275,8 +272,13 @@ typedef struct STableScanInfo {
int32_t curTWinIdx;
int32_t currentGroupId;
uint64_t queryId;
uint64_t taskId;
uint64_t queryId; // todo remove it
uint64_t taskId; // todo remove it
struct {
uint64_t uid;
int64_t t;
} scanStatus;
} STableScanInfo;
typedef struct STagScanInfo {
@ -321,31 +323,31 @@ typedef struct SessionWindowSupporter {
} SessionWindowSupporter;
typedef struct SStreamBlockScanInfo {
uint64_t tableUid; // queried super table uid
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id
SReadHandle readHandle;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SArray* pColMatchInfo; //
SNode* pCondition;
SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock
SSDataBlock* pUpdateRes; // update SSDataBlock
int32_t updateResIndex;
int32_t blockType; // current block type
int32_t validBlockIndex; // Is current data has returned?
SColumnInfo* pCols; // the output column info
uint64_t numOfExec; // execution times
void* streamBlockReader;// stream block reader handle
SArray* pColMatchInfo; //
SNode* pCondition;
int32_t tsArrayIndex;
SArray* tsArray;
uint64_t groupId;
SUpdateInfo* pUpdateInfo;
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id
SReadHandle readHandle;
uint64_t tableUid; // queried super table uid
EStreamScanMode scanMode;
SOperatorInfo* pSnapshotReadOp;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SArray* childIds;
SessionWindowSupporter sessionSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
@ -683,7 +685,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSup(SExprSupp* pSup);
void cleanupExprSupp(SExprSupp* pSup);
int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
@ -707,7 +709,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
void cleanupExecSupp(SExprSupp* pSupp);
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
SSDataBlock* loadNextDataBlock(void* param);

17
source/libs/executor/src/executorMain.c
View File

@ -191,16 +191,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
return TSDB_CODE_SUCCESS;
}
int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
return isTaskKilled(pTaskInfo);
}
void qDestroyTask(qTaskInfo_t qTaskHandle) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
@ -242,3 +232,10 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
}
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo;
return TSDB_CODE_SUCCESS;
}

27
source/libs/executor/src/executorimpl.c
View File

@ -273,8 +273,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
}
// 1. close current opened time window
if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId &&
pResult->offset != pResultRowInfo->cur.offset))) {
if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) {
SResultRowPosition pos = pResultRowInfo->cur;
SFilePage* pPage = getBufPage(pResultBuf, pos.pageId);
releaseBufPage(pResultBuf, pPage);
@ -1034,7 +1033,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
uint32_t status = BLK_DATA_NOT_LOAD;
int32_t numOfOutput = pTableScanInfo->numOfOutput;
int32_t numOfOutput = 0;//pTableScanInfo->numOfOutput;
for (int32_t i = 0; i < numOfOutput; ++i) {
int32_t functionId = pCtx[i].functionId;
int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
@ -2822,6 +2821,24 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
}
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamBlockScanInfo* pScanInfo = pOperator->info;
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
*uid = pSnapShotScanInfo->scanStatus.uid;
*ts = pSnapShotScanInfo->scanStatus.t;
} else {
if (pOperator->pDownstream[0] == NULL) {
return TSDB_CODE_INVALID_PARA;
} else {
doGetScanStatus(pOperator->pDownstream[0], uid, ts);
}
}
return TSDB_CODE_SUCCESS;
}
// this is a blocking operator
static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
if (OPTR_IS_OPENED(pOperator)) {
@ -3544,7 +3561,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pPseudoColInfo);
}
void cleanupExecSupp(SExprSupp* pSupp) {
void cleanupExprSupp(SExprSupp* pSupp) {
destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs);
destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
@ -3557,7 +3574,7 @@ static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pPseudoColInfo);
cleanupAggSup(&pInfo->aggSup);
cleanupExecSupp(&pInfo->scalarSup);
cleanupExprSupp(&pInfo->scalarSup);
}
void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {

4
source/libs/executor/src/groupoperator.c
View File

@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) {
taosMemoryFreeClear(pInfo->keyBuf);
taosArrayDestroy(pInfo->pGroupCols);
taosArrayDestroy(pInfo->pGroupColVals);
cleanupExecSupp(&pInfo->scalarSup);
cleanupExprSupp(&pInfo->scalarSup);
}
static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) {
@ -701,7 +701,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) {
taosHashCleanup(pInfo->pGroupSet);
taosMemoryFree(pInfo->columnOffset);
cleanupExecSupp(&pInfo->scalarSup);
cleanupExprSupp(&pInfo->scalarSup);
}
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {

8
source/libs/executor/src/scanoperator.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <vnode.h>
#include "filter.h"
#include "function.h"
@ -413,6 +414,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime;
// todo refactor
pTableScanInfo->scanStatus.uid = pBlock->info.uid;
pTableScanInfo->scanStatus.t = pBlock->info.window.ekey;
return pBlock;
}
return NULL;
@ -459,7 +465,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0);
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
}

5
source/libs/function/inc/builtinsimpl.h
View File

@ -191,6 +191,11 @@ bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool uniqueFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t uniqueFunction(SqlFunctionCtx *pCtx);
bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool modeFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t modeFunction(SqlFunctionCtx *pCtx);
int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t twaFunction(SqlFunctionCtx *pCtx);

1
source/libs/function/inc/functionMgtInt.h
View File

@ -46,6 +46,7 @@ extern "C" {
#define FUNC_MGT_FORBID_STREAM_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(17)
#define FUNC_MGT_FORBID_WINDOW_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18)
#define FUNC_MGT_FORBID_GROUP_BY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19)
#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

123
source/libs/function/src/builtins.c
View File

@ -1045,20 +1045,29 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3
return translateFirstLastImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isUnique) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
if (!nodesExprHasColumn(pPara)) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE must contain columns");
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns",
isUnique ? "UNIQUE" : "MODE");
}
pFunc->node.resType = ((SExprNode*)pPara)->resType;
return TSDB_CODE_SUCCESS;
}
static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateUniqueMode(pFunc, pErrBuf, len, true);
}
static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateUniqueMode(pFunc, pErrBuf, len, false);
}
static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (numOfParams == 0 || numOfParams > 2) {
@ -1220,19 +1229,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// The number of parameters has been limited by the syntax definition
//uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
// uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
// The function return type has been set during syntax parsing
uint8_t para2Type = pFunc->node.resType.type;
//if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
// para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
// para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
//}
//if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
// (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
//}
// if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
// para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
// para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
// }
// if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
// (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
// }
int32_t para2Bytes = pFunc->node.resType.bytes;
if (IS_VAR_DATA_TYPE(para2Type)) {
@ -1530,6 +1539,36 @@ static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return TSDB_CODE_SUCCESS;
}
static int32_t translateDatabaseFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = TSDB_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
static int32_t translateClientVersionFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = TSDB_VERSION_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
static int32_t translateServerVersionFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = TSDB_VERSION_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
static int32_t translateServerStatusFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateCurrentUserFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = TSDB_USER_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
static int32_t translateUserFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = TSDB_USER_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
// clang-format off
const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
@ -1882,7 +1921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "first",
.type = FUNCTION_TYPE_FIRST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -1895,7 +1934,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_first_partial",
.type = FUNCTION_TYPE_FIRST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastPartial,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -1906,7 +1945,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_first_merge",
.type = FUNCTION_TYPE_FIRST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -1917,7 +1956,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "last",
.type = FUNCTION_TYPE_LAST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -1930,7 +1969,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_last_partial",
.type = FUNCTION_TYPE_LAST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastPartial,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -1941,7 +1980,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_last_merge",
.type = FUNCTION_TYPE_LAST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -2109,7 +2148,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "unique",
.type = FUNCTION_TYPE_UNIQUE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC,
.translateFunc = translateUnique,
.getEnvFunc = getUniqueFuncEnv,
@ -2117,6 +2156,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = uniqueFunction,
.finalizeFunc = NULL
},
{
.name = "mode",
.type = FUNCTION_TYPE_MODE,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateMode,
.getEnvFunc = getModeFuncEnv,
.initFunc = modeFunctionSetup,
.processFunc = modeFunction,
.finalizeFunc = modeFinalize,
},
{
.name = "abs",
.type = FUNCTION_TYPE_ABS,
@ -2527,6 +2576,42 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.pPartialFunc = "_group_key",
.pMergeFunc = "_group_key"
},
{
.name = "database",
.type = FUNCTION_TYPE_DATABASE,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateDatabaseFunc,
},
{
.name = "client_version",
.type = FUNCTION_TYPE_CLIENT_VERSION,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateClientVersionFunc,
},
{
.name = "server_version",
.type = FUNCTION_TYPE_SERVER_VERSION,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateServerVersionFunc,
},
{
.name = "server_status",
.type = FUNCTION_TYPE_SERVER_STATUS,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateServerStatusFunc,
},
{
.name = "current_user",
.type = FUNCTION_TYPE_CURRENT_USER,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateCurrentUserFunc,
},
{
.name = "user",
.type = FUNCTION_TYPE_USER,
.classification = FUNC_MGT_SYSTEM_INFO_FUNC | FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateUserFunc,
},
};
// clang-format on

186
source/libs/function/src/builtinsimpl.c
View File

@ -32,6 +32,7 @@
#define TAIL_MAX_OFFSET 100
#define UNIQUE_MAX_RESULT_SIZE (1024 * 1024 * 10)
#define MODE_MAX_RESULT_SIZE UNIQUE_MAX_RESULT_SIZE
#define HLL_BUCKET_BITS 14 // The bits of the bucket
#define HLL_DATA_BITS (64 - HLL_BUCKET_BITS)
@ -246,6 +247,19 @@ typedef struct SUniqueInfo {
char pItems[];
} SUniqueInfo;
typedef struct SModeItem {
int64_t count;
char data[];
} SModeItem;
typedef struct SModeInfo {
int32_t numOfPoints;
uint8_t colType;
int16_t colBytes;
SHashObj* pHash;
char pItems[];
} SModeInfo;
typedef struct SDerivInfo {
double prevValue; // previous value
TSKEY prevTs; // previous timestamp
@ -2395,11 +2409,11 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
return TSDB_CODE_SUCCESS;
}
int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); }
int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); }
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t);
pEnv->calcMemSize = getFirstLastInfoSize(pNode->node.resType.bytes);
return true;
}
@ -2477,9 +2491,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
}
memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1;
break;
}
@ -2511,8 +2533,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
}
memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1;
break;
}
@ -2566,8 +2597,17 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
}
memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true;
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1;
}
break;
@ -2589,9 +2629,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
}
memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true;
pResInfo->numOfRes = 1;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
}
break;
}
@ -2601,7 +2650,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
SInputColumnInfoData* pColInfo = &pCtx->input;
int32_t start = pColInfo->startRowIndex;
pOutput->bytes = pInput->bytes;
TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes);
@ -2618,7 +2670,17 @@ static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput,
}
*tsOut = *tsIn;
memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY));
if (pCtx->subsidiaries.num > 0) {
if (!pOutput->hasResult) {
saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
}
}
pOutput->hasResult = true;
return;
}
@ -2633,7 +2695,7 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer
char* data = colDataGetData(pCol, start);
SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data);
firstLastTransferInfo(pInputInfo, pInfo, isFirstQuery);
firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery);
int32_t numOfElems = pInputInfo->hasResult ? 1 : 0;
@ -2655,6 +2717,9 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
return pResInfo->numOfRes;
}
@ -2673,6 +2738,9 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
colDataAppend(pCol, pBlock->info.rows, res, false);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
taosMemoryFree(res);
return 1;
@ -3029,7 +3097,9 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
pItem->uid = uid;
// save the data of this tuple
saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
if (pCtx->subsidiaries.num > 0) {
saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
}
// allocate the buffer and keep the data of this row into the new allocated buffer
pEntryInfo->numOfRes++;
@ -3048,7 +3118,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
pItem->uid = uid;
// save the data of this tuple by over writing the old data
copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
if (pCtx->subsidiaries.num > 0) {
copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
}
taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type,
topBotResComparFn, NULL, !isTopQuery);
}
@ -4694,21 +4767,100 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) {
return pInfo->numOfPoints;
}
int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SModeInfo) + MODE_MAX_RESULT_SIZE;
return true;
}
bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
if (!functionSetup(pCtx, pResInfo)) {
return false;
}
SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->numOfPoints = 0;
pInfo->colType = pCtx->resDataInfo.type;
pInfo->colBytes = pCtx->resDataInfo.bytes;
if (pInfo->pHash != NULL) {
taosHashClear(pInfo->pHash);
} else {
pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
return true;
}
static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) {
// ignore null elements
if (isNull) {
return;
}
int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes);
if (pHashItem == NULL) {
int32_t size = sizeof(SModeItem) + pInfo->colBytes;
SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size);
memcpy(pItem->data, data, pInfo->colBytes);
pItem->count += 1;
taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*));
pInfo->numOfPoints++;
} else {
(*pHashItem)->count += 1;
}
}
int32_t modeFunction(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
int32_t startOffset = pCtx->offset;
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
char* data = colDataGetData(pInputCol, i);
doModeAdd(pInfo, data, colDataIsNull_s(pInputCol, i));
if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) {
taosHashCleanup(pInfo->pHash);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
SET_VAL(pResInfo, 1, 1);
return TSDB_CODE_SUCCESS;
}
int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t currentRow = pBlock->info.rows;
for (int32_t i = 0; i < pResInfo->numOfRes; ++i) {
SUniqueItem* pItem = (SUniqueItem*)(pInfo->pItems + i * (sizeof(SUniqueItem) + pInfo->colBytes));
colDataAppend(pCol, i, pItem->data, false);
// TODO: handle ts output
int32_t resIndex;
int32_t maxCount = 0;
for (int32_t i = 0; i < pInfo->numOfPoints; ++i) {
SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes));
if (pItem->count > maxCount) {
maxCount = pItem->count;
resIndex = i;
} else if (pItem->count == maxCount) {
resIndex = -1;
}
}
SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes));
colDataAppend(pCol, currentRow, pResItem->data, (resIndex == -1) ? true : false);
return pResInfo->numOfRes;
}
bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(STwaInfo);
return true;

2
source/libs/function/src/functionMgt.c
View File

@ -179,6 +179,8 @@ bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId
bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); }
bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); }
bool fmIsInterpFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;

6
source/libs/function/test/udf1.c
View File

@ -2,12 +2,8 @@
#include <stdlib.h>
#include <stdio.h>
#include "tudf.h"
#include "taosudf.h"
#undef malloc
#define malloc malloc
#undef free
#define free free
DLL_EXPORT int32_t udf1_init() {
return 0;

8
source/libs/function/test/udf2.c
View File

@ -1,13 +1,9 @@
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include "tudf.h"
#undef malloc
#define malloc malloc
#undef free
#define free free
#include "taosudf.h"
DLL_EXPORT int32_t udf2_init() {
return 0;

1
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -100,6 +100,7 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
COPY_OBJECT_FIELD(resType, sizeof(SDataType));
COPY_CHAR_ARRAY_FIELD(aliasName);
COPY_CHAR_ARRAY_FIELD(userAlias);
COPY_SCALAR_FIELD(orderAlias);
return TSDB_CODE_SUCCESS;
}

3
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1500,7 +1500,8 @@ typedef struct SCollectFuncsCxt {
static EDealRes collectFuncs(SNode* pNode, void* pContext) {
SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) {
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId) &&
!(((SExprNode*)pNode)->orderAlias)) {
pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
}

6
source/libs/parser/inc/sql.y
View File

@ -685,6 +685,12 @@ literal_func(A) ::= NOW(B).
noarg_func(A) ::= NOW(B). { A = B; }
noarg_func(A) ::= TODAY(B). { A = B; }
noarg_func(A) ::= TIMEZONE(B). { A = B; }
noarg_func(A) ::= DATABASE(B). { A = B; }
noarg_func(A) ::= CLIENT_VERSION(B). { A = B; }
noarg_func(A) ::= SERVER_VERSION(B). { A = B; }
noarg_func(A) ::= SERVER_STATUS(B). { A = B; }
noarg_func(A) ::= CURRENT_USER(B). { A = B; }
noarg_func(A) ::= USER(B). { A = B; }
%type star_func { SToken }
%destructor star_func { }

27
source/libs/parser/src/parInsert.c
View File

@ -67,6 +67,13 @@ typedef struct SInsertParseContext {
SParseMetaCache* pMetaCache;
} SInsertParseContext;
typedef struct SInsertParseSyntaxCxt {
SParseContext* pComCxt;
char* pSql;
SMsgBuf msg;
SParseMetaCache* pMetaCache;
} SInsertParseSyntaxCxt;
typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
@ -1098,11 +1105,24 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName*
return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES);
}
static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt);
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) {
SToken sToken;
NEXT_TOKEN(pCxt->pSql, sToken);
SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL};
int32_t code = skipUsingClause(&cxt);
pCxt->pSql = cxt.pSql;
return code;
}
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tbFName) {
int32_t len = strlen(tbFName);
STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len);
if (NULL != pMeta) {
CHECK_CODE(ignoreAutoCreateTableClause(pCxt));
return cloneTableMeta(*pMeta, &pCxt->pTableMeta);
}
@ -1522,13 +1542,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
return code;
}
typedef struct SInsertParseSyntaxCxt {
SParseContext* pComCxt;
char* pSql;
SMsgBuf msg;
SParseMetaCache* pMetaCache;
} SInsertParseSyntaxCxt;
static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
SToken sToken;
int32_t expectRightParenthesis = 1;

418
source/libs/parser/src/parTokenizer.c
View File

@ -29,213 +29,217 @@ typedef struct SKeyword {
// clang-format off
// keywords in sql string
static SKeyword keywordTable[] = {
{"ACCOUNT", TK_ACCOUNT},
{"ACCOUNTS", TK_ACCOUNTS},
{"ADD", TK_ADD},
{"AGGREGATE", TK_AGGREGATE},
{"ALL", TK_ALL},
{"ALTER", TK_ALTER},
{"ANALYZE", TK_ANALYZE},
{"AND", TK_AND},
{"APPS", TK_APPS},
{"AS", TK_AS},
{"ASC", TK_ASC},
{"AT_ONCE", TK_AT_ONCE},
{"BALANCE", TK_BALANCE},
{"BETWEEN", TK_BETWEEN},
{"BINARY", TK_BINARY},
{"BIGINT", TK_BIGINT},
{"BNODE", TK_BNODE},
{"BNODES", TK_BNODES},
{"BOOL", TK_BOOL},
{"BUFFER", TK_BUFFER},
{"BUFSIZE", TK_BUFSIZE},
{"BY", TK_BY},
{"CACHE", TK_CACHE},
{"CACHELAST", TK_CACHELAST},
{"CAST", TK_CAST},
{"CLUSTER", TK_CLUSTER},
{"COLUMN", TK_COLUMN},
{"COMMENT", TK_COMMENT},
{"COMP", TK_COMP},
{"COMPACT", TK_COMPACT},
{"CONNS", TK_CONNS},
{"CONNECTION", TK_CONNECTION},
{"CONNECTIONS", TK_CONNECTIONS},
{"CONSUMER", TK_CONSUMER},
{"CONSUMERS", TK_CONSUMERS},
{"COUNT", TK_COUNT},
{"CREATE", TK_CREATE},
{"CONTAINS", TK_CONTAINS},
{"DATABASE", TK_DATABASE},
{"DATABASES", TK_DATABASES},
{"DBS", TK_DBS},
{"DELETE", TK_DELETE},
{"DESC", TK_DESC},
{"DESCRIBE", TK_DESCRIBE},
{"DISTINCT", TK_DISTINCT},
{"DISTRIBUTED", TK_DISTRIBUTED},
{"DNODE", TK_DNODE},
{"DNODES", TK_DNODES},
{"DOUBLE", TK_DOUBLE},
{"DROP", TK_DROP},
{"DURATION", TK_DURATION},
{"ENABLE", TK_ENABLE},
{"EXISTS", TK_EXISTS},
{"EXPLAIN", TK_EXPLAIN},
{"EVERY", TK_EVERY},
{"FILL", TK_FILL},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT},
{"FROM", TK_FROM},
{"FSYNC", TK_FSYNC},
{"FUNCTION", TK_FUNCTION},
{"FUNCTIONS", TK_FUNCTIONS},
{"GRANT", TK_GRANT},
{"GRANTS", TK_GRANTS},
{"GROUP", TK_GROUP},
{"HAVING", TK_HAVING},
{"IF", TK_IF},
{"IMPORT", TK_IMPORT},
{"IN", TK_IN},
{"INDEX", TK_INDEX},
{"INDEXES", TK_INDEXES},
{"INNER", TK_INNER},
{"INT", TK_INT},
{"INSERT", TK_INSERT},
{"INTEGER", TK_INTEGER},
{"INTERVAL", TK_INTERVAL},
{"INTO", TK_INTO},
{"IS", TK_IS},
{"JOIN", TK_JOIN},
{"JSON", TK_JSON},
{"KEEP", TK_KEEP},
{"KILL", TK_KILL},
{"LAST", TK_LAST},
{"LAST_ROW", TK_LAST_ROW},
{"LICENCE", TK_LICENCE},
{"LIKE", TK_LIKE},
{"LIMIT", TK_LIMIT},
{"LINEAR", TK_LINEAR},
{"LOCAL", TK_LOCAL},
{"MATCH", TK_MATCH},
{"MAXROWS", TK_MAXROWS},
{"MAX_DELAY", TK_MAX_DELAY},
{"MERGE", TK_MERGE},
{"META", TK_META},
{"MINROWS", TK_MINROWS},
{"MINUS", TK_MINUS},
{"MNODE", TK_MNODE},
{"MNODES", TK_MNODES},
{"MODIFY", TK_MODIFY},
{"MODULES", TK_MODULES},
{"NCHAR", TK_NCHAR},
{"NEXT", TK_NEXT},
{"NMATCH", TK_NMATCH},
{"NONE", TK_NONE},
{"NOT", TK_NOT},
{"NOW", TK_NOW},
{"NULL", TK_NULL},
{"NULLS", TK_NULLS},
{"OFFSET", TK_OFFSET},
{"ON", TK_ON},
{"OR", TK_OR},
{"ORDER", TK_ORDER},
{"OUTPUTTYPE", TK_OUTPUTTYPE},
{"PARTITION", TK_PARTITION},
{"PASS", TK_PASS},
{"PAGES", TK_PAGES},
{"PAGESIZE", TK_PAGESIZE},
{"PORT", TK_PORT},
{"PPS", TK_PPS},
{"PRECISION", TK_PRECISION},
// {"PRIVILEGE", TK_PRIVILEGE},
{"PREV", TK_PREV},
{"QNODE", TK_QNODE},
{"QNODES", TK_QNODES},
{"QTIME", TK_QTIME},
{"QUERIES", TK_QUERIES},
{"QUERY", TK_QUERY},
{"RANGE", TK_RANGE},
{"RATIO", TK_RATIO},
{"READ", TK_READ},
{"REDISTRIBUTE", TK_REDISTRIBUTE},
{"RENAME", TK_RENAME},
{"REPLICA", TK_REPLICA},
{"RESET", TK_RESET},
{"RETENTIONS", TK_RETENTIONS},
{"REVOKE", TK_REVOKE},
{"ROLLUP", TK_ROLLUP},
{"SCHEMALESS", TK_SCHEMALESS},
{"SCORES", TK_SCORES},
{"SELECT", TK_SELECT},
{"SESSION", TK_SESSION},
{"SET", TK_SET},
{"SHOW", TK_SHOW},
{"SINGLE_STABLE", TK_SINGLE_STABLE},
{"SLIDING", TK_SLIDING},
{"SLIMIT", TK_SLIMIT},
{"SMA", TK_SMA},
{"SMALLINT", TK_SMALLINT},
{"SNODE", TK_SNODE},
{"SNODES", TK_SNODES},
{"SOFFSET", TK_SOFFSET},
{"SPLIT", TK_SPLIT},
{"STABLE", TK_STABLE},
{"STABLES", TK_STABLES},
{"STATE", TK_STATE},
{"STATE_WINDOW", TK_STATE_WINDOW},
{"STORAGE", TK_STORAGE},
{"STREAM", TK_STREAM},
{"STREAMS", TK_STREAMS},
{"STRICT", TK_STRICT},
{"SUBSCRIPTIONS", TK_SUBSCRIPTIONS},
{"SYNCDB", TK_SYNCDB},
{"SYSINFO", TK_SYSINFO},
{"TABLE", TK_TABLE},
{"TABLES", TK_TABLES},
{"TAG", TK_TAG},
{"TAGS", TK_TAGS},
{"TBNAME", TK_TBNAME},
{"TIMESTAMP", TK_TIMESTAMP},
{"TIMEZONE", TK_TIMEZONE},
{"TINYINT", TK_TINYINT},
{"TO", TK_TO},
{"TODAY", TK_TODAY},
{"TOPIC", TK_TOPIC},
{"TOPICS", TK_TOPICS},
{"TRANSACTION", TK_TRANSACTION},
{"TRANSACTIONS", TK_TRANSACTIONS},
{"TRIGGER", TK_TRIGGER},
{"TSERIES", TK_TSERIES},
{"TTL", TK_TTL},
{"UNION", TK_UNION},
{"UNSIGNED", TK_UNSIGNED},
{"USE", TK_USE},
{"USER", TK_USER},
{"USERS", TK_USERS},
{"USING", TK_USING},
{"VALUE", TK_VALUE},
{"VALUES", TK_VALUES},
{"VARCHAR", TK_VARCHAR},
{"VARIABLES", TK_VARIABLES},
{"VERBOSE", TK_VERBOSE},
{"VGROUP", TK_VGROUP},
{"VGROUPS", TK_VGROUPS},
{"VNODES", TK_VNODES},
{"WAL", TK_WAL},
{"WATERMARK", TK_WATERMARK},
{"WHERE", TK_WHERE},
{"WINDOW_CLOSE", TK_WINDOW_CLOSE},
{"WITH", TK_WITH},
{"WRITE", TK_WRITE},
{"_C0", TK_ROWTS},
{"_QENDTS", TK_QENDTS},
{"_QSTARTTS", TK_QSTARTTS},
{"_ROWTS", TK_ROWTS},
{"_WDURATION", TK_WDURATION},
{"_WENDTS", TK_WENDTS},
{"_WSTARTTS", TK_WSTARTTS},
{"ACCOUNT", TK_ACCOUNT},
{"ACCOUNTS", TK_ACCOUNTS},
{"ADD", TK_ADD},
{"AGGREGATE", TK_AGGREGATE},
{"ALL", TK_ALL},
{"ALTER", TK_ALTER},
{"ANALYZE", TK_ANALYZE},
{"AND", TK_AND},
{"APPS", TK_APPS},
{"AS", TK_AS},
{"ASC", TK_ASC},
{"AT_ONCE", TK_AT_ONCE},
{"BALANCE", TK_BALANCE},
{"BETWEEN", TK_BETWEEN},
{"BINARY", TK_BINARY},
{"BIGINT", TK_BIGINT},
{"BNODE", TK_BNODE},
{"BNODES", TK_BNODES},
{"BOOL", TK_BOOL},
{"BUFFER", TK_BUFFER},
{"BUFSIZE", TK_BUFSIZE},
{"BY", TK_BY},
{"CACHE", TK_CACHE},
{"CACHELAST", TK_CACHELAST},
{"CAST", TK_CAST},
{"CLIENT_VERSION", TK_CLIENT_VERSION},
{"CLUSTER", TK_CLUSTER},
{"COLUMN", TK_COLUMN},
{"COMMENT", TK_COMMENT},
{"COMP", TK_COMP},
{"COMPACT", TK_COMPACT},
{"CONNS", TK_CONNS},
{"CONNECTION", TK_CONNECTION},
{"CONNECTIONS", TK_CONNECTIONS},
{"CONSUMER", TK_CONSUMER},
{"CONSUMERS", TK_CONSUMERS},
{"CONTAINS", TK_CONTAINS},
{"COUNT", TK_COUNT},
{"CREATE", TK_CREATE},
{"CURRENT_USER", TK_CURRENT_USER},
{"DATABASE", TK_DATABASE},
{"DATABASES", TK_DATABASES},
{"DBS", TK_DBS},
{"DELETE", TK_DELETE},
{"DESC", TK_DESC},
{"DESCRIBE", TK_DESCRIBE},
{"DISTINCT", TK_DISTINCT},
{"DISTRIBUTED", TK_DISTRIBUTED},
{"DNODE", TK_DNODE},
{"DNODES", TK_DNODES},
{"DOUBLE", TK_DOUBLE},
{"DROP", TK_DROP},
{"DURATION", TK_DURATION},
{"ENABLE", TK_ENABLE},
{"EXISTS", TK_EXISTS},
{"EXPLAIN", TK_EXPLAIN},
{"EVERY", TK_EVERY},
{"FILL", TK_FILL},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT},
{"FROM", TK_FROM},
{"FSYNC", TK_FSYNC},
{"FUNCTION", TK_FUNCTION},
{"FUNCTIONS", TK_FUNCTIONS},
{"GRANT", TK_GRANT},
{"GRANTS", TK_GRANTS},
{"GROUP", TK_GROUP},
{"HAVING", TK_HAVING},
{"IF", TK_IF},
{"IMPORT", TK_IMPORT},
{"IN", TK_IN},
{"INDEX", TK_INDEX},
{"INDEXES", TK_INDEXES},
{"INNER", TK_INNER},
{"INT", TK_INT},
{"INSERT", TK_INSERT},
{"INTEGER", TK_INTEGER},
{"INTERVAL", TK_INTERVAL},
{"INTO", TK_INTO},
{"IS", TK_IS},
{"JOIN", TK_JOIN},
{"JSON", TK_JSON},
{"KEEP", TK_KEEP},
{"KILL", TK_KILL},
{"LAST", TK_LAST},
{"LAST_ROW", TK_LAST_ROW},
{"LICENCE", TK_LICENCE},
{"LIKE", TK_LIKE},
{"LIMIT", TK_LIMIT},
{"LINEAR", TK_LINEAR},
{"LOCAL", TK_LOCAL},
{"MATCH", TK_MATCH},
{"MAXROWS", TK_MAXROWS},
{"MAX_DELAY", TK_MAX_DELAY},
{"MERGE", TK_MERGE},
{"META", TK_META},
{"MINROWS", TK_MINROWS},
{"MINUS", TK_MINUS},
{"MNODE", TK_MNODE},
{"MNODES", TK_MNODES},
{"MODIFY", TK_MODIFY},
{"MODULES", TK_MODULES},
{"NCHAR", TK_NCHAR},
{"NEXT", TK_NEXT},
{"NMATCH", TK_NMATCH},
{"NONE", TK_NONE},
{"NOT", TK_NOT},
{"NOW", TK_NOW},
{"NULL", TK_NULL},
{"NULLS", TK_NULLS},
{"OFFSET", TK_OFFSET},
{"ON", TK_ON},
{"OR", TK_OR},
{"ORDER", TK_ORDER},
{"OUTPUTTYPE", TK_OUTPUTTYPE},
{"PARTITION", TK_PARTITION},
{"PASS", TK_PASS},
{"PAGES", TK_PAGES},
{"PAGESIZE", TK_PAGESIZE},
{"PORT", TK_PORT},
{"PPS", TK_PPS},
{"PRECISION", TK_PRECISION},
// {"PRIVILEGE", TK_PRIVILEGE},
{"PREV", TK_PREV},
{"QNODE", TK_QNODE},
{"QNODES", TK_QNODES},
{"QTIME", TK_QTIME},
{"QUERIES", TK_QUERIES},
{"QUERY", TK_QUERY},
{"RANGE", TK_RANGE},
{"RATIO", TK_RATIO},
{"READ", TK_READ},
{"REDISTRIBUTE", TK_REDISTRIBUTE},
{"RENAME", TK_RENAME},
{"REPLICA", TK_REPLICA},
{"RESET", TK_RESET},
{"RETENTIONS", TK_RETENTIONS},
{"REVOKE", TK_REVOKE},
{"ROLLUP", TK_ROLLUP},
{"SCHEMALESS", TK_SCHEMALESS},
{"SCORES", TK_SCORES},
{"SELECT", TK_SELECT},
{"SERVER_STATUS", TK_SERVER_STATUS},
{"SERVER_VERSION", TK_SERVER_VERSION},
{"SESSION", TK_SESSION},
{"SET", TK_SET},
{"SHOW", TK_SHOW},
{"SINGLE_STABLE", TK_SINGLE_STABLE},
{"SLIDING", TK_SLIDING},
{"SLIMIT", TK_SLIMIT},
{"SMA", TK_SMA},
{"SMALLINT", TK_SMALLINT},
{"SNODE", TK_SNODE},
{"SNODES", TK_SNODES},
{"SOFFSET", TK_SOFFSET},
{"SPLIT", TK_SPLIT},
{"STABLE", TK_STABLE},
{"STABLES", TK_STABLES},
{"STATE", TK_STATE},
{"STATE_WINDOW", TK_STATE_WINDOW},
{"STORAGE", TK_STORAGE},
{"STREAM", TK_STREAM},
{"STREAMS", TK_STREAMS},
{"STRICT", TK_STRICT},
{"SUBSCRIPTIONS", TK_SUBSCRIPTIONS},
{"SYNCDB", TK_SYNCDB},
{"SYSINFO", TK_SYSINFO},
{"TABLE", TK_TABLE},
{"TABLES", TK_TABLES},
{"TAG", TK_TAG},
{"TAGS", TK_TAGS},
{"TBNAME", TK_TBNAME},
{"TIMESTAMP", TK_TIMESTAMP},
{"TIMEZONE", TK_TIMEZONE},
{"TINYINT", TK_TINYINT},
{"TO", TK_TO},
{"TODAY", TK_TODAY},
{"TOPIC", TK_TOPIC},
{"TOPICS", TK_TOPICS},
{"TRANSACTION", TK_TRANSACTION},
{"TRANSACTIONS", TK_TRANSACTIONS},
{"TRIGGER", TK_TRIGGER},
{"TSERIES", TK_TSERIES},
{"TTL", TK_TTL},
{"UNION", TK_UNION},
{"UNSIGNED", TK_UNSIGNED},
{"USE", TK_USE},
{"USER", TK_USER},
{"USERS", TK_USERS},
{"USING", TK_USING},
{"VALUE", TK_VALUE},
{"VALUES", TK_VALUES},
{"VARCHAR", TK_VARCHAR},
{"VARIABLES", TK_VARIABLES},
{"VERBOSE", TK_VERBOSE},
{"VGROUP", TK_VGROUP},
{"VGROUPS", TK_VGROUPS},
{"VNODES", TK_VNODES},
{"WAL", TK_WAL},
{"WATERMARK", TK_WATERMARK},
{"WHERE", TK_WHERE},
{"WINDOW_CLOSE", TK_WINDOW_CLOSE},
{"WITH", TK_WITH},
{"WRITE", TK_WRITE},
{"_C0", TK_ROWTS},
{"_QENDTS", TK_QENDTS},
{"_QSTARTTS", TK_QSTARTTS},
{"_ROWTS", TK_ROWTS},
{"_WDURATION", TK_WDURATION},
{"_WENDTS", TK_WENDTS},
{"_WSTARTTS", TK_WSTARTTS},
// {"ID", TK_ID},
// {"STRING", TK_STRING},
// {"EQ", TK_EQ},

384
source/libs/parser/src/parTranslater.c
View File

@ -35,8 +35,7 @@ typedef struct STranslateContext {
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
ESqlClause currClause;
SSelectStmt* pCurrSelectStmt;
SSetOperator* pCurrSetOperator;
SNode* pCurrStmt;
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
@ -397,6 +396,34 @@ static void destroyTranslateContext(STranslateContext* pCxt) {
taosHashCleanup(pCxt->pTables);
}
static bool isSelectStmt(SNode* pCurrStmt) {
return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt);
}
static bool isSetOperator(SNode* pCurrStmt) {
return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt);
}
static SNodeList* getProjectListFromCurrStmt(SNode* pCurrStmt) {
if (isSelectStmt(pCurrStmt)) {
return ((SSelectStmt*)pCurrStmt)->pProjectionList;
}
if (isSetOperator(pCurrStmt)) {
return ((SSetOperator*)pCurrStmt)->pProjectionList;
}
return NULL;
}
static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) {
if (isSelectStmt(pCurrStmt)) {
return ((SSelectStmt*)pCurrStmt)->precision;
}
if (isSetOperator(pCurrStmt)) {
return ((SSetOperator*)pCurrStmt)->precision;
}
return defaultVal;
}
static bool isAliasColumn(const SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]));
}
@ -426,7 +453,8 @@ static bool isVectorFunc(const SNode* pNode) {
}
static bool isDistinctOrderBy(STranslateContext* pCxt) {
return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrSelectStmt->isDistinct);
return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && isSelectStmt(pCxt->pCurrStmt) &&
((SSelectStmt*)pCxt->pCurrStmt)->isDistinct);
}
static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) {
@ -646,7 +674,7 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
}
if (!found) {
if (isInternalPk) {
if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY);
}
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_INTERNAL_PK);
@ -657,18 +685,8 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
return DEAL_RES_CONTINUE;
}
static SNodeList* getProjectListFromCxt(STranslateContext* pCxt) {
if (NULL != pCxt->pCurrSelectStmt) {
return pCxt->pCurrSelectStmt->pProjectionList;
} else if (NULL != pCxt->pCurrSetOperator) {
return pCxt->pCurrSetOperator->pProjectionList;
} else {
return NULL;
}
}
static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol, bool* pFound) {
SNodeList* pProjectionList = getProjectListFromCxt(pCxt);
SNodeList* pProjectionList = getProjectListFromCurrStmt(pCxt->pCurrStmt);
SNode* pNode;
FOREACH(pNode, pProjectionList) {
SExprNode* pExpr = (SExprNode*)pNode;
@ -690,7 +708,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
}
static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
if (NULL != pCxt->pCurrSelectStmt && NULL == pCxt->pCurrSelectStmt->pFromTable) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
}
@ -708,7 +726,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
res = translateColumnUseAlias(pCxt, pCol, &found);
}
if (DEAL_RES_ERROR != res && !found) {
if (NULL != pCxt->pCurrSetOperator) {
if (isSetOperator(pCxt->pCurrStmt)) {
res = generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName);
} else {
res = translateColumnWithoutPrefix(pCxt, pCol);
@ -760,9 +778,9 @@ static int32_t parseBoolFromValueNode(STranslateContext* pCxt, SValueNode* pVal)
}
static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) {
uint8_t precision = (NULL != pCxt->pCurrSelectStmt ? pCxt->pCurrSelectStmt->precision : targetDt.precision);
uint8_t precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, targetDt.precision);
pVal->node.resType.precision = precision;
if (pVal->placeholderNo > 0) {
if (pVal->placeholderNo > 0 || pVal->isNull) {
return DEAL_RES_CONTINUE;
}
if (pVal->isDuration) {
@ -1102,6 +1120,8 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) {
}
static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
// the time precision of the function execution environment
pFunc->node.resType.precision = getPrecisionFromCurrStmt(pCxt->pCurrStmt, TSDB_TIME_PRECISION_MILLI);
int32_t code = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
if (TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION == code) {
code = getUdfInfo(pCxt, pFunc);
@ -1119,7 +1139,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (hasInvalidFuncNesting(pFunc->pParameterList)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING);
}
if (NULL != pCxt->pCurrSelectStmt && pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) {
if (isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
@ -1134,7 +1154,8 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN
return TSDB_CODE_SUCCESS;
}
if (0 == LIST_LENGTH(pFunc->pParameterList)) {
if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrSelectStmt->pFromTable)) {
if (!isSelectStmt(pCxt->pCurrStmt) ||
QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME);
}
} else {
@ -1152,8 +1173,8 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod
if (!fmIsIndefiniteRowsFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc ||
pCxt->pCurrSelectStmt->hasAggFuncs) {
if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause ||
((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
if (hasInvalidFuncNesting(pFunc->pParameterList)) {
@ -1162,13 +1183,20 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod
return TSDB_CODE_SUCCESS;
}
static bool hasFillClause(SNode* pCurrStmt) {
if (!isSelectStmt(pCurrStmt)) {
return false;
}
SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
return NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow) &&
NULL != ((SIntervalWindowNode*)pSelect->pWindow)->pFill;
}
static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (!fmIsForbidFillFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (NULL != pCxt->pCurrSelectStmt->pWindow &&
QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow) &&
NULL != ((SIntervalWindowNode*)pCxt->pCurrSelectStmt->pWindow)->pFill) {
if (hasFillClause(pCxt->pCurrStmt)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, pFunc->functionName);
}
return TSDB_CODE_SUCCESS;
@ -1178,7 +1206,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio
if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (NULL == pCxt->pCurrSelectStmt->pWindow) {
if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC);
}
return TSDB_CODE_SUCCESS;
@ -1188,7 +1216,7 @@ static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode*
if (!fmIsForbidWindowFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (NULL != pCxt->pCurrSelectStmt->pWindow) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName);
}
return TSDB_CODE_SUCCESS;
@ -1208,14 +1236,15 @@ static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode
if (!fmIsForbidGroupByFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (NULL != pCxt->pCurrSelectStmt->pGroupByList) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName);
}
return TSDB_CODE_SUCCESS;
}
static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) {
if (NULL != pSelect) {
static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) {
SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId);
pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId);
pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId);
@ -1226,41 +1255,138 @@ static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) {
}
}
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
static int32_t rewriteSystemInfoFuncImpl(STranslateContext* pCxt, char* pLiteral, SNode** pNode) {
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName);
pVal->node.resType = ((SExprNode*)*pNode)->resType;
if (NULL == pLiteral) {
pVal->isNull = true;
} else {
pVal->literal = pLiteral;
}
if (DEAL_RES_ERROR != translateValue(pCxt, pVal)) {
*pNode = (SNode*)pVal;
} else {
nodesDestroyNode((SNode*)pVal);
}
return pCxt->errCode;
}
static int32_t rewriteDatabaseFunc(STranslateContext* pCxt, SNode** pNode) {
char* pCurrDb = NULL;
if (NULL != pCxt->pParseCxt->db) {
pCurrDb = taosMemoryStrDup((void*)pCxt->pParseCxt->db);
if (NULL == pCurrDb) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return rewriteSystemInfoFuncImpl(pCxt, pCurrDb, pNode);
}
static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) {
char* pVer = taosMemoryStrDup((void*)version);
if (NULL == pVer) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return rewriteSystemInfoFuncImpl(pCxt, pVer, pNode);
}
static int32_t rewriteServerVersionFunc(STranslateContext* pCxt, SNode** pNode) {
char* pVer = taosMemoryStrDup((void*)pCxt->pParseCxt->svrVer);
if (NULL == pVer) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return rewriteSystemInfoFuncImpl(pCxt, pVer, pNode);
}
static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) {
if (pCxt->pParseCxt->nodeOffline) {
return TSDB_CODE_RPC_NETWORK_UNAVAIL;
}
char* pStatus = taosMemoryStrDup((void*)"1");
return rewriteSystemInfoFuncImpl(pCxt, pStatus, pNode);
}
static int32_t rewriteUserFunc(STranslateContext* pCxt, SNode** pNode) {
char userConn[TSDB_USER_LEN + 1 + TSDB_FQDN_LEN] = {0}; // format 'user@host'
int32_t len = snprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser);
taosGetFqdn(userConn + len);
char* pUserConn = taosMemoryStrDup((void*)userConn);
if (NULL == pUserConn) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return rewriteSystemInfoFuncImpl(pCxt, pUserConn, pNode);
}
static int32_t rewriteSystemInfoFunc(STranslateContext* pCxt, SNode** pNode) {
switch (((SFunctionNode*)*pNode)->funcType) {
case FUNCTION_TYPE_DATABASE:
return rewriteDatabaseFunc(pCxt, pNode);
case FUNCTION_TYPE_CLIENT_VERSION:
return rewriteClentVersionFunc(pCxt, pNode);
case FUNCTION_TYPE_SERVER_VERSION:
return rewriteServerVersionFunc(pCxt, pNode);
case FUNCTION_TYPE_SERVER_STATUS:
return rewriteServerStatusFunc(pCxt, pNode);
case FUNCTION_TYPE_CURRENT_USER:
case FUNCTION_TYPE_USER:
return rewriteUserFunc(pCxt, pNode);
default:
break;
}
return TSDB_CODE_PAR_INTERNAL_ERROR;
}
static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
int32_t code = translateAggFunc(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == code) {
code = translateScanPseudoColumnFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateIndefiniteRowsFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateForbidFillFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateWindowPseudoColumnFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateForbidWindowFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateForbidStreamFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateForbidGroupByFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == code) {
setFuncClassification(pCxt->pCurrStmt, pFunc);
}
return code;
}
static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pFunc) {
if (fmIsSystemInfoFunc((*pFunc)->funcId)) {
return rewriteSystemInfoFunc(pCxt, (SNode**)pFunc);
}
return translateNoramlFunction(pCxt, *pFunc);
}
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
SNode* pParam = NULL;
FOREACH(pParam, pFunc->pParameterList) {
FOREACH(pParam, (*pFunc)->pParameterList) {
if (isMultiResFunc(pParam)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pParam)->aliasName);
}
}
pCxt->errCode = getFuncInfo(pCxt, pFunc);
pCxt->errCode = getFuncInfo(pCxt, *pFunc);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateAggFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateScanPseudoColumnFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateIndefiniteRowsFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateForbidFillFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateWindowPseudoColumnFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateForbidWindowFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateForbidStreamFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pCxt->errCode = translateForbidGroupByFunc(pCxt, pFunc);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
setFuncClassification(pCxt->pCurrSelectStmt, pFunc);
pCxt->errCode = translateFunctionImpl(pCxt, pFunc);
}
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
@ -1285,7 +1411,7 @@ static EDealRes doTranslateExpr(SNode** pNode, void* pContext) {
case QUERY_NODE_OPERATOR:
return translateOperator(pCxt, (SOperatorNode**)pNode);
case QUERY_NODE_FUNCTION:
return translateFunction(pCxt, (SFunctionNode*)*pNode);
return translateFunction(pCxt, (SFunctionNode**)pNode);
case QUERY_NODE_LOGIC_CONDITION:
return translateLogicCond(pCxt, (SLogicConditionNode*)*pNode);
case QUERY_NODE_TEMP_TABLE:
@ -1308,9 +1434,9 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) {
static SNodeList* getGroupByList(STranslateContext* pCxt) {
if (isDistinctOrderBy(pCxt)) {
return pCxt->pCurrSelectStmt->pProjectionList;
return ((SSelectStmt*)pCxt->pCurrStmt)->pProjectionList;
}
return pCxt->pCurrSelectStmt->pGroupByList;
return ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList;
}
static SNode* getGroupByNode(SNode* pNode) {
@ -1348,32 +1474,13 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
*pNode = (SNode*)pFunc;
pCxt->pCurrSelectStmt->hasSelectValFunc = true;
((SSelectStmt*)pCxt->pCurrStmt)->hasSelectValFunc = true;
} else {
nodesDestroyNode((SNode*)pFunc);
}
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
}
static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
strcpy(pFunc->functionName, "_group_key");
strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName);
pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
*pNode = (SNode*)pFunc;
pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
}
pCxt->pCurrSelectStmt->hasAggFuncs = true;
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
}
static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext;
if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) {
@ -1393,7 +1500,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
SNode* pGroupNode = NULL;
FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) {
if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) {
return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode);
return DEAL_RES_IGNORE_CHILD;
}
}
SNode* pPartKey = NULL;
FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) {
if (nodesEqualNode(pPartKey, *pNode)) {
return DEAL_RES_IGNORE_CHILD;
}
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
@ -1451,25 +1564,6 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt*
return pCxt->errCode;
}
static EDealRes rewriteExprsToGroupKeyFuncImpl(SNode** pNode, void* pContext) {
STranslateContext* pCxt = pContext;
SNode* pPartKey = NULL;
FOREACH(pPartKey, pCxt->pCurrSelectStmt->pPartitionByList) {
if (nodesEqualNode(pPartKey, *pNode)) {
return rewriteExprToGroupKeyFunc(pCxt, pNode);
}
}
return DEAL_RES_CONTINUE;
}
static int32_t rewriteExprsToGroupKeyFunc(STranslateContext* pCxt, SSelectStmt* pSelect) {
nodesRewriteExprs(pSelect->pProjectionList, rewriteExprsToGroupKeyFuncImpl, pCxt);
if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) {
nodesRewriteExprs(pSelect->pOrderByList, rewriteExprsToGroupKeyFuncImpl, pCxt);
}
return pCxt->errCode;
}
typedef struct CheckAggColCoexistCxt {
STranslateContext* pTranslateCxt;
bool existAggFunc;
@ -1495,7 +1589,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) {
return DEAL_RES_IGNORE_CHILD;
}
SNode* pPartKey = NULL;
FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) {
FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) {
if (nodesEqualNode(pPartKey, pNode)) {
return DEAL_RES_IGNORE_CHILD;
}
@ -1529,9 +1623,6 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
if (cxt.existIndefiniteRowsFunc && cxt.existCol) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
}
if (cxt.existAggFunc && NULL != pSelect->pPartitionByList) {
return rewriteExprsToGroupKeyFunc(pCxt, pSelect);
}
return TSDB_CODE_SUCCESS;
}
@ -1662,7 +1753,8 @@ static uint8_t getStmtPrecision(SNode* pStmt) {
static bool stmtIsSingleTable(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
return ((STableNode*)((SSelectStmt*)pStmt)->pFromTable)->singleTable;
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
return NULL == pSelect->pFromTable || ((STableNode*)pSelect->pFromTable)->singleTable;
}
return false;
}
@ -1690,8 +1782,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo
if (pCxt->createStream || QUERY_SMA_OPTIMIZE_DISABLE == tsQuerySmaOptimize) {
return TSDB_CODE_SUCCESS;
}
if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow &&
QUERY_NODE_INTERVAL_WINDOW == nodeType(pCxt->pCurrSelectStmt->pWindow)) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow &&
QUERY_NODE_INTERVAL_WINDOW == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pWindow)) {
return getTableIndex(pCxt, pName, &pRealTable->pSmaIndexes);
}
return TSDB_CODE_SUCCESS;
@ -2404,58 +2496,10 @@ static EDealRes rewriteSeletcValueFunc(STranslateContext* pCxt, SNode** pNode) {
nodesDestroyNode(*pNode);
*pNode = (SNode*)pFirst;
pCxt->errCode = fmGetFuncInfo(pFirst, pCxt->msgBuf.buf, pCxt->msgBuf.len);
pCxt->pCurrSelectStmt->hasAggFuncs = true;
((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs = true;
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
}
static EDealRes rewriteUniqueFunc(SNode** pNode, void* pContext) {
SRwriteUniqueCxt* pCxt = pContext;
if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
SFunctionNode* pFunc = (SFunctionNode*)*pNode;
if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) {
SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0);
NODES_CLEAR_LIST(pFunc->pParameterList);
strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName);
nodesDestroyNode(*pNode);
*pNode = pExpr;
pCxt->pExpr = pExpr;
return DEAL_RES_IGNORE_CHILD;
} else if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType) {
return rewriteSeletcValueFunc(pCxt->pTranslateCxt, pNode);
}
}
return DEAL_RES_CONTINUE;
}
static SNode* createGroupingSet(SNode* pExpr) {
SGroupingSetNode* pGroupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET);
if (NULL == pGroupingSet) {
return NULL;
}
pGroupingSet->groupingSetType = GP_TYPE_NORMAL;
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pGroupingSet->pParameterList, nodesCloneNode(pExpr))) {
nodesDestroyNode((SNode*)pGroupingSet);
return NULL;
}
return (SNode*)pGroupingSet;
}
// from: select unique(expr), col1 + col2 from t where_clause partition_by_clause order_by_clause ...
// to: select expr, first(col1) + first(col2) from t where_clause partition_by_clause group by expr order_by_clause ...
static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasUniqueFunc) {
return TSDB_CODE_SUCCESS;
}
SRwriteUniqueCxt cxt = {.pTranslateCxt = pCxt, .pExpr = NULL};
nodesRewriteExprs(pSelect->pProjectionList, rewriteUniqueFunc, &cxt);
if (TSDB_CODE_SUCCESS == cxt.pTranslateCxt->errCode) {
cxt.pTranslateCxt->errCode = nodesListMakeStrictAppend(&pSelect->pGroupByList, createGroupingSet(cxt.pExpr));
}
pSelect->hasIndefiniteRowsFunc = false;
return cxt.pTranslateCxt->errCode;
}
typedef struct SReplaceOrderByAliasCxt {
STranslateContext* pTranslateCxt;
SNodeList* pProjectionList;
@ -2474,6 +2518,7 @@ static EDealRes replaceOrderByAliasImpl(SNode** pNode, void* pContext) {
pCxt->pTranslateCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
((SExprNode*)pNew)->orderAlias = true;
nodesDestroyNode(*pNode);
*pNode = pNew;
return DEAL_RES_CONTINUE;
@ -2490,13 +2535,13 @@ static int32_t replaceOrderByAlias(STranslateContext* pCxt, SNodeList* pProjecti
}
static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
pCxt->pCurrSelectStmt = pSelect;
pCxt->pCurrStmt = (SNode*)pSelect;
pCxt->currClause = SQL_CLAUSE_SELECT;
return translateExprList(pCxt, pSelect->pProjectionList);
}
static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
pCxt->pCurrSelectStmt = pSelect;
pCxt->pCurrStmt = (SNode*)pSelect;
int32_t code = translateFrom(pCxt, pSelect->pFromTable);
if (TSDB_CODE_SUCCESS == code) {
pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision;
@ -2529,9 +2574,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = translateInterp(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteUniqueStmt(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteTimelineFunc(pCxt, pSelect);
}
@ -2623,8 +2665,7 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
if (TSDB_CODE_SUCCESS == code) {
if (other) {
pCxt->currClause = SQL_CLAUSE_ORDER_BY;
pCxt->pCurrSelectStmt = NULL;
pCxt->pCurrSetOperator = pSetOperator;
pCxt->pCurrStmt = (SNode*)pSetOperator;
code = translateExprList(pCxt, pSetOperator->pOrderByList);
}
}
@ -4504,12 +4545,12 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
++(pCxt->currLevel);
ESqlClause currClause = pCxt->currClause;
SSelectStmt* pCurrStmt = pCxt->pCurrSelectStmt;
int32_t code = translateQuery(pCxt, pNode);
ESqlClause currClause = pCxt->currClause;
SNode* pCurrStmt = pCxt->pCurrStmt;
int32_t code = translateQuery(pCxt, pNode);
--(pCxt->currLevel);
pCxt->currClause = currClause;
pCxt->pCurrSelectStmt = pCurrStmt;
pCxt->pCurrStmt = pCurrStmt;
return code;
}
@ -5961,6 +6002,11 @@ static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
switch (nodeType(pQuery->pRoot)) {
case QUERY_NODE_SELECT_STMT:
if (NULL == ((SSelectStmt*)pQuery->pRoot)->pFromTable) {
pQuery->execMode = QUERY_EXEC_MODE_LOCAL;
pQuery->haveResultSet = true;
break;
}
case QUERY_NODE_SET_OPERATOR:
case QUERY_NODE_EXPLAIN_STMT:
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;

4332
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

5
source/libs/parser/test/parInsertTest.cpp
View File

@ -265,6 +265,11 @@ TEST_F(InsertTest, autoCreateTableTest) {
"insert into st1s1 using st1 (tag1, tag2) tags(1, 'wxy') values (now, 1, \"beijing\")"
"(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")");
ASSERT_EQ(runAsync(), TSDB_CODE_SUCCESS);
bind(
"insert into st1s1 using st1 tags(1, 'wxy', now) values (now, 1, \"beijing\")"
"st1s1 using st1 tags(1, 'wxy', now) values (now+1s, 2, \"shanghai\")");
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
}
TEST_F(InsertTest, toleranceTest) {

14
source/libs/parser/test/parSelectTest.cpp
View File

@ -232,8 +232,6 @@ TEST_F(ParserSelectTest, groupBySemanticCheck) {
run("SELECT COUNT(*) cnt, c1 FROM t1 WHERE c1 > 0", TSDB_CODE_PAR_NOT_SINGLE_GROUP);
run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 GROUP BY c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION);
run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 PARTITION BY c2 GROUP BY c1",
TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION);
}
TEST_F(ParserSelectTest, orderBy) {
@ -425,6 +423,18 @@ TEST_F(ParserSelectTest, withoutFrom) {
useDb("root", "test");
run("SELECT 1");
run("SELECT DATABASE()");
run("SELECT CLIENT_VERSION()");
run("SELECT SERVER_VERSION()");
run("SELECT SERVER_STATUS()");
run("SELECT CURRENT_USER()");
run("SELECT USER()");
}
} // namespace ParserTest

1
source/libs/parser/test/parTestUtil.cpp
View File

@ -203,6 +203,7 @@ class ParserTestBaseImpl {
pCxt->pMsg = stmtEnv_.msgBuf_.data();
pCxt->msgLen = stmtEnv_.msgBuf_.max_size();
pCxt->async = async;
pCxt->svrVer = "3.0.0.0";
}
void doParse(SParseContext* pCxt, SQuery** pQuery) {

45
source/libs/planner/src/planLogicCreater.c
View File

@ -437,6 +437,33 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr)
return pCol;
}
static SNode* createGroupingSetNode(SNode* pExpr) {
SGroupingSetNode* pGroupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET);
if (NULL == pGroupingSet) {
return NULL;
}
pGroupingSet->groupingSetType = GP_TYPE_NORMAL;
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pGroupingSet->pParameterList, nodesCloneNode(pExpr))) {
nodesDestroyNode((SNode*)pGroupingSet);
return NULL;
}
return (SNode*)pGroupingSet;
}
static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOutput) {
SNodeList* pGroupKeys = NULL;
SNode* pPartKey = NULL;
FOREACH(pPartKey, pPartKeys) {
int32_t code = nodesListMakeStrictAppend(&pGroupKeys, createGroupingSetNode(pPartKey));
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pGroupKeys);
return code;
}
}
*pOutput = pGroupKeys;
return TSDB_CODE_SUCCESS;
}
static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) {
return TSDB_CODE_SUCCESS;
@ -459,10 +486,18 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (NULL != pSelect->pPartitionByList) {
code = createGroupKeysFromPartKeys(pSelect->pPartitionByList, &pAgg->pGroupKeys);
}
if (NULL != pSelect->pGroupByList) {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
if (NULL != pAgg->pGroupKeys) {
code = nodesListStrictAppendList(pAgg->pGroupKeys, nodesCloneList(pSelect->pGroupByList));
} else {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
}
@ -508,6 +543,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
}
pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc;
pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc;
// indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
@ -809,7 +845,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
}
static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pPartitionByList) {
if (NULL == pSelect->pPartitionByList || (pSelect->hasAggFuncs && NULL == pSelect->pWindow) ||
NULL != pSelect->pGroupByList) {
return TSDB_CODE_SUCCESS;
}

372
source/libs/planner/src/planOptimizer.c
View File

@ -276,7 +276,8 @@ static int32_t pushDownCondOptAppendCond(SNode** pCond, SNode** pAdditionalCond)
}
int32_t code = TSDB_CODE_SUCCESS;
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond)) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCond)->condType) {
code = nodesListAppend(((SLogicConditionNode*)*pCond)->pParameterList, *pAdditionalCond);
if (TSDB_CODE_SUCCESS == code) {
*pAdditionalCond = NULL;
@ -987,6 +988,7 @@ static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogi
code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex);
taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex);
pScan->pSmaIndexes = NULL;
pCxt->optimized = true;
break;
}
}
@ -1033,12 +1035,30 @@ static SNodeList* partTagsGetPartKeys(SLogicNode* pNode) {
}
}
static SNodeList* partTagsGetFuncs(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
return NULL;
} else {
return ((SAggLogicNode*)pNode)->pAggFuncs;
}
}
static bool partTagsOptAreSupportedFuncs(SNodeList* pFuncs) {
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
if (fmIsIndefiniteRowsFunc(((SFunctionNode*)pFunc)->funcId) && !fmIsSelectFunc(((SFunctionNode*)pFunc)->funcId)) {
return false;
}
}
return true;
}
static bool partTagsOptMayBeOptimized(SLogicNode* pNode) {
if (!partTagsIsOptimizableNode(pNode)) {
return false;
}
return !partTagsOptHasCol(partTagsGetPartKeys(pNode));
return !partTagsOptHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode));
}
static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) {
@ -1064,6 +1084,64 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) {
return code;
}
// todo refact: just to mask compilation warnings
static void partTagsSetAlias(char* pAlias, int32_t len, const char* pTableAlias, const char* pColName) {
snprintf(pAlias, len, "%s.%s", pTableAlias, pColName);
}
static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
return NULL;
}
strcpy(pFunc->functionName, pFuncName);
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName);
} else {
strcpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName);
}
int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pNode));
if (TSDB_CODE_SUCCESS == code) {
code = fmGetFuncInfo(pFunc, NULL, 0);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc);
return NULL;
}
return (SNode*)pFunc;
}
static bool partTagsHasIndefRowsSelectFunc(SNodeList* pFuncs) {
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
if (fmIsIndefiniteRowsFunc(((SFunctionNode*)pFunc)->funcId)) {
return true;
}
}
return false;
}
static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, SNodeList* pAggFuncs) {
bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAggFuncs);
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pGroupTags) {
if (hasIndefRowsSelectFunc) {
code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode));
} else {
code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_group_key", pNode));
}
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
return code;
}
static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized);
if (NULL == pNode) {
@ -1080,15 +1158,17 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
nodesDestroyNode((SNode*)pNode);
}
} else {
SNode* pGroupKey = NULL;
FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) {
SAggLogicNode* pAgg = (SAggLogicNode*)pNode;
SNode* pGroupKey = NULL;
FOREACH(pGroupKey, pAgg->pGroupKeys) {
code = nodesListMakeStrictAppend(
&pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0)));
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys);
NODES_DESTORY_LIST(pAgg->pGroupKeys);
code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, pAgg->pAggFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = partTagsOptRebuildTbanme(pScan->pGroupTags);
@ -1125,7 +1205,8 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
}
SProjectLogicNode* pProjectNode = (SProjectLogicNode*)pNode;
if (NULL != pProjectNode->node.pLimit || NULL != pProjectNode->node.pSlimit) {
if (NULL != pProjectNode->node.pLimit || NULL != pProjectNode->node.pSlimit ||
NULL != pProjectNode->node.pConditions) {
return false;
}
@ -1163,6 +1244,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
NODES_CLEAR_LIST(pProjectNode->node.pChildren);
nodesDestroyNode((SNode*)pProjectNode);
}
pCxt->optimized = true;
return code;
}
@ -1306,6 +1388,7 @@ static int32_t rewriteTailOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pL
nodesDestroyNode((SNode*)pSort);
nodesDestroyNode((SNode*)pProject);
}
pCxt->optimized = true;
return code;
}
@ -1344,6 +1427,7 @@ static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
pSetOpNode->pChildren = NULL;
ERASE_NODE(pSetOpNode->pParent->pChildren);
pCxt->optimized = true;
return TSDB_CODE_SUCCESS;
}
}
@ -1360,6 +1444,261 @@ static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo
return eliminateSetOpOptimizeImpl(pCxt, pLogicSubplan, pSetOpNode);
}
static bool rewriteUniqueOptMayBeOptimized(SLogicNode* pNode) {
return QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC == nodeType(pNode) && ((SIndefRowsFuncLogicNode*)pNode)->isUniqueFunc;
}
static SNode* rewriteUniqueOptCreateGroupingSet(SNode* pExpr) {
SGroupingSetNode* pGroupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET);
if (NULL == pGroupingSet) {
return NULL;
}
pGroupingSet->groupingSetType = GP_TYPE_NORMAL;
SExprNode* pGroupExpr = (SExprNode*)nodesCloneNode(pExpr);
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pGroupingSet->pParameterList, (SNode*)pGroupExpr)) {
nodesDestroyNode((SNode*)pGroupingSet);
return NULL;
}
return (SNode*)pGroupingSet;
}
static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode* pCol) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
return NULL;
}
strcpy(pFunc->functionName, "first");
if (NULL != pSelectValue) {
strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName);
} else {
snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc);
}
int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol));
if (TSDB_CODE_SUCCESS == code) {
code = fmGetFuncInfo(pFunc, NULL, 0);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc);
return NULL;
}
return (SNode*)pFunc;
}
static int32_t rewriteUniqueOptCreateAgg(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) {
SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
if (NULL == pAgg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pAgg->node.pChildren, pIndef->node.pChildren);
pAgg->node.precision = pIndef->node.precision;
int32_t code = TSDB_CODE_SUCCESS;
bool hasSelectPrimaryKey = false;
SNode* pPrimaryKey = NULL;
SNode* pNode = NULL;
FOREACH(pNode, pIndef->pFuncs) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0);
if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) {
pPrimaryKey = nodesListGetNode(pFunc->pParameterList, 1);
code = nodesListMakeStrictAppend(&pAgg->pGroupKeys, rewriteUniqueOptCreateGroupingSet(pExpr));
} else if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId) { // _select_value(ts) => first(ts)
hasSelectPrimaryKey = true;
code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, rewriteUniqueOptCreateFirstFunc(pFunc, pExpr));
} else { // _select_value(other_col)
code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pNode));
}
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) {
code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code && !hasSelectPrimaryKey && NULL != pAgg->pAggFuncs) {
code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, rewriteUniqueOptCreateFirstFunc(NULL, pPrimaryKey));
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = (SLogicNode*)pAgg;
} else {
nodesDestroyNode((SNode*)pAgg);
}
return code;
}
static SNode* rewriteUniqueOptCreateProjectCol(SFunctionNode* pFunc) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
pCol->node.resType = pFunc->node.resType;
if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) {
SExprNode* pExpr = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
strcpy(pCol->colName, ((SColumnNode*)pExpr)->colName);
} else {
strcpy(pCol->colName, pExpr->aliasName);
}
} else {
strcpy(pCol->colName, pFunc->node.aliasName);
}
strcpy(pCol->node.aliasName, pFunc->node.aliasName);
return (SNode*)pCol;
}
static int32_t rewriteUniqueOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SLogicNode** pOutput) {
SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
if (NULL == pProject) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pProject->node.pTargets, pIndef->node.pTargets);
pProject->node.precision = pIndef->node.precision;
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pIndef->pFuncs) {
code = nodesListMakeStrictAppend(&pProject->pProjections, rewriteUniqueOptCreateProjectCol((SFunctionNode*)pNode));
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = (SLogicNode*)pProject;
} else {
nodesDestroyNode((SNode*)pProject);
}
return code;
}
static int32_t rewriteUniqueOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
SIndefRowsFuncLogicNode* pIndef) {
SLogicNode* pAgg = NULL;
SLogicNode* pProject = NULL;
int32_t code = rewriteUniqueOptCreateAgg(pIndef, &pAgg);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteUniqueOptCreateProject(pIndef, &pProject);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pAgg);
pAgg->pParent = pProject;
pAgg = NULL;
}
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject);
}
if (TSDB_CODE_SUCCESS == code) {
nodesDestroyNode((SNode*)pIndef);
} else {
nodesDestroyNode((SNode*)pAgg);
nodesDestroyNode((SNode*)pProject);
}
pCxt->optimized = true;
return code;
}
static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SIndefRowsFuncLogicNode* pIndef =
(SIndefRowsFuncLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, rewriteUniqueOptMayBeOptimized);
if (NULL == pIndef) {
return TSDB_CODE_SUCCESS;
}
return rewriteUniqueOptimizeImpl(pCxt, pLogicSubplan, pIndef);
}
// merge projects
static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) {
return false;
}
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0);
if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pChild) || 1 < LIST_LENGTH(pChild->pChildren) ||
NULL != pChild->pConditions || NULL != pChild->pLimit || NULL != pChild->pSlimit) {
return false;
}
return true;
}
typedef struct SMergeProjectionsContext {
SProjectLogicNode* pChildProj;
int32_t errCode;
} SMergeProjectionsContext;
static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) {
SMergeProjectionsContext* pCxt = pContext;
SProjectLogicNode* pChildProj = pCxt->pChildProj;
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
SNode* pTarget;
FOREACH(pTarget, ((SLogicNode*)pChildProj)->pTargets) {
if (nodesEqualNode(pTarget, *pNode)) {
SNode* pProjection;
FOREACH(pProjection, pChildProj->pProjections) {
if (0 == strcmp(((SColumnNode*)pTarget)->colName, ((SExprNode*)pProjection)->aliasName)) {
SNode* pExpr = nodesCloneNode(pProjection);
if (pExpr == NULL) {
pCxt->errCode = terrno;
return DEAL_RES_ERROR;
}
nodesDestroyNode(*pNode);
*pNode = pExpr;
}
}
}
}
return DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
}
static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) {
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0);
SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt);
int32_t code = cxt.errCode;
if (TSDB_CODE_SUCCESS == code) {
if (1 == LIST_LENGTH(pChild->pChildren)) {
SLogicNode* pGrandChild = (SLogicNode*)nodesListGetNode(pChild->pChildren, 0);
code = replaceLogicNode(pLogicSubplan, pChild, pGrandChild);
} else { // no grand child
NODES_CLEAR_LIST(pSelfNode->pChildren);
}
}
if (TSDB_CODE_SUCCESS == code) {
NODES_CLEAR_LIST(pChild->pChildren);
}
nodesDestroyNode((SNode*)pChild);
pCxt->optimized = true;
return code;
}
static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pProjectNode = optFindPossibleNode(pLogicSubplan->pNode, mergeProjectsMayBeOptimized);
if (NULL == pProjectNode) {
return TSDB_CODE_SUCCESS;
}
return mergeProjectsOptimizeImpl(pCxt, pLogicSubplan, pProjectNode);
}
// clang-format off
static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "ScanPath", .optimizeFunc = scanPathOptimize},
@ -1367,25 +1706,40 @@ static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize},
{.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize},
{.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
{.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize},
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
{.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
{.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}
{.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize},
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}
};
// clang-format on
static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
char* pStr = NULL;
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
qDebugL("apply optimize %s rule: %s", pRuleName, pStr);
taosMemoryFree(pStr);
}
static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) {
SOptimizeContext cxt = {.pPlanCxt = pCxt, .optimized = false};
bool optimized = false;
do {
cxt.optimized = false;
optimized = false;
for (int32_t i = 0; i < optimizeRuleNum; ++i) {
cxt.optimized = false;
int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicSubplan);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (cxt.optimized) {
optimized = true;
dumpLogicSubplan(optimizeRuleSet[i].pName, pLogicSubplan);
}
}
} while (cxt.optimized);
} while (optimized);
return TSDB_CODE_SUCCESS;
}

2
source/libs/planner/src/planSpliter.c
View File

@ -1237,7 +1237,7 @@ static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
char* pStr = NULL;
nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL);
qDebugL("apply %s rule: %s", pRuleName, pStr);
qDebugL("apply split %s rule: %s", pRuleName, pStr);
taosMemoryFree(pStr);
}

17
source/libs/planner/src/planner.c
View File

@ -85,6 +85,23 @@ int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstream
return setSubplanExecutionNode(subplan->pNode, groupId, pSource);
}
static void clearSubplanExecutionNode(SPhysiNode* pNode) {
if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pNode)) {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode;
NODES_DESTORY_LIST(pExchange->pSrcEndPoints);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == nodeType(pNode)) {
SMergePhysiNode* pMerge = (SMergePhysiNode*)pNode;
pMerge->numOfChannels = LIST_LENGTH(pMerge->node.pChildren);
SNode* pChild = NULL;
FOREACH(pChild, pMerge->node.pChildren) { NODES_DESTORY_LIST(((SExchangePhysiNode*)pChild)->pSrcEndPoints); }
}
SNode* pChild = NULL;
FOREACH(pChild, pNode->pChildren) { clearSubplanExecutionNode((SPhysiNode*)pChild); }
}
void qClearSubplanExecutionNode(SSubplan* pSubplan) { clearSubplanExecutionNode(pSubplan->pNode); }
int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) {
if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) {
SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink;

14
source/libs/planner/test/planBasicTest.cpp
View File

@ -56,6 +56,8 @@ TEST_F(PlanBasicTest, uniqueFunc) {
run("SELECT UNIQUE(c2 + 10) FROM t1 WHERE c1 > 10");
run("SELECT UNIQUE(c2 + 10), c2 FROM t1 WHERE c1 > 10");
run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10");
run("SELECT UNIQUE(c1) a FROM t1 ORDER BY a");
@ -75,6 +77,8 @@ TEST_F(PlanBasicTest, tailFunc) {
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5");
run("SELECT TAIL(c1, 2, 1) FROM st1s1 UNION ALL SELECT c1 FROM st1s2");
}
TEST_F(PlanBasicTest, interpFunc) {
@ -97,6 +101,16 @@ TEST_F(PlanBasicTest, lastRowFunc) {
run("SELECT LAST_ROW(c1) FROM st1");
}
TEST_F(PlanBasicTest, sampleFunc) {
useDb("root", "test");
run("SELECT SAMPLE(c1, 10) FROM t1");
run("SELECT SAMPLE(c1, 10) FROM st1");
run("SELECT SAMPLE(c1, 10) FROM st1 PARTITION BY TBNAME");
}
TEST_F(PlanBasicTest, withoutFrom) {
useDb("root", "test");

8
source/libs/planner/test/planOptimizeTest.cpp
View File

@ -20,7 +20,7 @@ using namespace std;
class PlanOptimizeTest : public PlannerTestBase {};
TEST_F(PlanOptimizeTest, optimizeScanData) {
TEST_F(PlanOptimizeTest, scanPath) {
useDb("root", "test");
run("SELECT COUNT(*) FROM t1");
@ -32,7 +32,7 @@ TEST_F(PlanOptimizeTest, optimizeScanData) {
run("SELECT PERCENTILE(c1, 40), COUNT(*) FROM t1");
}
TEST_F(PlanOptimizeTest, ConditionPushDown) {
TEST_F(PlanOptimizeTest, pushDownCondition) {
useDb("root", "test");
run("SELECT ts, c1 FROM st1 WHERE tag1 > 4");
@ -42,9 +42,11 @@ TEST_F(PlanOptimizeTest, ConditionPushDown) {
run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello'");
run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello' AND c1 > 10");
run("SELECT ts, c1 FROM (SELECT * FROM st1) WHERE tag1 > 4");
}
TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
TEST_F(PlanOptimizeTest, sortPrimaryKey) {
useDb("root", "test");
run("SELECT c1 FROM t1 ORDER BY ts");

1
source/libs/planner/test/planTestUtil.cpp
View File

@ -310,6 +310,7 @@ class PlannerTestBaseImpl {
cxt.sqlLen = stmtEnv_.sql_.length();
cxt.pMsg = stmtEnv_.msgBuf_.data();
cxt.msgLen = stmtEnv_.msgBuf_.max_size();
cxt.svrVer = "3.0.0.0";
DO_WITH_THROW(qParseSql, &cxt, pQuery);
if (prepare) {

2
source/libs/qcom/src/queryUtil.c
View File

@ -420,7 +420,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
taosHashCancelIterate(pSrc->vgHash, pIter);
taosHashCleanup((*pDst)->vgHash);
taosMemoryFreeClear(*pDst);
return TSDB_CODE_CTG_MEM_ERROR;
return TSDB_CODE_OUT_OF_MEMORY;
}
pIter = taosHashIterate(pSrc->vgHash, pIter);

2
source/libs/transport/inc/transComm.h
View File

@ -229,7 +229,7 @@ typedef struct {
SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb);
void transDestroyAsyncPool(SAsyncPool* pool);
int transSendAsync(SAsyncPool* pool, queue* mq);
int transAsyncSend(SAsyncPool* pool, queue* mq);
#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \
do { \

10
source/libs/transport/src/transCli.c
View File

@ -967,7 +967,7 @@ void cliSendQuit(SCliThrd* thrd) {
// cli can stop gracefully
SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg));
msg->type = Quit;
transSendAsync(thrd->asyncPool, &msg->q);
transAsyncSend(thrd->asyncPool, &msg->q);
}
void cliWalkCb(uv_handle_t* handle, void* arg) {
if (!uv_is_closing(handle)) {
@ -1138,7 +1138,7 @@ void transReleaseCliHandle(void* handle) {
cmsg->msg = tmsg;
cmsg->type = Release;
transSendAsync(pThrd->asyncPool, &cmsg->q);
transAsyncSend(pThrd->asyncPool, &cmsg->q);
return;
}
@ -1171,7 +1171,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra
STraceId* trace = &pReq->info.traceId;
tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0);
ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0);
return;
}
@ -1205,7 +1205,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
transSendAsync(pThrd->asyncPool, &(cliMsg->q));
transAsyncSend(pThrd->asyncPool, &(cliMsg->q));
tsem_wait(sem);
tsem_destroy(sem);
taosMemoryFree(sem);
@ -1234,7 +1234,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i];
tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid);
transSendAsync(thrd->asyncPool, &(cliMsg->q));
transAsyncSend(thrd->asyncPool, &(cliMsg->q));
}
}
#endif

2
source/libs/transport/src/transComm.c
View File

@ -202,7 +202,7 @@ void transDestroyAsyncPool(SAsyncPool* pool) {
taosMemoryFree(pool->asyncs);
taosMemoryFree(pool);
}
int transSendAsync(SAsyncPool* pool, queue* q) {
int transAsyncSend(SAsyncPool* pool, queue* q) {
int idx = pool->index;
idx = idx % pool->nAsync;
// no need mutex here

8
source/libs/transport/src/transSvr.c
View File

@ -999,7 +999,7 @@ void sendQuitToWorkThrd(SWorkThrd* pThrd) {
SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg));
msg->type = Quit;
tDebug("server send quit msg to work thread");
transSendAsync(pThrd->asyncPool, &msg->q);
transAsyncSend(pThrd->asyncPool, &msg->q);
}
void transCloseServer(void* arg) {
@ -1070,7 +1070,7 @@ void transReleaseSrvHandle(void* handle) {
m->type = Release;
tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle);
transSendAsync(pThrd->asyncPool, &m->q);
transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId);
return;
_return1:
@ -1099,7 +1099,7 @@ void transSendResponse(const STransMsg* msg) {
STraceId* trace = (STraceId*)&msg->info.traceId;
tGTrace("conn %p start to send resp (1/2)", exh->handle);
transSendAsync(pThrd->asyncPool, &m->q);
transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId);
return;
_return1:
@ -1128,7 +1128,7 @@ void transRegisterMsg(const STransMsg* msg) {
m->type = Register;
tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle);
transSendAsync(pThrd->asyncPool, &m->q);
transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId);
return;

16
source/os/src/osSocket.c
View File

@ -925,10 +925,24 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
}
int32_t taosGetFqdn(char *fqdn) {
#ifdef WINDOWS
// Initialize Winsock
WSADATA wsaData;
int iResult;
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
// printf("WSAStartup failed: %d\n", iResult);
return 1;
}
#endif
char hostname[1024];
hostname[1023] = '\0';
if (gethostname(hostname, 1023) == -1) {
// printf("failed to get hostname, reason:%s", strerror(errno));
#ifdef WINDOWS
printf("failed to get hostname, reason:%s", strerror(WSAGetLastError()));
#else
printf("failed to get hostname, reason:%s", strerror(errno));
#endif
assert(0);
return -1;
}

5
source/util/src/terror.c
View File

@ -453,11 +453,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error")
TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "catalog memory error")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_DB_DROPPED, "Database is dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_OUT_OF_SERVICE, "catalog is out of service")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_VG_META_MISMATCH, "table meta and vgroup mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_CTG_EXIT, "catalog exit")
//scheduler
TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error")
@ -581,10 +582,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_PTR, "Invalid tsma pointe
TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_PARA, "Invalid tsma parameters")
TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in cache")
//rsma
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env")
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state")
TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo creation error")
//tq
TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "No committed offset")

13
tests/pytest/a.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
for i in {1..100}
do
echo $i
python3 ./test.py -f query/nestedQuery/nestedQuery_datacheck.py >>log 2>&1
if [ $? -eq 0 ]
then
echo success
else
echo failed
break
fi
done

10
tests/pytest/util/cluster.py
View File

@ -26,7 +26,7 @@ class ClusterDnodes(TDDnodes):
class ConfigureyCluster:
"""This will create defined number of dnodes and create a cluster.
at the same time, it will return TDDnodes list: dnodes, """
hostname= socket.gethostname()
hostname = socket.gethostname()
def __init__(self):
self.dnodes = []
@ -86,7 +86,13 @@ class ConfigureyCluster:
time.sleep(1)
else:
tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
def checkConnectStatus(self,dnodeNo,hostname=hostname):
dnodeNo = int(dnodeNo)
tdLog.info("check dnode-%d connection"%(dnodeNo+1))
hostname = socket.gethostname()
port = 6030 + dnodeNo*100
connectToDnode = tdCom.newcon(host=hostname,port=port)
return connectToDnode
cluster = ConfigureyCluster()

5
tests/pytest/util/common.py
View File

@ -379,6 +379,11 @@ class TDCom:
tdLog.info("cfgPath: %s" % cfgPath)
return cfgPath
def newcon(self,host='localhost',port=6030,user='root',password='taosdata'):
con=taos.connect(host=host, user=user, password=password, port=port)
print(con)
return con
def newcur(self,host='localhost',port=6030,user='root',password='taosdata'):
cfgPath = self.getClientCfgPath()
con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port)

39
tests/script/jenkins/basic.txt
View File

@ -8,21 +8,35 @@
./test.sh -f tsim/user/privilege_sysinfo.sim
## ---- db
./test.sh -f tsim/db/create_all_options.sim
./test.sh -f tsim/db/alter_option.sim
./test.sh -f tsim/db/alter_replica_13.sim
#./test.sh -f tsim/db/alter_replica_31.sim
./test.sh -f tsim/db/alter_replica_31.sim
./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic2.sim
./test.sh -f tsim/db/basic3.sim
./test.sh -f tsim/db/basic6.sim
./test.sh -f tsim/db/basic7.sim
#./test.sh -f tsim/db/commit.sim
./test.sh -f tsim/db/create_all_options.sim
#./test.sh -f tsim/db/delete_part.sim
#./test.sh -f tsim/db/delete.sim
./test.sh -f tsim/db/error1.sim
./test.sh -f tsim/db/taosdlog.sim
# ---- dnode
./test.sh -f tsim/dnode/balance_replica1.sim
#./test.sh -f tsim/dnode/balance_replica3.sim
./test.sh -f tsim/dnode/balance_replica3.sim
./test.sh -f tsim/dnode/balance1.sim
./test.sh -f tsim/dnode/balance2.sim
./test.sh -f tsim/dnode/balance3.sim
./test.sh -f tsim/dnode/balancex.sim
#./test.sh -f tsim/dnode/cluster_alter.sim
#./test.sh -f tsim/dnode/cluster_balance1.sim
#./test.sh -f tsim/dnode/cluster_balance2.sim
#./test.sh -f tsim/dnode/cluster_balance3.sim
#./test.sh -f tsim/dnode/cluster_cache.sim
#./test.sh -f tsim/dnode/cluster_flowctrl.sim
#./test.sh -f tsim/dnode/cluster_vgroup100.sim
./test.sh -f tsim/dnode/create_dnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@ -30,11 +44,13 @@
./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
./test.sh -f tsim/dnode/offline_reason.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
./test.sh -f tsim/dnode/vnode_clean.sim
# ---- insert
./test.sh -f tsim/insert/basic0.sim
@ -72,7 +88,7 @@
./test.sh -f tsim/mnode/basic2.sim
./test.sh -f tsim/mnode/basic3.sim
./test.sh -f tsim/mnode/basic4.sim
#./test.sh -f tsim/mnode/basic5.sim
./test.sh -f tsim/mnode/basic5.sim
# ---- show
./test.sh -f tsim/show/basic.sim
@ -85,6 +101,7 @@
./test.sh -f tsim/stream/basic1.sim
./test.sh -f tsim/stream/basic2.sim
./test.sh -f tsim/stream/distributeInterval0.sim
# ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
# ./test.sh -f tsim/stream/distributesession0.sim
# ./test.sh -f tsim/stream/session0.sim
./test.sh -f tsim/stream/session1.sim
@ -96,7 +113,6 @@
./test.sh -f tsim/stream/schedSnode.sim
./test.sh -f tsim/stream/windowClose.sim
# ---- transaction
./test.sh -f tsim/trans/lossdata1.sim
./test.sh -f tsim/trans/create_db.sim
@ -155,6 +171,19 @@
# --- valgrind
./test.sh -f tsim/valgrind/checkError.sim -v
# --- vnode
#./test.sh -f tsim/vnode/replica3_basic.sim
#./test.sh -f tsim/vnode/replica3_repeat.sim
./test.sh -f tsim/vnode/replica3_vgroup.sim
#./test.sh -f tsim/vnode/replica3_many.sim
#./test.sh -f tsim/vnode/replica3_import.sim
./test.sh -f tsim/vnode/stable_balance_replica1.sim
./test.sh -f tsim/vnode/stable_dnode2_stop.sim
./test.sh -f tsim/vnode/stable_dnode2.sim
./test.sh -f tsim/vnode/stable_dnode3.sim
./test.sh -f tsim/vnode/stable_replica3_dnode6.sim
./test.sh -f tsim/vnode/stable_replica3_vnode3.sim
# --- sync
./test.sh -f tsim/sync/3Replica1VgElect.sim
./test.sh -f tsim/sync/3Replica5VgElect.sim

27
tests/script/sh/exec.bat
View File

@ -6,6 +6,7 @@ if %1 == -n set NODE_NAME=%2
if %1 == -s set EXEC_OPTON=%2
if %3 == -n set NODE_NAME=%4
if %3 == -s set EXEC_OPTON=%4
if "%5" == "-x" set EXEC_SIGNAL=%6
rem echo NODE_NAME: %NODE_NAME%
rem echo NODE: %EXEC_OPTON%
@ -39,7 +40,7 @@ rem echo TAOS_LOG: %TAOS_LOG%
if %EXEC_OPTON% == start (
rm -rf %TAOS_LOG%
echo start %TAOSD% -c %CFG_DIR%
start %TAOSD% -c %CFG_DIR%
mintty -h never %TAOSD% -c %CFG_DIR%
set /a check_num=0
:check_online
sleep 1
@ -58,13 +59,25 @@ if %EXEC_OPTON% == stop (
rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE
rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1
for /f "tokens=1 skip=1" %%A in (
'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId '
) do (
rem echo taskkill /IM %%A
taskkill /IM %%A > NUL 2>&1
for /f "tokens=2" %%A in ('wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ^| xargs echo') do (
for /f "tokens=1" %%B in ('ps ^| grep %%A') do (
if "%EXEC_SIGNAL%" == "SIGKILL" (
kill -9 %%B
) else (
kill -INT %%B
call :check_offline
)
)
goto :finish
)
)
:finish
goto :eof
:finish
:check_offline
sleep 1
for /f "tokens=2" %%C in ('wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ^| xargs echo') do (
echo check taosd offline
goto :check_offline
)
goto :eof

56
tests/script/sh/move_dnode.sh
View File

@ -1,56 +0,0 @@
#!/bin/bash
echo "Executing move_dnode.sh"
UNAME_BIN=`which uname`
OS_TYPE=`$UNAME_BIN`
SCRIPT_DIR=`dirname $0`
cd $SCRIPT_DIR/../
SCRIPT_DIR=`pwd`
echo "SCRIPT_DIR: $SCRIPT_DIR"
IN_TDINTERNAL="community"
if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
cd ../../..
else
cd ../../
fi
TAOS_DIR=`pwd`
TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
if [[ "$OS_TYPE" != "Darwin" ]]; then
cut_opt="--field="
else
cut_opt="-f "
fi
if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
else
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
fi
BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
SIM_DIR=$TAOS_DIR/sim
NODE_DIR=$SIM_DIR/$NODE_NAME
if [ -d "$SIM_DIR/$2" ];then
rm -rf $SIM_DIR/$2
fi
mv $SIM_DIR/$1 $SIM_DIR/$2
if [[ $2 =~ "dnode2" ]];then
sed -i 's/serverPort 7100/serverPort 7200/g' $SIM_DIR/$2/cfg/taos.cfg
sed -i 's/dnode1/dnode2/g' $SIM_DIR/$2/cfg/taos.cfg
sed -i 's/7100/7200/g' $SIM_DIR/$2/data/dnode/dnodeEps.json
elif [[ $2 =~ "dnode4" ]];then
sed -i 's/serverPort 7100/serverPort 7400/g' $SIM_DIR/$2/cfg/taos.cfg
sed -i 's/dnode1/dnode4/g' $SIM_DIR/$2/cfg/taos.cfg
sed -i 's/7100/7400/g' $SIM_DIR/dnode2/data/dnode/dnodeEps.json
sed -i 's/7100/7400/g' $SIM_DIR/dnode3/data/dnode/dnodeEps.json
sed -i 's/7100/7400/g' $SIM_DIR/$2/data/dnode/dnodeEps.json
fi

6
tests/script/test-all.bat
View File

@ -19,7 +19,7 @@ for /F "usebackq tokens=*" %%i in (!caseFile!) do (
call :GetTimeSeconds !time!
set time1=!_timeTemp!
echo Start at !time!
call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt || set /a errorlevel=8
if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
)
)
@ -62,7 +62,5 @@ goto :eof
:CheckSkipCase
set skipCase=false
if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true )
if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true )
@REM if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
:goto eof

0
tests/script/unique/arbitrator/check_cluster_cfg_para.sim → tests/script/tsim/backup/arbitrator/check_cluster_cfg_para.sim
View File

0
tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim → tests/script/tsim/backup/arbitrator/dn2_mn1_cache_file_sync.sim
View File

0
tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim → tests/script/tsim/backup/arbitrator/dn2_mn1_cache_file_sync_second.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_full_createTableFail.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_full_dropDnodeFail.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_multiCreateDropTable.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_r2_vnode_delDir.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_r3_vnode_delDir.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
View File

0
tests/script/unique/arbitrator/dn3_mn1_replica_change.sim → tests/script/tsim/backup/arbitrator/dn3_mn1_replica_change.sim
View File

Some files were not shown because too many files have changed in this diff Show More