xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

other: merge 3.0

This commit is contained in:
Haojun Liao 2022-04-28 15:40:03 +08:00
parent 3227a413bc 0260686faa
commit 96363c411d
111 changed files with 6092 additions and 3766 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/systable.h
View File

@ -53,7 +53,8 @@ extern "C" {
#define TSDB_PERFS_TABLE_CONSUMERS "consumers" #define TSDB_PERFS_TABLE_CONSUMERS "consumers"
#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" #define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
#define TSDB_PERFS_TABLE_OFFSETS "offsets" #define TSDB_PERFS_TABLE_OFFSETS "offsets"
#define TSDB_PERFS_TABLE_TRANS "trans" #define TSDB_PERFS_TABLE_TRANS "trans"
#define TSDB_PERFS_TABLE_STREAMS "streams"
typedef struct SSysDbTableSchema { typedef struct SSysDbTableSchema {
const char *name; const char *name;

2
include/common/taosdef.h
View File

@ -82,6 +82,8 @@ extern char *qtypeStr[];
#define TSDB_PORT_HTTP 11 #define TSDB_PORT_HTTP 11
#undef TD_DEBUG_PRINT_ROW
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
include/common/tmsg.h
View File

@ -1281,8 +1281,9 @@ typedef struct {
#define STREAM_TRIGGER_WINDOW_CLOSE 2 #define STREAM_TRIGGER_WINDOW_CLOSE 2
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TABLE_FNAME_LEN];
char outputSTbName[TSDB_TABLE_FNAME_LEN]; char sourceDB[TSDB_DB_FNAME_LEN];
char targetStbFullName[TSDB_TABLE_FNAME_LEN];
int8_t igExists; int8_t igExists;
char* sql; char* sql;
char* ast; char* ast;
@ -1521,8 +1522,8 @@ typedef struct {
char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2 char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2
} SRSmaParam; } SRSmaParam;
int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam); int32_t tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
int tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam); int32_t tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
typedef struct SVCreateStbReq { typedef struct SVCreateStbReq {
const char* name; const char* name;

29
include/common/trow.h
View File

@ -214,6 +214,16 @@ STSRow *tdRowDup(STSRow *row);
static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) { static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx; return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx;
} }
static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
ASSERT(idx >= 0);
if (idx == 0) {
return PRIMARYKEY_TIMESTAMP_COL_ID;
}
return ((SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx - 1)->colId;
}
static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); } static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); }
#define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow? #define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow?
@ -668,7 +678,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
case TD_ROW_KV: case TD_ROW_KV:
#ifdef TD_SUPPORT_BITMAP #ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
#endif #endif
len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
pBuilder->nBoundBitmaps; // add pBuilder->nBoundBitmaps; // add
@ -1092,7 +1102,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
STSRow *pRow = pIter->pRow; STSRow *pRow = pIter->pRow;
SKvRowIdx *pKvIdx = NULL; SKvRowIdx *pKvIdx = NULL;
bool colFound = false; bool colFound = false;
col_id_t kvNCols = tdRowGetNCols(pRow); col_id_t kvNCols = tdRowGetNCols(pRow) - 1;
while (*nIdx < kvNCols) { while (*nIdx < kvNCols) {
pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
if (pKvIdx->colId == colId) { if (pKvIdx->colId == colId) {
@ -1108,7 +1118,14 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
} }
} }
if (!colFound) return false; if (!colFound) {
if (colId <= pIter->maxColId) {
pVal->valType = TD_VTYPE_NONE;
return true;
} else {
return false;
}
}
#ifdef TD_SUPPORT_BITMAP #ifdef TD_SUPPORT_BITMAP
int16_t colIdx = -1; int16_t colIdx = -1;
@ -1352,14 +1369,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
} }
} }
static void tdSRowPrint(STSRow *row, STSchema *pSchema) { static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) {
STSRowIter iter = {0}; STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterInit(&iter, pSchema);
tdSTSRowIterReset(&iter, row); tdSTSRowIterReset(&iter, row);
printf(">>>"); printf("%s >>>", tag);
for (int i = 0; i < pSchema->numOfCols; ++i) { for (int i = 0; i < pSchema->numOfCols; ++i) {
STColumn *stCol = pSchema->columns + i; STColumn *stCol = pSchema->columns + i;
SCellVal sVal = { 255, NULL}; SCellVal sVal = {.valType = 255, .val = NULL};
if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
break; break;
} }

474
include/common/ttokendef.h
View File

@ -16,244 +16,244 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_USER 33 #define TK_USER 33
#define TK_PRIVILEGE 34 #define TK_PRIVILEGE 34
#define TK_DROP 35 #define TK_DROP 35
#define TK_DNODE 36 #define TK_DNODE 36
#define TK_PORT 37 #define TK_PORT 37
#define TK_NK_INTEGER 38 #define TK_NK_INTEGER 38
#define TK_DNODES 39 #define TK_DNODES 39
#define TK_NK_IPTOKEN 40 #define TK_NK_IPTOKEN 40
#define TK_LOCAL 41 #define TK_LOCAL 41
#define TK_QNODE 42 #define TK_QNODE 42
#define TK_ON 43 #define TK_ON 43
#define TK_BNODE 44 #define TK_BNODE 44
#define TK_SNODE 45 #define TK_SNODE 45
#define TK_MNODE 46 #define TK_MNODE 46
#define TK_DATABASE 47 #define TK_DATABASE 47
#define TK_USE 48 #define TK_USE 48
#define TK_IF 49 #define TK_IF 49
#define TK_NOT 50 #define TK_NOT 50
#define TK_EXISTS 51 #define TK_EXISTS 51
#define TK_BLOCKS 52 #define TK_BUFFER 52
#define TK_CACHE 53 #define TK_CACHELAST 53
#define TK_CACHELAST 54 #define TK_COMP 54
#define TK_COMP 55 #define TK_DAYS 55
#define TK_DAYS 56 #define TK_NK_VARIABLE 56
#define TK_NK_VARIABLE 57 #define TK_FSYNC 57
#define TK_FSYNC 58 #define TK_MAXROWS 58
#define TK_MAXROWS 59 #define TK_MINROWS 59
#define TK_MINROWS 60 #define TK_KEEP 60
#define TK_KEEP 61 #define TK_PAGES 61
#define TK_PRECISION 62 #define TK_PAGESIZE 62
#define TK_QUORUM 63 #define TK_PRECISION 63
#define TK_REPLICA 64 #define TK_REPLICA 64
#define TK_TTL 65 #define TK_STRICT 65
#define TK_WAL 66 #define TK_WAL 66
#define TK_VGROUPS 67 #define TK_VGROUPS 67
#define TK_SINGLE_STABLE 68 #define TK_SINGLE_STABLE 68
#define TK_STREAM_MODE 69 #define TK_RETENTIONS 69
#define TK_RETENTIONS 70 #define TK_NK_COMMA 70
#define TK_STRICT 71 #define TK_NK_COLON 71
#define TK_NK_COMMA 72 #define TK_TABLE 72
#define TK_NK_COLON 73 #define TK_NK_LP 73
#define TK_TABLE 74 #define TK_NK_RP 74
#define TK_NK_LP 75 #define TK_STABLE 75
#define TK_NK_RP 76 #define TK_ADD 76
#define TK_STABLE 77 #define TK_COLUMN 77
#define TK_ADD 78 #define TK_MODIFY 78
#define TK_COLUMN 79 #define TK_RENAME 79
#define TK_MODIFY 80 #define TK_TAG 80
#define TK_RENAME 81 #define TK_SET 81
#define TK_TAG 82 #define TK_NK_EQ 82
#define TK_SET 83 #define TK_USING 83
#define TK_NK_EQ 84 #define TK_TAGS 84
#define TK_USING 85 #define TK_NK_DOT 85
#define TK_TAGS 86 #define TK_COMMENT 86
#define TK_NK_DOT 87 #define TK_BOOL 87
#define TK_COMMENT 88 #define TK_TINYINT 88
#define TK_BOOL 89 #define TK_SMALLINT 89
#define TK_TINYINT 90 #define TK_INT 90
#define TK_SMALLINT 91 #define TK_INTEGER 91
#define TK_INT 92 #define TK_BIGINT 92
#define TK_INTEGER 93 #define TK_FLOAT 93
#define TK_BIGINT 94 #define TK_DOUBLE 94
#define TK_FLOAT 95 #define TK_BINARY 95
#define TK_DOUBLE 96 #define TK_TIMESTAMP 96
#define TK_BINARY 97 #define TK_NCHAR 97
#define TK_TIMESTAMP 98 #define TK_UNSIGNED 98
#define TK_NCHAR 99 #define TK_JSON 99
#define TK_UNSIGNED 100 #define TK_VARCHAR 100
#define TK_JSON 101 #define TK_MEDIUMBLOB 101
#define TK_VARCHAR 102 #define TK_BLOB 102
#define TK_MEDIUMBLOB 103 #define TK_VARBINARY 103
#define TK_BLOB 104 #define TK_DECIMAL 104
#define TK_VARBINARY 105 #define TK_DELAY 105
#define TK_DECIMAL 106 #define TK_FILE_FACTOR 106
#define TK_SMA 107 #define TK_NK_FLOAT 107
#define TK_ROLLUP 108 #define TK_ROLLUP 108
#define TK_FILE_FACTOR 109 #define TK_TTL 109
#define TK_NK_FLOAT 110 #define TK_SMA 110
#define TK_DELAY 111 #define TK_SHOW 111
#define TK_SHOW 112 #define TK_DATABASES 112
#define TK_DATABASES 113 #define TK_TABLES 113
#define TK_TABLES 114 #define TK_STABLES 114
#define TK_STABLES 115 #define TK_MNODES 115
#define TK_MNODES 116 #define TK_MODULES 116
#define TK_MODULES 117 #define TK_QNODES 117
#define TK_QNODES 118 #define TK_FUNCTIONS 118
#define TK_FUNCTIONS 119 #define TK_INDEXES 119
#define TK_INDEXES 120 #define TK_FROM 120
#define TK_FROM 121 #define TK_ACCOUNTS 121
#define TK_ACCOUNTS 122 #define TK_APPS 122
#define TK_APPS 123 #define TK_CONNECTIONS 123
#define TK_CONNECTIONS 124 #define TK_LICENCE 124
#define TK_LICENCE 125 #define TK_GRANTS 125
#define TK_GRANTS 126 #define TK_QUERIES 126
#define TK_QUERIES 127 #define TK_SCORES 127
#define TK_SCORES 128 #define TK_TOPICS 128
#define TK_TOPICS 129 #define TK_VARIABLES 129
#define TK_VARIABLES 130 #define TK_BNODES 130
#define TK_BNODES 131 #define TK_SNODES 131
#define TK_SNODES 132 #define TK_CLUSTER 132
#define TK_CLUSTER 133 #define TK_LIKE 133
#define TK_LIKE 134 #define TK_INDEX 134
#define TK_INDEX 135 #define TK_FULLTEXT 135
#define TK_FULLTEXT 136 #define TK_FUNCTION 136
#define TK_FUNCTION 137 #define TK_INTERVAL 137
#define TK_INTERVAL 138 #define TK_TOPIC 138
#define TK_TOPIC 139 #define TK_AS 139
#define TK_AS 140 #define TK_WITH 140
#define TK_WITH 141 #define TK_SCHEMA 141
#define TK_SCHEMA 142 #define TK_DESC 142
#define TK_DESC 143 #define TK_DESCRIBE 143
#define TK_DESCRIBE 144 #define TK_RESET 144
#define TK_RESET 145 #define TK_QUERY 145
#define TK_QUERY 146 #define TK_CACHE 146
#define TK_EXPLAIN 147 #define TK_EXPLAIN 147
#define TK_ANALYZE 148 #define TK_ANALYZE 148
#define TK_VERBOSE 149 #define TK_VERBOSE 149
#define TK_NK_BOOL 150 #define TK_NK_BOOL 150
#define TK_RATIO 151 #define TK_RATIO 151
#define TK_COMPACT 152 #define TK_COMPACT 152
#define TK_VNODES 153 #define TK_VNODES 153
#define TK_IN 154 #define TK_IN 154
#define TK_OUTPUTTYPE 155 #define TK_OUTPUTTYPE 155
#define TK_AGGREGATE 156 #define TK_AGGREGATE 156
#define TK_BUFSIZE 157 #define TK_BUFSIZE 157
#define TK_STREAM 158 #define TK_STREAM 158
#define TK_INTO 159 #define TK_INTO 159
#define TK_TRIGGER 160 #define TK_TRIGGER 160
#define TK_AT_ONCE 161 #define TK_AT_ONCE 161
#define TK_WINDOW_CLOSE 162 #define TK_WINDOW_CLOSE 162
#define TK_WATERMARK 163 #define TK_WATERMARK 163
#define TK_KILL 164 #define TK_KILL 164
#define TK_CONNECTION 165 #define TK_CONNECTION 165
#define TK_MERGE 166 #define TK_MERGE 166
#define TK_VGROUP 167 #define TK_VGROUP 167
#define TK_REDISTRIBUTE 168 #define TK_REDISTRIBUTE 168
#define TK_SPLIT 169 #define TK_SPLIT 169
#define TK_SYNCDB 170 #define TK_SYNCDB 170
#define TK_NULL 171 #define TK_NULL 171
#define TK_NK_QUESTION 172 #define TK_NK_QUESTION 172
#define TK_NK_ARROW 173 #define TK_NK_ARROW 173
#define TK_ROWTS 174 #define TK_ROWTS 174
#define TK_TBNAME 175 #define TK_TBNAME 175
#define TK_QSTARTTS 176 #define TK_QSTARTTS 176
#define TK_QENDTS 177 #define TK_QENDTS 177
#define TK_WSTARTTS 178 #define TK_WSTARTTS 178
#define TK_WENDTS 179 #define TK_WENDTS 179
#define TK_WDURATION 180 #define TK_WDURATION 180
#define TK_CAST 181 #define TK_CAST 181
#define TK_NOW 182 #define TK_NOW 182
#define TK_TODAY 183 #define TK_TODAY 183
#define TK_TIMEZONE 184 #define TK_TIMEZONE 184
#define TK_COUNT 185 #define TK_COUNT 185
#define TK_FIRST 186 #define TK_FIRST 186
#define TK_LAST 187 #define TK_LAST 187
#define TK_LAST_ROW 188 #define TK_LAST_ROW 188
#define TK_BETWEEN 189 #define TK_BETWEEN 189
#define TK_IS 190 #define TK_IS 190
#define TK_NK_LT 191 #define TK_NK_LT 191
#define TK_NK_GT 192 #define TK_NK_GT 192
#define TK_NK_LE 193 #define TK_NK_LE 193
#define TK_NK_GE 194 #define TK_NK_GE 194
#define TK_NK_NE 195 #define TK_NK_NE 195
#define TK_MATCH 196 #define TK_MATCH 196
#define TK_NMATCH 197 #define TK_NMATCH 197
#define TK_CONTAINS 198 #define TK_CONTAINS 198
#define TK_JOIN 199 #define TK_JOIN 199
#define TK_INNER 200 #define TK_INNER 200
#define TK_SELECT 201 #define TK_SELECT 201
#define TK_DISTINCT 202 #define TK_DISTINCT 202
#define TK_WHERE 203 #define TK_WHERE 203
#define TK_PARTITION 204 #define TK_PARTITION 204
#define TK_BY 205 #define TK_BY 205
#define TK_SESSION 206 #define TK_SESSION 206
#define TK_STATE_WINDOW 207 #define TK_STATE_WINDOW 207
#define TK_SLIDING 208 #define TK_SLIDING 208
#define TK_FILL 209 #define TK_FILL 209
#define TK_VALUE 210 #define TK_VALUE 210
#define TK_NONE 211 #define TK_NONE 211
#define TK_PREV 212 #define TK_PREV 212
#define TK_LINEAR 213 #define TK_LINEAR 213
#define TK_NEXT 214 #define TK_NEXT 214
#define TK_GROUP 215 #define TK_GROUP 215
#define TK_HAVING 216 #define TK_HAVING 216
#define TK_ORDER 217 #define TK_ORDER 217
#define TK_SLIMIT 218 #define TK_SLIMIT 218
#define TK_SOFFSET 219 #define TK_SOFFSET 219
#define TK_LIMIT 220 #define TK_LIMIT 220
#define TK_OFFSET 221 #define TK_OFFSET 221
#define TK_ASC 222 #define TK_ASC 222
#define TK_NULLS 223 #define TK_NULLS 223
#define TK_ID 224 #define TK_ID 224
#define TK_NK_BITNOT 225 #define TK_NK_BITNOT 225
#define TK_INSERT 226 #define TK_INSERT 226
#define TK_VALUES 227 #define TK_VALUES 227
#define TK_IMPORT 228 #define TK_IMPORT 228
#define TK_NK_SEMI 229 #define TK_NK_SEMI 229
#define TK_FILE 230 #define TK_FILE 230
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301
#define TK_NK_ILLEGAL 302 #define TK_NK_ILLEGAL 302
#define TK_NK_HEX 303 // hex number 0x123 #define TK_NK_HEX 303 // hex number 0x123
#define TK_NK_OCT 304 // oct number #define TK_NK_OCT 304 // oct number
#define TK_NK_BIN 305 // bin format data 0b111 #define TK_NK_BIN 305 // bin format data 0b111
#define TK_NK_NIL 65535 #define TK_NK_NIL 65535
#endif /*_TD_COMMON_TOKEN_H_*/ #endif /*_TD_COMMON_TOKEN_H_*/

1
include/dnode/mnode/sdb/sdb.h
View File

@ -331,7 +331,6 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize); SSdbRow *sdbAllocRow(int32_t objSize);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
void *sdbGetRowObj(SSdbRow *pRow); void *sdbGetRowObj(SSdbRow *pRow);
typedef struct SSdb { typedef struct SSdb {

4
include/libs/function/function.h
View File

@ -206,6 +206,8 @@ typedef struct SqlFunctionCtx {
struct SDiskbasedBuf *pBuf; struct SDiskbasedBuf *pBuf;
struct SSDataBlock *pSrcBlock; struct SSDataBlock *pSrcBlock;
int32_t curBufPage; int32_t curBufPage;
char udfName[TSDB_FUNC_NAME_LEN];
} SqlFunctionCtx; } SqlFunctionCtx;
enum { enum {
@ -334,8 +336,6 @@ int32_t udfcOpen();
*/ */
int32_t udfcClose(); int32_t udfcClose();
typedef void *UdfcFuncHandle;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
include/libs/function/functionMgt.h
View File

@ -162,6 +162,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
#ifdef __cplusplus #ifdef __cplusplus
} }

16
source/libs/function/inc/tudf.h → include/libs/function/tudf.h
View File

@ -42,8 +42,7 @@ enum {
UDFC_CODE_INVALID_STATE = -5 UDFC_CODE_INVALID_STATE = -5
}; };
typedef void *UdfcFuncHandle;
/** /**
* setup udf * setup udf
@ -95,6 +94,7 @@ typedef struct SUdfDataBlock {
typedef struct SUdfInterBuf { typedef struct SUdfInterBuf {
int32_t bufLen; int32_t bufLen;
char* buf; char* buf;
int8_t numOfResult; //zero or one
} SUdfInterBuf; } SUdfInterBuf;
// output: interBuf // output: interBuf
@ -118,6 +118,10 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
*/ */
int32_t teardownUdf(UdfcFuncHandle handle); int32_t teardownUdf(UdfcFuncHandle handle);
bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
// end API to taosd and qworker // end API to taosd and qworker
//============================================================================================================================= //=============================================================================================================================
// begin API to UDF writer. // begin API to UDF writer.
@ -133,11 +137,11 @@ typedef int32_t (*TUdfTeardownFunc)();
//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data); //typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf); typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf); typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
// end API to UDF writer // end API to UDF writer

269
include/libs/nodes/cmdnodes.h
View File

@ -23,294 +23,291 @@ extern "C" {
#include "query.h" #include "query.h"
#include "querynodes.h" #include "querynodes.h"
#define DESCRIBE_RESULT_COLS 4 #define DESCRIBE_RESULT_COLS 4
#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
typedef struct SDatabaseOptions { typedef struct SDatabaseOptions {
ENodeType type; ENodeType type;
SValueNode* pNumOfBlocks; int32_t buffer;
SValueNode* pCacheBlockSize; int8_t cachelast;
SValueNode* pCachelast; int8_t compressionLevel;
SValueNode* pCompressionLevel; int32_t daysPerFile;
SValueNode* pDaysPerFile; SValueNode* pDaysPerFile;
SValueNode* pFsyncPeriod; int32_t fsyncPeriod;
SValueNode* pMaxRowsPerBlock; int32_t maxRowsPerBlock;
SValueNode* pMinRowsPerBlock; int32_t minRowsPerBlock;
SNodeList* pKeep; SNodeList* pKeep;
SValueNode* pPrecision; int32_t keep[3];
SValueNode* pQuorum; int32_t pages;
SValueNode* pReplica; int32_t pagesize;
SValueNode* pTtl; char precisionStr[3];
SValueNode* pWalLevel; int8_t precision;
SValueNode* pNumOfVgroups; int8_t replica;
SValueNode* pSingleStable; int8_t strict;
SValueNode* pStreamMode; int8_t walLevel;
SValueNode* pStrict; int32_t numOfVgroups;
SNodeList* pRetentions; int8_t singleStable;
SNodeList* pRetentions;
} SDatabaseOptions; } SDatabaseOptions;
typedef struct SCreateDatabaseStmt { typedef struct SCreateDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SDatabaseOptions* pOptions; SDatabaseOptions* pOptions;
} SCreateDatabaseStmt; } SCreateDatabaseStmt;
typedef struct SUseDatabaseStmt { typedef struct SUseDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
} SUseDatabaseStmt; } SUseDatabaseStmt;
typedef struct SDropDatabaseStmt { typedef struct SDropDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropDatabaseStmt; } SDropDatabaseStmt;
typedef struct SAlterDatabaseStmt { typedef struct SAlterDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
SDatabaseOptions* pOptions; SDatabaseOptions* pOptions;
} SAlterDatabaseStmt; } SAlterDatabaseStmt;
typedef struct STableOptions { typedef struct STableOptions {
ENodeType type; ENodeType type;
SNodeList* pKeep; char comment[TSDB_STB_COMMENT_LEN];
SValueNode* pTtl; int32_t delay;
SValueNode* pComments; float filesFactor;
SNodeList* pRollupFuncs;
int32_t ttl;
SNodeList* pSma; SNodeList* pSma;
SNodeList* pFuncs;
SValueNode* pFilesFactor;
SValueNode* pDelay;
} STableOptions; } STableOptions;
typedef struct SColumnDefNode { typedef struct SColumnDefNode {
ENodeType type; ENodeType type;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SDataType dataType; SDataType dataType;
char comments[TSDB_STB_COMMENT_LEN]; char comments[TSDB_STB_COMMENT_LEN];
bool sma; bool sma;
} SColumnDefNode; } SColumnDefNode;
typedef struct SCreateTableStmt { typedef struct SCreateTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNodeList* pCols; SNodeList* pCols;
SNodeList* pTags; SNodeList* pTags;
STableOptions* pOptions; STableOptions* pOptions;
} SCreateTableStmt; } SCreateTableStmt;
typedef struct SCreateSubTableClause { typedef struct SCreateSubTableClause {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
char useDbName[TSDB_DB_NAME_LEN]; char useDbName[TSDB_DB_NAME_LEN];
char useTableName[TSDB_TABLE_NAME_LEN]; char useTableName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNodeList* pSpecificTags; SNodeList* pSpecificTags;
SNodeList* pValsOfTags; SNodeList* pValsOfTags;
} SCreateSubTableClause; } SCreateSubTableClause;
typedef struct SCreateMultiTableStmt { typedef struct SCreateMultiTableStmt {
ENodeType type; ENodeType type;
SNodeList* pSubTables; SNodeList* pSubTables;
} SCreateMultiTableStmt; } SCreateMultiTableStmt;
typedef struct SDropTableClause { typedef struct SDropTableClause {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropTableClause; } SDropTableClause;
typedef struct SDropTableStmt { typedef struct SDropTableStmt {
ENodeType type; ENodeType type;
SNodeList* pTables; SNodeList* pTables;
} SDropTableStmt; } SDropTableStmt;
typedef struct SDropSuperTableStmt { typedef struct SDropSuperTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropSuperTableStmt; } SDropSuperTableStmt;
typedef struct SAlterTableStmt { typedef struct SAlterTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType; int8_t alterType;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN]; char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions; STableOptions* pOptions;
SDataType dataType; SDataType dataType;
SValueNode* pVal; SValueNode* pVal;
} SAlterTableStmt; } SAlterTableStmt;
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
} SCreateUserStmt; } SCreateUserStmt;
typedef struct SAlterUserStmt { typedef struct SAlterUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t alterType; int8_t alterType;
} SAlterUserStmt; } SAlterUserStmt;
typedef struct SDropUserStmt { typedef struct SDropUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
} SDropUserStmt; } SDropUserStmt;
typedef struct SCreateDnodeStmt { typedef struct SCreateDnodeStmt {
ENodeType type; ENodeType type;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
int32_t port; int32_t port;
} SCreateDnodeStmt; } SCreateDnodeStmt;
typedef struct SDropDnodeStmt { typedef struct SDropDnodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
int32_t port; int32_t port;
} SDropDnodeStmt; } SDropDnodeStmt;
typedef struct SAlterDnodeStmt { typedef struct SAlterDnodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN]; char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt; } SAlterDnodeStmt;
typedef struct SShowStmt { typedef struct SShowStmt {
ENodeType type; ENodeType type;
SNode* pDbName; // SValueNode SNode* pDbName; // SValueNode
SNode* pTbNamePattern; // SValueNode SNode* pTbNamePattern; // SValueNode
} SShowStmt; } SShowStmt;
typedef struct SShowCreatStmt { typedef struct SShowCreatStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
} SShowCreatStmt; } SShowCreatStmt;
typedef enum EIndexType { typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType;
INDEX_TYPE_SMA = 1,
INDEX_TYPE_FULLTEXT
} EIndexType;
typedef struct SIndexOptions { typedef struct SIndexOptions {
ENodeType type; ENodeType type;
SNodeList* pFuncs; SNodeList* pFuncs;
SNode* pInterval; SNode* pInterval;
SNode* pOffset; SNode* pOffset;
SNode* pSliding; SNode* pSliding;
} SIndexOptions; } SIndexOptions;
typedef struct SCreateIndexStmt { typedef struct SCreateIndexStmt {
ENodeType type; ENodeType type;
EIndexType indexType; EIndexType indexType;
bool ignoreExists; bool ignoreExists;
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
SNodeList* pCols; SNodeList* pCols;
SIndexOptions* pOptions; SIndexOptions* pOptions;
} SCreateIndexStmt; } SCreateIndexStmt;
typedef struct SDropIndexStmt { typedef struct SDropIndexStmt {
ENodeType type; ENodeType type;
bool ignoreNotExists; bool ignoreNotExists;
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
} SDropIndexStmt; } SDropIndexStmt;
typedef struct SCreateComponentNodeStmt { typedef struct SCreateComponentNodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
} SCreateComponentNodeStmt; } SCreateComponentNodeStmt;
typedef struct SDropComponentNodeStmt { typedef struct SDropComponentNodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
} SDropComponentNodeStmt; } SDropComponentNodeStmt;
typedef struct STopicOptions { typedef struct STopicOptions {
ENodeType type; ENodeType type;
bool withTable; bool withTable;
bool withSchema; bool withSchema;
bool withTag; bool withTag;
} STopicOptions; } STopicOptions;
typedef struct SCreateTopicStmt { typedef struct SCreateTopicStmt {
ENodeType type; ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN]; char topicName[TSDB_TABLE_NAME_LEN];
char subscribeDbName[TSDB_DB_NAME_LEN]; char subscribeDbName[TSDB_DB_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNode* pQuery; SNode* pQuery;
STopicOptions* pOptions; STopicOptions* pOptions;
} SCreateTopicStmt; } SCreateTopicStmt;
typedef struct SDropTopicStmt { typedef struct SDropTopicStmt {
ENodeType type; ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN]; char topicName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropTopicStmt; } SDropTopicStmt;
typedef struct SAlterLocalStmt { typedef struct SAlterLocalStmt {
ENodeType type; ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN]; char value[TSDB_DNODE_VALUE_LEN];
} SAlterLocalStmt; } SAlterLocalStmt;
typedef struct SDescribeStmt { typedef struct SDescribeStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
STableMeta* pMeta; STableMeta* pMeta;
} SDescribeStmt; } SDescribeStmt;
typedef struct SKillStmt { typedef struct SKillStmt {
ENodeType type; ENodeType type;
int32_t targetId; int32_t targetId;
} SKillStmt; } SKillStmt;
typedef struct SStreamOptions { typedef struct SStreamOptions {
ENodeType type; ENodeType type;
int8_t triggerType; int8_t triggerType;
SNode* pWatermark; SNode* pWatermark;
} SStreamOptions; } SStreamOptions;
typedef struct SCreateStreamStmt { typedef struct SCreateStreamStmt {
ENodeType type; ENodeType type;
char streamName[TSDB_TABLE_NAME_LEN]; char streamName[TSDB_TABLE_NAME_LEN];
char targetDbName[TSDB_DB_NAME_LEN]; char targetDbName[TSDB_DB_NAME_LEN];
char targetTabName[TSDB_TABLE_NAME_LEN]; char targetTabName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SStreamOptions* pOptions; SStreamOptions* pOptions;
SNode* pQuery; SNode* pQuery;
} SCreateStreamStmt; } SCreateStreamStmt;
typedef struct SDropStreamStmt { typedef struct SDropStreamStmt {
ENodeType type; ENodeType type;
char streamName[TSDB_TABLE_NAME_LEN]; char streamName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropStreamStmt; } SDropStreamStmt;
typedef struct SCreateFunctionStmt { typedef struct SCreateFunctionStmt {
ENodeType type; ENodeType type;
bool ignoreExists; bool ignoreExists;
char funcName[TSDB_FUNC_NAME_LEN]; char funcName[TSDB_FUNC_NAME_LEN];
bool isAgg; bool isAgg;
char libraryPath[PATH_MAX]; char libraryPath[PATH_MAX];
SDataType outputDt; SDataType outputDt;
int32_t bufSize; int32_t bufSize;
} SCreateFunctionStmt; } SCreateFunctionStmt;
#ifdef __cplusplus #ifdef __cplusplus

2
include/libs/nodes/plannodes.h
View File

@ -340,7 +340,7 @@ typedef struct SQueryPlan {
int32_t numOfSubplans; int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
SExplainInfo explainInfo; SExplainInfo explainInfo;
SNodeList* pPlaceholderValues; SArray* pPlaceholderValues;
} SQueryPlan; } SQueryPlan;
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

1
include/libs/parser/parser.h
View File

@ -73,6 +73,7 @@ typedef struct SQuery {
SArray* pDbList; SArray* pDbList;
SArray* pTableList; SArray* pTableList;
bool showRewrite; bool showRewrite;
int32_t placeholderNum;
} SQuery; } SQuery;
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

4
include/libs/planner/planner.h
View File

@ -34,7 +34,7 @@ typedef struct SPlanContext {
bool showRewrite; bool showRewrite;
int8_t triggerType; int8_t triggerType;
int64_t watermark; int64_t watermark;
bool isStmtQuery; int32_t placeholderNum;
void* pTransporter; void* pTransporter;
struct SCatalog* pCatalog; struct SCatalog* pCatalog;
char* pMsg; char* pMsg;
@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
// @pSource one execution location of this group of datasource subplans // @pSource one execution location of this group of datasource subplans
int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
// Convert to subplan to string for the scheduler to send to the executor // Convert to subplan to string for the scheduler to send to the executor
int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

46
include/libs/stream/tstreamUpdate.h Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_
#include "taosdef.h"
#include "tarray.h"
#include "tmsg.h"
#include "tscalablebf.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SUpdateInfo {
SArray *pTsBuckets;
uint64_t numBuckets;
SArray *pTsSBFs;
uint64_t numSBFs;
int64_t interval;
int64_t watermark;
TSKEY minTS;
} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
void updateInfoDestroy(SUpdateInfo *pInfo);
#ifdef __cplusplus
}
#endif
#endif /* ifndef _TSTREAMUPDATE_H_ */

1
include/util/taoserror.h
View File

@ -586,7 +586,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619) #define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619)
#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A) #define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A)
#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B) #define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B)
#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x261C)
#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D) #define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D)
#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E) #define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E)
#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F) #define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F)

7
include/util/tarray.h
View File

@ -218,6 +218,13 @@ void taosArrayClear(SArray* pArray);
*/ */
void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
/**
* clear the array (remove all element)
* @param pArray
* @param fp
*/
void taosArrayClearP(SArray* pArray, FDelete fp);
void* taosArrayDestroy(SArray* pArray); void* taosArrayDestroy(SArray* pArray);
void taosArrayDestroyP(SArray* pArray, FDelete fp); void taosArrayDestroyP(SArray* pArray, FDelete fp);
void taosArrayDestroyEx(SArray* pArray, FDelete fp); void taosArrayDestroyEx(SArray* pArray, FDelete fp);

50
include/util/tbloomfilter.h Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_BLOOMFILTER_H_
#define _TD_UTIL_BLOOMFILTER_H_
#include "os.h"
#include "thash.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SBloomFilter {
uint32_t hashFunctions;
uint64_t expectedEntries;
uint64_t numUnits;
uint64_t numBits;
uint64_t size;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
void *buffer;
double errorRate;
} SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
uint32_t len);
void tBloomFilterDestroy(SBloomFilter *pBF);
void tBloomFilterDump(const SBloomFilter *pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_BLOOMFILTER_H_*/

138
include/util/tdef.h
View File

@ -277,72 +277,80 @@ typedef enum ELogicConditionType {
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
#define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096 #define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_DEFAULT_VN_PER_DB 2
#define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MIN_CACHE_BLOCK_SIZE 1
#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode #define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
#define TSDB_MIN_TOTAL_BLOCKS 3 #define TSDB_MIN_TOTAL_BLOCKS 3
#define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 6 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MIN_DAYS_PER_FILE 60 // unit minute
#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000 #define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100 #define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200 #define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000 #define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096 #define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_COMMIT_TIME 30 #define TSDB_MIN_COMMIT_TIME 30
#define TSDB_MAX_COMMIT_TIME 40960 #define TSDB_MAX_COMMIT_TIME 40960
#define TSDB_DEFAULT_COMMIT_TIME 3600 #define TSDB_DEFAULT_COMMIT_TIME 3600
#define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
#define TSDB_MIN_DB_TTL 1 #define TSDB_MIN_WAL_LEVEL 1
#define TSDB_DEFAULT_DB_TTL 1 #define TSDB_MAX_WAL_LEVEL 2
#define TSDB_MIN_WAL_LEVEL 1 #define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MAX_WAL_LEVEL 2 #define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO
#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO #define TSDB_MIN_COMP_LEVEL 0
#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_MAX_COMP_LEVEL 2
#define TSDB_MIN_COMP_LEVEL 0 #define TSDB_DEFAULT_COMP_LEVEL 2
#define TSDB_MAX_COMP_LEVEL 2 #define TSDB_MIN_DB_REPLICA 1
#define TSDB_DEFAULT_COMP_LEVEL 2 #define TSDB_MAX_DB_REPLICA 3
#define TSDB_MIN_DB_REPLICA 1 #define TSDB_DEFAULT_DB_REPLICA 1
#define TSDB_MAX_DB_REPLICA 3 #define TSDB_DB_STRICT_OFF 0
#define TSDB_DEFAULT_DB_REPLICA 1 #define TSDB_DB_STRICT_ON 1
#define TSDB_DB_STRICT_OFF 0 #define TSDB_DEFAULT_DB_STRICT 0
#define TSDB_DB_STRICT_ON 1 #define TSDB_MIN_DB_UPDATE 0
#define TSDB_DEFAULT_DB_STRICT 0 #define TSDB_MAX_DB_UPDATE 2
#define TSDB_MIN_DB_UPDATE 0 #define TSDB_DEFAULT_DB_UPDATE 0
#define TSDB_MAX_DB_UPDATE 2 #define TSDB_MIN_DB_CACHE_LAST_ROW 0
#define TSDB_DEFAULT_DB_UPDATE 0 #define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_MIN_DB_CACHE_LAST_ROW 0 #define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_DB_STREAM_MODE_OFF 0
#define TSDB_DEFAULT_CACHE_LAST_ROW 0 #define TSDB_DB_STREAM_MODE_ON 1
#define TSDB_DB_STREAM_MODE_OFF 0 #define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_DB_STREAM_MODE_ON 1 #define TSDB_DB_SINGLE_STABLE_ON 0
#define TSDB_DEFAULT_DB_STREAM_MODE 0 #define TSDB_DB_SINGLE_STABLE_OFF 1
#define TSDB_DB_SINGLE_STABLE_ON 0 #define TSDB_DEFAULT_DB_SINGLE_STABLE 0
#define TSDB_DB_SINGLE_STABLE_OFF 1 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB
#define TSDB_DEFAULT_DB_SINGLE_STABLE 0 #define TSDB_DEFAULT_BUFFER_PER_VNODE 96
#define TSDB_MIN_PAGES_PER_VNODE 64
#define TSDB_DEFAULT_PAGES_PER_VNODE 256
#define TSDB_MIN_PAGESIZE_PER_VNODE 1 // unit KB
#define TSDB_MAX_PAGESIZE_PER_VNODE 16384
#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
#define TSDB_MIN_DB_FILE_FACTOR 0 #define TSDB_MIN_ROLLUP_FILE_FACTOR 0
#define TSDB_MAX_DB_FILE_FACTOR 1 #define TSDB_MAX_ROLLUP_FILE_FACTOR 1
#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1 #define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1
#define TSDB_MIN_DB_DELAY 1 #define TSDB_MIN_ROLLUP_DELAY 1
#define TSDB_MAX_DB_DELAY 10 #define TSDB_MAX_ROLLUP_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2 #define TSDB_DEFAULT_ROLLUP_DELAY 2
#define TSDB_MIN_EXPLAIN_RATIO 0 #define TSDB_MIN_TABLE_TTL 0
#define TSDB_MAX_EXPLAIN_RATIO 1 #define TSDB_DEFAULT_TABLE_TTL 0
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_MIN_EXPLAIN_RATIO 0
#define TSDB_MAX_EXPLAIN_RATIO 1
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MAX_UNION_CLAUSE 5

42
include/util/tscalablebf.h Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_SCALABLEBF_H_
#define _TD_UTIL_SCALABLEBF_H_
#include "tbloomfilter.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
} SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf);
void tScalableBfDump(const SScalableBf *pSBf);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_SCALABLEBF_H_*/

5
source/client/inc/clientInt.h
View File

@ -308,9 +308,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
void hbMgrInitMqHbRspHandle(); void hbMgrInitMqHbRspHandle();
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
#ifdef __cplusplus #ifdef __cplusplus
} }

27
source/client/inc/clientStmt.h
View File

@ -46,6 +46,12 @@ typedef struct SStmtTableCache {
void* boundTags; void* boundTags;
} SStmtTableCache; } SStmtTableCache;
typedef struct SQueryFields {
TAOS_FIELD* fields;
TAOS_FIELD* userFields;
uint32_t numOfCols;
} SQueryFields;
typedef struct SStmtBindInfo { typedef struct SStmtBindInfo {
bool needParse; bool needParse;
uint64_t tbUid; uint64_t tbUid;
@ -66,16 +72,17 @@ typedef struct SStmtExecInfo {
} SStmtExecInfo; } SStmtExecInfo;
typedef struct SStmtSQLInfo { typedef struct SStmtSQLInfo {
STMT_TYPE type; STMT_TYPE type;
STMT_STATUS status; STMT_STATUS status;
bool autoCreate; bool autoCreate;
uint64_t runTimes; uint64_t runTimes;
SHashObj* pTableCache; //SHash<SStmtTableCache> SHashObj* pTableCache; //SHash<SStmtTableCache>
SQuery* pQuery; SQuery* pQuery;
char* sqlStr; char* sqlStr;
int32_t sqlLen; int32_t sqlLen;
SArray* nodeList; SArray* nodeList;
SQueryPlan* pQueryPlan; SQueryPlan* pQueryPlan;
SQueryFields fields;
} SStmtSQLInfo; } SStmtSQLInfo;
typedef struct STscStmt { typedef struct STscStmt {

3
source/client/src/clientImpl.c
View File

@ -234,7 +234,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.showRewrite = pQuery->showRewrite, .showRewrite = pQuery->showRewrite,
.pTransporter = pRequest->pTscObj->pAppInfo->pTransporter, .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
.pMsg = pRequest->msgBuf, .pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.placeholderNum = pQuery->placeholderNum};
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = qCreateQueryPlan(&cxt, pPlan, pNodeList); code = qCreateQueryPlan(&cxt, pPlan, pNodeList);

27
source/client/src/clientMain.c
View File

@ -110,16 +110,23 @@ int taos_errno(TAOS_RES *tres) {
return terrno; return terrno;
} }
if (TD_RES_TMQ(tres)) {
return 0;
}
return ((SRequestObj *)tres)->code; return ((SRequestObj *)tres)->code;
} }
const char *taos_errstr(TAOS_RES *res) { const char *taos_errstr(TAOS_RES *res) {
SRequestObj *pRequest = (SRequestObj *)res; if (res == NULL) {
if (pRequest == NULL) {
return (const char *)tstrerror(terrno); return (const char *)tstrerror(terrno);
} }
if (TD_RES_TMQ(res)) {
return "success";
}
SRequestObj *pRequest = (SRequestObj *)res;
if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf; return pRequest->msgBuf;
} else { } else {
@ -131,7 +138,7 @@ void taos_free_result(TAOS_RES *res) {
if (NULL == res) { if (NULL == res) {
return; return;
} }
if (TD_RES_QUERY(res)) { if (TD_RES_QUERY(res)) {
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest); destroyRequest(pRequest);
@ -632,9 +639,7 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
return stmtSetTbName(stmt, name); return stmtSetTbName(stmt, name);
} }
int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); }
return taos_stmt_set_tbname(stmt, name);
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
if (stmt == NULL || bind == NULL) { if (stmt == NULL || bind == NULL) {
@ -648,7 +653,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
return stmtBindBatch(stmt, bind, -1); return stmtBindBatch(stmt, bind, -1);
} }
@ -696,7 +701,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, in
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
return stmtBindBatch(stmt, bind, colIdx); return stmtBindBatch(stmt, bind, colIdx);
} }
@ -750,9 +755,7 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
return stmtUseResult(stmt); return stmtUseResult(stmt);
} }
char *taos_stmt_errstr(TAOS_STMT *stmt) { char *taos_stmt_errstr(TAOS_STMT *stmt) { return (char *)stmtErrstr(stmt); }
return (char *)stmtErrstr(stmt);
}
int taos_stmt_affected_rows(TAOS_STMT *stmt) { int taos_stmt_affected_rows(TAOS_STMT *stmt) {
if (stmt == NULL) { if (stmt == NULL) {

56
source/client/src/clientStmt.c
View File

@ -73,6 +73,22 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtBackupQueryFields(STscStmt* pStmt) {
SQueryFields *pFields = &pStmt->sql.fields;
int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD);
pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols;
pFields->fields = taosMemoryMalloc(size);
pFields->userFields = taosMemoryMalloc(size);
if (NULL == pFields->fields || NULL == pFields->userFields) {
STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
}
memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size);
memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size);
return TSDB_CODE_SUCCESS;
}
int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
@ -258,37 +274,42 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
uint64_t uid = pTableMeta->uid;
if (pTableMeta->uid == pStmt->bInfo.tbUid) { uint64_t suid = pTableMeta->suid;
int8_t tableType = pTableMeta->tableType;
taosMemoryFree(pTableMeta);
if (uid == pStmt->bInfo.tbUid) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) { if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) {
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
if (NULL == pCache) { if (NULL == pCache) {
tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid); tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid);
STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
} }
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbUid = uid;
pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.boundTags = pCache->boundTags;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
if (pCache) { if (pCache) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbUid = uid;
pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.boundTags = pCache->boundTags;
STableDataBlocks* pNewBlock = NULL; STableDataBlocks* pNewBlock = NULL;
@ -475,9 +496,10 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
pStmt->exec.pRequest->body.pDag = NULL; pStmt->exec.pRequest->body.pDag = NULL;
STMT_ERR_RET(stmtBackupQueryFields(pStmt));
} }
STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx)); STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId));
} }
STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
@ -549,6 +571,8 @@ int stmtClose(TAOS_STMT *stmt) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
STMT_RET(stmtCleanSQLInfo(pStmt)); STMT_RET(stmtCleanSQLInfo(pStmt));
taosMemoryFree(stmt);
} }
const char *stmtErrstr(TAOS_STMT *stmt) { const char *stmtErrstr(TAOS_STMT *stmt) {
@ -601,7 +625,7 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
pStmt->exec.pRequest->body.pDag = NULL; pStmt->exec.pRequest->body.pDag = NULL;
} }
*nums = (pStmt->sql.pQueryPlan->pPlaceholderValues) ? pStmt->sql.pQueryPlan->pPlaceholderValues->length : 0; *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
} else { } else {
STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
} }

2
source/client/src/tmq.c
View File

@ -739,7 +739,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
.sql = (char*)sql, .sql = (char*)sql,
}; };
tNameExtractFullName(&name, req.name); tNameExtractFullName(&name, req.name);
strcpy(req.outputSTbName, tbName); strcpy(req.targetStbFullName, tbName);
int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req); int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
void* buf = taosMemoryMalloc(tlen); void* buf = taosMemoryMalloc(tlen);

9
source/client/test/clientTests.cpp
View File

@ -659,10 +659,15 @@ TEST(testCase, agg_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use db");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "select now() from m1"); pRes = taos_query(pConn, "select tbname from st1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);

13
source/common/src/systable.c
View File

@ -306,6 +306,18 @@ static const SSysDbTableSchema querySchema[] = {
{.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
}; };
static const SSysDbTableSchema streamSchema[] = {
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = TSDB_SHOW_SQL_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SSysTableMeta perfsMeta[] = { static const SSysTableMeta perfsMeta[] = {
{TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)}, {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)},
{TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema)}, {TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema)},
@ -315,6 +327,7 @@ static const SSysTableMeta perfsMeta[] = {
{TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)}, {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
{TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)},
{TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
{TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
}; };
void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) {

57
source/common/src/tmsg.c
View File

@ -3569,10 +3569,12 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->outputSTbName) < 0) return -1; if (tEncodeCStr(&encoder, pReq->targetStbFullName) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, astLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
@ -3592,7 +3594,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->outputSTbName) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->targetStbFullName) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1;
@ -3621,6 +3623,43 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
taosMemoryFreeClear(pReq->ast); taosMemoryFreeClear(pReq->ast);
} }
int32_t tEncodeSRSmaParam(SCoder *pCoder, const SRSmaParam *pRSmaParam) {
if (tEncodeFloat(pCoder, pRSmaParam->xFilesFactor) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->delay) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->qmsg1Len) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->qmsg2Len) < 0) return -1;
if (pRSmaParam->qmsg1Len > 0) {
if (tEncodeBinary(pCoder, pRSmaParam->qmsg1, (uint64_t)pRSmaParam->qmsg1Len) < 0) // qmsg1Len contains len of '\0'
return -1;
}
if (pRSmaParam->qmsg2Len > 0) {
if (tEncodeBinary(pCoder, pRSmaParam->qmsg2, (uint64_t)pRSmaParam->qmsg2Len) < 0) // qmsg2Len contains len of '\0'
return -1;
}
return 0;
}
int32_t tDecodeSRSmaParam(SCoder *pCoder, SRSmaParam *pRSmaParam) {
if (tDecodeFloat(pCoder, &pRSmaParam->xFilesFactor) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->delay) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->qmsg1Len) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->qmsg2Len) < 0) return -1;
if (pRSmaParam->qmsg1Len > 0) {
uint64_t len;
if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg1, &len) < 0) return -1; // qmsg1Len contains len of '\0'
} else {
pRSmaParam->qmsg1 = NULL;
}
if (pRSmaParam->qmsg2Len > 0) {
uint64_t len;
if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg2, &len) < 0) return -1; // qmsg2Len contains len of '\0'
} else {
pRSmaParam->qmsg2 = NULL;
}
return 0;
}
int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) { int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1; if (tStartEncode(pCoder) < 0) return -1;
@ -3629,9 +3668,9 @@ int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) {
if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1; if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
// if (pReq->rollup) { if (pReq->rollup) {
// if (tEncodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1; if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
// } }
tEndEncode(pCoder); tEndEncode(pCoder);
return 0; return 0;
@ -3645,9 +3684,9 @@ int tDecodeSVCreateStbReq(SCoder *pCoder, SVCreateStbReq *pReq) {
if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1; if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
// if (pReq->rollup) { if (pReq->rollup) {
// if (tDecodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1; if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
// } }
tEndDecode(pCoder); tEndDecode(pCoder);
return 0; return 0;
@ -3769,4 +3808,4 @@ int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) {
tEndDecode(pCoder); tEndDecode(pCoder);
return 0; return 0;
} }

2
source/common/src/trow.c
View File

@ -220,7 +220,7 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
} }
/** /**
* @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker. * @brief Merge bitmap from 2 bits to 1 bit, and the memory buffer should be guaranteed by the invoker.
* *
* @param srcBitmap * @param srcBitmap
* @param nBits * @param nBits

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -107,7 +107,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->vgId = pCreate->vgId; pCfg->vgId = pCreate->vgId;
strcpy(pCfg->dbname, pCreate->db); strcpy(pCfg->dbname, pCreate->db);
pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024; // pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024;
pCfg->streamMode = pCreate->streamMode; pCfg->streamMode = pCreate->streamMode;
pCfg->isWeak = true; pCfg->isWeak = true;
pCfg->tsdbCfg.days = 10; pCfg->tsdbCfg.days = 10;

22
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -514,12 +514,12 @@ void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
typedef struct { typedef struct {
int64_t consumerId; // -1 for unassigned int64_t consumerId; // -1 for unassigned
SArray* vgs; // SArray<SMqVgEp*> SArray* vgs; // SArray<SMqVgEp*>
} SMqConsumerEpInSub; } SMqConsumerEp;
SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub); SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp);
void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub); void tDeleteSMqConsumerEp(SMqConsumerEp* pEp);
int32_t tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub); int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp);
void* tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub); void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp);
typedef struct { typedef struct {
char key[TSDB_SUBSCRIBE_KEY_LEN]; char key[TSDB_SUBSCRIBE_KEY_LEN];
@ -529,9 +529,8 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub SHashObj* consumerHash; // consumerId -> SMqConsumerEp
// TODO put -1 into unassignVgs SArray* unassignedVgs; // SArray<SMqVgEp*>
// SArray* unassignedVgs;
} SMqSubscribeObj; } SMqSubscribeObj;
SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
@ -542,7 +541,7 @@ void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
typedef struct { typedef struct {
int32_t epoch; int32_t epoch;
SArray* consumers; // SArray<SMqConsumerEpInSub*> SArray* consumers; // SArray<SMqConsumerEp*>
} SMqSubActionLogEntry; } SMqSubActionLogEntry;
SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
@ -583,8 +582,9 @@ typedef struct {
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN]; char sourceDb[TSDB_DB_FNAME_LEN];
char outputSTbName[TSDB_TABLE_FNAME_LEN]; char targetDb[TSDB_DB_FNAME_LEN];
char targetSTbName[TSDB_TABLE_FNAME_LEN];
int64_t createTime; int64_t createTime;
int64_t updateTime; int64_t updateTime;
int64_t uid; int64_t uid;

7
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -302,8 +302,8 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
// 2.2 iterate all vg assigned to the consumer of that topic // 2.2 iterate all vg assigned to the consumer of that topic
SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
int32_t vgNum = taosArrayGetSize(pEpInSub->vgs); int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs);
topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
if (topicEp.vgs == NULL) { if (topicEp.vgs == NULL) {
@ -313,7 +313,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
} }
for (int32_t j = 0; j < vgNum; j++) { for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
char offsetKey[TSDB_PARTITION_KEY_LEN]; char offsetKey[TSDB_PARTITION_KEY_LEN];
mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
// 2.2.1 build vg ep // 2.2.1 build vg ep
@ -812,6 +812,7 @@ static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock
colDataAppend(pColInfo, numOfRows, (const char *)status, false); colDataAppend(pColInfo, numOfRows, (const char *)status, false);
// subscribed topics // subscribed topics
// TODO: split into multiple rows
char topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0}; char topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
char *showStr = taosShowStrArray(pConsumer->assignedTopics); char *showStr = taosShowStrArray(pConsumer->assignedTopics);
tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN); tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);

6
source/dnode/mnode/impl/src/mndDb.c
View File

@ -269,8 +269,10 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
/*
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
*/
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
@ -283,7 +285,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->minRows > pCfg->maxRows) return -1; if (pCfg->minRows > pCfg->maxRows) return -1;
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1; if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pCfg->ttl < TSDB_MIN_DB_TTL) return -1; // if (pCfg->ttl < TSDB_MIN_TABLE_TTL) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
@ -311,7 +313,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK;
if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL; if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_TABLE_TTL;
if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;

100
source/dnode/mnode/impl/src/mndDef.c
View File

@ -211,42 +211,47 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
return (void *)buf; return (void *)buf;
} }
SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) { SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
if (pEpInSubNew == NULL) return NULL; if (pConsumerEpNew == NULL) return NULL;
pEpInSubNew->consumerId = pEpInSub->consumerId; pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp); pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpOld->vgs, (FCopy)tCloneSMqVgEp);
return pEpInSubNew; return pConsumerEpNew;
} }
void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) { void tDeleteSMqConsumerEp(SMqConsumerEp *pConsumerEp) {
taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp); //
taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
} }
int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) { int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
int32_t sz = taosArrayGetSize(pEpInSub->vgs); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
#if 0
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
tlen += taosEncodeFixedI32(buf, sz); tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
tlen += tEncodeSMqVgEp(buf, pVgEp); tlen += tEncodeSMqVgEp(buf, pVgEp);
} }
/*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/ #endif
return tlen; return tlen;
} }
void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) {
buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
/*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/ buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
#if 0
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pEpInSub->vgs = taosArrayInit(sz, sizeof(void *)); pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
buf = tDecodeSMqVgEp(buf, pVgEp); buf = tDecodeSMqVgEp(buf, pVgEp);
taosArrayPush(pEpInSub->vgs, &pVgEp); taosArrayPush(pConsumerEp->vgs, &pVgEp);
} }
#endif
return (void *)buf; return (void *)buf;
} }
@ -258,13 +263,11 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
taosInitRWLatch(&pSubNew->lock); taosInitRWLatch(&pSubNew->lock);
pSubNew->vgNum = 0; pSubNew->vgNum = 0;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
// TODO set free fp // TODO set hash free fp
SMqConsumerEpInSub epInSub = { /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
.consumerId = -1,
.vgs = taosArrayInit(0, sizeof(void *)), pSubNew->unassignedVgs = taosArrayInit(0, sizeof(void *));
};
int64_t unexistKey = -1;
taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
return pSubNew; return pSubNew;
} }
@ -281,25 +284,27 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->vgNum = pSub->vgNum; pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
/*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/ // TODO set hash free fp
void *pIter = NULL; /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
SMqConsumerEpInSub *pEpInSub = NULL; void *pIter = NULL;
SMqConsumerEp *pConsumerEp = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter); pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
pEpInSub = (SMqConsumerEpInSub *)pIter; pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerEpInSub newEp = { SMqConsumerEp newEp = {
.consumerId = pEpInSub->consumerId, .consumerId = pConsumerEp->consumerId,
.vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp), .vgs = taosArrayDeepCopy(pConsumerEp->vgs, (FCopy)tCloneSMqVgEp),
}; };
taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub)); taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp));
} }
pSubNew->unassignedVgs = taosArrayDeepCopy(pSub->unassignedVgs, (FCopy)tCloneSMqVgEp);
return pSubNew; return pSubNew;
} }
void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
/*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
taosHashCleanup(pSub->consumerHash); taosHashCleanup(pSub->consumerHash);
taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp);
} }
int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
@ -319,12 +324,12 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
while (1) { while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter); pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub); tlen += tEncodeSMqConsumerEp(buf, pConsumerEp);
cnt++; cnt++;
} }
ASSERT(cnt == sz); ASSERT(cnt == sz);
/*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/ tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
return tlen; return tlen;
} }
@ -342,13 +347,12 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
/*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/ SMqConsumerEp consumerEp = {0};
SMqConsumerEpInSub epInSub = {0}; buf = tDecodeSMqConsumerEp(buf, &consumerEp);
buf = tDecodeSMqConsumerEpInSub(buf, &epInSub); taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp));
taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
} }
/*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/ buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp));
return (void *)buf; return (void *)buf;
} }
@ -356,12 +360,12 @@ SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
if (pEntryNew == NULL) return NULL; if (pEntryNew == NULL) return NULL;
pEntryNew->epoch = pEntry->epoch; pEntryNew->epoch = pEntry->epoch;
pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub); pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEp);
return pEntryNew; return pEntryNew;
} }
void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub); taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
} }
int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
@ -381,12 +385,12 @@ SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
if (pLogNew == NULL) return pLogNew; if (pLogNew == NULL) return pLogNew;
memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub); pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEp);
return pLogNew; return pLogNew;
} }
void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub); taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
} }
int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
@ -406,7 +410,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
int32_t sz = 0; int32_t sz = 0;
/*int32_t outputNameSz = 0;*/ /*int32_t outputNameSz = 0;*/
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
@ -452,7 +456,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;

265
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -14,272 +14,9 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndInfoSchema.h" #include "systable.h"
#include "common/systable.h"
#include "mndInt.h" #include "mndInt.h"
struct SSysTableIter {
SMnode *pMnode;
int32_t type;
void *pData;
SHashObj *pHashMap;
};
//#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
//#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
//#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
//
//static const SSysTableSchema dnodesSchema[] = {
// {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
// {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
// {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
// {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema mnodesSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema modulesSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema qnodesSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema snodesSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema bnodesSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema clusterSchema[] = {
// {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema userDBSchema[] = {
// {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
// {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
// {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update
//};
//
//static const SSysTableSchema userFuncSchema[] = {
// {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
//};
//
//static const SSysTableSchema userIdxSchema[] = {
// {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema userStbsSchema[] = {
// {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema userStreamsSchema[] = {
// {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema userTblsSchema[] = {
// {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "table_comment", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema userTblDistSchema[] = {
// {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
// {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
//};
//
//static const SSysTableSchema userUsersSchema[] = {
// {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema grantsSchema[] = {
// {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema vgroupsSchema[] = {
// {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
//};
//
//static const SSysTableSchema smaSchema[] = {
// {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema transSchema[] = {
// {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema configSchema[] = {
// {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableSchema connSchema[] = {
// {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
//};
//
//static const SSysTableSchema querySchema[] = {
// {.name = "queryId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
// {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL},
// {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
// {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
//};
//
//static const SSysTableMeta infosMeta[] = {
// {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
// {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
// {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
// {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)},
// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)},
// {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)},
// {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
// {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
// {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
// {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
// {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
// {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
// {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
// {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
// {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
// {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
// {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},
// {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
// {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)},
// {TSDB_INS_TABLE_CONNS, connSchema, tListLen(connSchema)},
// {TSDB_INS_TABLE_QUERIES, querySchema, tListLen(querySchema)},
//};
static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {

21
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -382,7 +382,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
pTask->dispatchType = TASK_DISPATCH__SHUFFLE; pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC; pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SDbObj* pDb = mndAcquireDb(pMnode, pStream->db); SDbObj* pDb = mndAcquireDb(pMnode, pStream->sourceDb);
ASSERT(pDb); ASSERT(pDb);
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
@ -504,11 +504,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
plan = nodesListGetNode(inner->pNodeList, 0); plan = nodesListGetNode(inner->pNodeList, 0);
} }
int64_t unexistKey = -1; ASSERT(pSub->unassignedVgs);
SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
ASSERT(pEpInSub);
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
@ -524,7 +521,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pEpInSub->vgs, &pVgEp); taosArrayPush(pSub->unassignedVgs, &pVgEp);
mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId); mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId);
@ -543,17 +540,11 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
} else { } else {
pVgEp->qmsg = strdup(""); pVgEp->qmsg = strdup("");
} }
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
/*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
} }
pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(pEpInSub->vgs->size > 0); ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);

22
source/dnode/mnode/impl/src/mndSma.c
View File

@ -40,7 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq);
static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq);
static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp);
static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp);
static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter);
int32_t mndInitSma(SMnode *pMnode) { int32_t mndInitSma(SMnode *pMnode) {
@ -406,7 +406,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
SStreamObj streamObj = {0}; SStreamObj streamObj = {0};
tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
streamObj.createTime = taosGetTimestampMs(); streamObj.createTime = taosGetTimestampMs();
streamObj.updateTime = streamObj.createTime; streamObj.updateTime = streamObj.createTime;
streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@ -686,9 +686,9 @@ _OVER:
return code; return code;
} }
int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) {
int32_t code = -1; int32_t code = -1;
SSmaObj *pSma = NULL; SSmaObj *pSma = NULL;
pSma = mndAcquireSma(pMnode, indexReq->indexFName); pSma = mndAcquireSma(pMnode, indexReq->indexFName);
if (pSma == NULL) { if (pSma == NULL) {
@ -701,13 +701,14 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserI
strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA); strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA);
SNodeList *pList = NULL; SNodeList *pList = NULL;
int32_t extOffset = 0; int32_t extOffset = 0;
code = nodesStringToList(pSma->expr, &pList); code = nodesStringToList(pSma->expr, &pList);
if (0 == code) { if (0 == code) {
SNode *node = NULL; SNode *node = NULL;
FOREACH(node, pList) { FOREACH(node, pList) {
SFunctionNode *pFunc = (SFunctionNode *)node; SFunctionNode *pFunc = (SFunctionNode *)node;
extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? ",":""), pFunc->functionName); extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s",
(extOffset ? "," : ""), pFunc->functionName);
} }
*exist = true; *exist = true;
@ -718,13 +719,12 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserI
return code; return code;
} }
static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) { static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) {
mndTransProcessRsp(pRsp); mndTransProcessRsp(pRsp);
return 0; return 0;
} }
static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
@ -758,8 +758,8 @@ static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo
char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName)); STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char*) n, false); colDataAppend(pColInfo, numOfRows, (const char *)n, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);

81
source/dnode/mnode/impl/src/mndStream.c
View File

@ -40,7 +40,7 @@ static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp);
/*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/
static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq);
static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter);
int32_t mndInitStream(SMnode *pMnode) { int32_t mndInitStream(SMnode *pMnode) {
@ -58,8 +58,8 @@ int32_t mndInitStream(SMnode *pMnode) {
/*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
/*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
/*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/ mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
@ -294,8 +294,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
mDebug("stream:%s to create", pCreate->name); mDebug("stream:%s to create", pCreate->name);
SStreamObj streamObj = {0}; SStreamObj streamObj = {0};
tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
tstrncpy(streamObj.outputSTbName, pCreate->outputSTbName, TSDB_TABLE_FNAME_LEN); tstrncpy(streamObj.targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN);
streamObj.createTime = taosGetTimestampMs(); streamObj.createTime = taosGetTimestampMs();
streamObj.updateTime = streamObj.createTime; streamObj.updateTime = streamObj.createTime;
streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@ -424,58 +424,55 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS
return 0; return 0;
} }
static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t cols = 0;
char *pWrite;
char prefix[TSDB_DB_FNAME_LEN] = {0};
SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return 0;
tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = (int32_t)strlen(prefix);
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pStream);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
if (pStream->dbUid != pDb->uid) { SColumnInfoData *pColInfo;
if (strncmp(pStream->name, prefix, prefixLen) != 0) { SName n;
mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid); int32_t cols = 0;
}
sdbRelease(pSdb, pStream); char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
continue; tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB);
} tNameGetDbName(&n, varDataVal(streamName));
varDataSetLen(streamName, strlen(varDataVal(streamName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
cols = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
char streamName[TSDB_TABLE_NAME_LEN] = {0}; char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN); tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
STR_TO_VARSTR(pWrite, streamName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
cols++; colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
*(int64_t *)pWrite = pStream->createTime; colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true);
cols++;
numOfRows++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
sdbRelease(pSdb, pStream); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->waterMark, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false);
} }
return 0;
mndReleaseDb(pMnode, pDb);
pShow->numOfRows += numOfRows;
return numOfRows;
} }
static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {

106
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -85,7 +85,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->withSchema = pTopic->withSchema; pSub->withSchema = pTopic->withSchema;
pSub->withTag = pTopic->withTag; pSub->withTag = pTopic->withTag;
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(pSub->unassignedVgs->size == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
tDeleteSubscribeObj(pSub); tDeleteSubscribeObj(pSub);
@ -93,7 +94,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
return NULL; return NULL;
} }
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
return pSub; return pSub;
} }
@ -185,7 +187,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pInput->pTopic != NULL) { if (pInput->pTopic != NULL) {
// create subscribe // create subscribe
pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1); ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 0);
} else { } else {
pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
} }
@ -196,21 +198,20 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
// 2. check and get actual removed consumers, put their vg into hash // 2. check and get actual removed consumers, put their vg into hash
int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t actualRemoved = 0; int32_t actualRemoved = 0;
for (int32_t i = 0; i < removedNum; i++) { for (int32_t i = 0; i < removedNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
ASSERT(consumerId > 0); ASSERT(consumerId > 0);
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
ASSERT(pEpInSub); ASSERT(pConsumerEp);
if (pEpInSub) { if (pConsumerEp) {
ASSERT(consumerId == pEpInSub->consumerId); ASSERT(consumerId == pConsumerEp->consumerId);
actualRemoved++; actualRemoved++;
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
for (int32_t j = 0; j < consumerVgNum; j++) { for (int32_t j = 0; j < consumerVgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = consumerId, .oldConsumerId = consumerId,
.newConsumerId = -1, .newConsumerId = -1,
@ -224,16 +225,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
} }
} }
ASSERT(removedNum == actualRemoved); ASSERT(removedNum == actualRemoved);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
// if previously no consumer, there are vgs not assigned // if previously no consumer, there are vgs not assigned
{ {
int64_t unexistKey = -1; int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs);
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
ASSERT(pEpInSub);
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
for (int32_t i = 0; i < consumerVgNum; i++) { for (int32_t i = 0; i < consumerVgNum; i++) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
SMqRebOutputVg rebOutput = { SMqRebOutputVg rebOutput = {
.oldConsumerId = -1, .oldConsumerId = -1,
.newConsumerId = -1, .newConsumerId = -1,
@ -246,7 +243,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// 3. calc vg number of each consumer // 3. calc vg number of each consumer
int32_t oldSz = 0; int32_t oldSz = 0;
if (pInput->pOldSub) { if (pInput->pOldSub) {
oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1; oldSz = taosHashGetSize(pInput->pOldSub->consumerHash);
} }
int32_t afterRebConsumerNum = int32_t afterRebConsumerNum =
oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
@ -264,23 +261,22 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0); int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
// all old consumers still existing are touched // all old consumers still existing are touched
// TODO optimize: touch only consumer whose vgs changed // TODO optimize: touch only consumer whose vgs changed
taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId); taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) { if (consumerVgNum > minVgCnt) {
if (imbCnt < imbConsumerNum) { if (imbCnt < imbConsumerNum) {
if (consumerVgNum == minVgCnt + 1) { if (consumerVgNum == minVgCnt + 1) {
continue; continue;
} else { } else {
// pop until equal minVg + 1 // pop until equal minVg + 1
while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = pEpInSub->consumerId, .oldConsumerId = pConsumerEp->consumerId,
.newConsumerId = -1, .newConsumerId = -1,
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
@ -290,10 +286,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
} }
} else { } else {
// pop until equal minVg // pop until equal minVg
while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = pEpInSub->consumerId, .oldConsumerId = pConsumerEp->consumerId,
.newConsumerId = -1, .newConsumerId = -1,
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
@ -309,14 +305,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
for (int32_t i = 0; i < consumerNum; i++) { for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
ASSERT(consumerId > 0); ASSERT(consumerId > 0);
SMqConsumerEpInSub newConsumerEp; SMqConsumerEp newConsumerEp;
newConsumerEp.consumerId = consumerId; newConsumerEp.consumerId = consumerId;
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
sizeof(SMqConsumerEpInSub));
/*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
/*ASSERT(pTestNew->consumerId == consumerId);*/
/*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
taosArrayPush(pOutput->newConsumers, &consumerId); taosArrayPush(pOutput->newConsumers, &consumerId);
} }
} }
@ -329,25 +321,24 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0);
// push until equal minVg // push until equal minVg
while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) {
// iter hash and find one vg // iter hash and find one vg
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
ASSERT(pRemovedIter); ASSERT(pRemovedIter);
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
// push // push
taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pEpInSub->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
} }
} }
// 7. handle unassigned vg // 7. handle unassigned vg
if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) { if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) {
// if has consumer, assign all left vg // if has consumer, assign all left vg
while (1) { while (1) {
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
@ -355,20 +346,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
ASSERT(pIter); ASSERT(pIter);
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId;
pRebVg->newConsumerId = pEpInSub->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
} }
} else { } else {
// if all consumer is removed, put all vg into unassigned // if all consumer is removed, put all vg into unassigned
int64_t unexistKey = -1;
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
ASSERT(pEpInSub);
ASSERT(pEpInSub->consumerId == -1);
pIter = NULL; pIter = NULL;
SMqRebOutputVg *pRebOutput = NULL; SMqRebOutputVg *pRebOutput = NULL;
while (1) { while (1) {
@ -376,12 +361,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pIter == NULL) break; if (pIter == NULL) break;
pRebOutput = (SMqRebOutputVg *)pIter; pRebOutput = (SMqRebOutputVg *)pIter;
ASSERT(pRebOutput->newConsumerId == -1); ASSERT(pRebOutput->newConsumerId == -1);
taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput); taosArrayPush(pOutput->rebVgs, pRebOutput);
} }
} }
// 8. generate logs // 8. TODO generate logs
mInfo("rebalance calculation completed, rebalanced vg:");
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
mInfo("vg: %d moved from consumer %ld to consumer %ld", pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId,
pOutputRebVg->newConsumerId);
}
// 9. clear // 9. clear
taosHashCleanup(pHash); taosHashCleanup(pHash);
@ -459,7 +450,9 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
goto REB_FAIL; goto REB_FAIL;
} }
} }
// 4. commit log: modification log // 4. TODO commit log: modification log
// 5. execution
if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -512,6 +505,7 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
// possibly no vg is changed // possibly no vg is changed
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
// TODO replace assert with error check
ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
if (rebInput.pTopic) { if (rebInput.pTopic) {
@ -631,6 +625,10 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc
pOldSub->consumerHash = pNewSub->consumerHash; pOldSub->consumerHash = pNewSub->consumerHash;
pNewSub->consumerHash = tmp; pNewSub->consumerHash = tmp;
SArray *tmp1 = pOldSub->unassignedVgs;
pOldSub->unassignedVgs = pNewSub->unassignedVgs;
pNewSub->unassignedVgs = tmp1;
taosWUnLockLatch(&pOldSub->lock); taosWUnLockLatch(&pOldSub->lock);
return 0; return 0;
} }

18
source/dnode/mnode/impl/src/mndUser.c
View File

@ -39,14 +39,16 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
int32_t mndInitUser(SMnode *pMnode) { int32_t mndInitUser(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_USER, SSdbTable table = {
.keyType = SDB_KEY_BINARY, .sdbType = SDB_USER,
.deployFp = (SdbDeployFp)mndCreateDefaultUsers, .keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndUserActionEncode, .deployFp = (SdbDeployFp)mndCreateDefaultUsers,
.decodeFp = (SdbDecodeFp)mndUserActionDecode, .encodeFp = (SdbEncodeFp)mndUserActionEncode,
.insertFp = (SdbInsertFp)mndUserActionInsert, .decodeFp = (SdbDecodeFp)mndUserActionDecode,
.updateFp = (SdbUpdateFp)mndUserActionUpdate, .insertFp = (SdbInsertFp)mndUserActionInsert,
.deleteFp = (SdbDeleteFp)mndUserActionDelete}; .updateFp = (SdbUpdateFp)mndUserActionUpdate,
.deleteFp = (SdbDeleteFp)mndUserActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_USER, mndProcessCreateUserReq); mndSetMsgHandle(pMnode, TDMT_MND_CREATE_USER, mndProcessCreateUserReq);
mndSetMsgHandle(pMnode, TDMT_MND_ALTER_USER, mndProcessAlterUserReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_USER, mndProcessAlterUserReq);

339
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -9,75 +9,314 @@
* *
*/ */
#include "sut.h" #include <gtest/gtest.h>
class MndTestShow : public ::testing::Test { #include "sdb.h"
class MndTestSdb : public ::testing::Test {
protected: protected:
static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); } static void SetUpTestSuite() {}
static void TearDownTestSuite() { test.Cleanup(); } static void TearDownTestSuite() {}
static Testbase test;
public: public:
void SetUp() override {} void SetUp() override {}
void TearDown() override {} void TearDown() override {}
}; };
Testbase MndTestShow::test; typedef struct SMnode {
int32_t v100;
int32_t v200;
SSdb *pSdb;
} SMnode;
TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { typedef struct SStrObj {
SShowReq showReq = {0}; char key[24];
showReq.type = TSDB_MGMT_TABLE_MAX; int8_t v8;
int16_t v16;
int32_t v32;
int64_t v64;
char vstr[32];
char unused[48];
} SStrObj;
int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); typedef struct SI32Obj {
void* pReq = rpcMallocCont(contLen); int32_t key;
tSerializeSShowReq(pReq, contLen, &showReq); int8_t v8;
tFreeSShowReq(&showReq); int16_t v16;
int32_t v32;
int64_t v64;
char vstr[32];
char unused[48];
} SI32Obj;
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); typedef struct SI64Obj {
ASSERT_NE(pRsp, nullptr); int64_t key;
ASSERT_NE(pRsp->code, 0); int8_t v8;
int16_t v16;
int32_t v32;
int64_t v64;
char vstr[32];
char unused[48];
} SI64Obj;
SSdbRaw *strEncode(SStrObj *pObj) {
int32_t dataPos = 0;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, 1, sizeof(SStrObj));
sdbSetRawBinary(pRaw, dataPos, pObj->key, sizeof(pObj->key));
dataPos += sizeof(pObj->key);
sdbSetRawInt8(pRaw, dataPos, pObj->v8);
dataPos += sizeof(pObj->v8);
sdbSetRawInt16(pRaw, dataPos, pObj->v16);
dataPos += sizeof(pObj->v16);
sdbSetRawInt32(pRaw, dataPos, pObj->v32);
dataPos += sizeof(pObj->v32);
sdbSetRawInt64(pRaw, dataPos, pObj->v64);
dataPos += sizeof(pObj->v64);
sdbSetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr));
dataPos += sizeof(pObj->vstr);
sdbSetRawDataLen(pRaw, dataPos);
return pRaw;
} }
TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { SSdbRow *strDecode(SSdbRaw *pRaw) {
SShowReq showReq = {0}; int8_t sver = 0;
showReq.type = TSDB_MGMT_TABLE_START; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != 1) return NULL;
int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); SSdbRow *pRow = sdbAllocRow(sizeof(SStrObj));
void* pReq = rpcMallocCont(contLen); if (pRow == NULL) return NULL;
tSerializeSShowReq(pReq, contLen, &showReq);
tFreeSShowReq(&showReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); SStrObj *pObj = (SStrObj *)sdbGetRowObj(pRow);
ASSERT_NE(pRsp, nullptr); if (pObj == NULL) return NULL;
ASSERT_NE(pRsp->code, 0);
int32_t dataPos = 0;
sdbGetRawBinary(pRaw, dataPos, pObj->key, sizeof(pObj->key));
dataPos += sizeof(pObj->key);
sdbGetRawInt8(pRaw, dataPos, &pObj->v8);
dataPos += sizeof(pObj->v8);
sdbGetRawInt16(pRaw, dataPos, &pObj->v16);
dataPos += sizeof(pObj->v16);
sdbGetRawInt32(pRaw, dataPos, &pObj->v32);
dataPos += sizeof(pObj->v32);
sdbGetRawInt64(pRaw, dataPos, &pObj->v64);
dataPos += sizeof(pObj->v64);
sdbGetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr));
dataPos += sizeof(pObj->vstr);
return pRow;
} }
TEST_F(MndTestShow, 03_ShowMsg_Conn) { int32_t strInsert(SSdb *pSdb, SStrObj *pObj) { return 0; }
char passwd[] = "taosdata";
char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
SConnectReq connectReq = {0}; int32_t strDelete(SSdb *pSdb, SStrObj *pObj, bool callFunc) { return 0; }
connectReq.pid = 1234;
strcpy(connectReq.app, "mnode_test_show");
strcpy(connectReq.db, "");
strcpy(connectReq.user, "root");
strcpy(connectReq.passwd, secretEncrypt);
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); int32_t strUpdate(SSdb *pSdb, SStrObj *pOld, SStrObj *pNew) {
void* pReq = rpcMallocCont(contLen); pOld->v8 = pNew->v8;
tSerializeSConnectReq(pReq, contLen, &connectReq); pOld->v16 = pNew->v16;
pOld->v32 = pNew->v32;
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); pOld->v64 = pNew->v64;
ASSERT_NE(pRsp, nullptr); strcpy(pOld->vstr, pNew->vstr);
ASSERT_EQ(pRsp->code, 0); return 0;
test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", "");
// EXPECT_EQ(test.GetShowRows(), 1);
} }
TEST_F(MndTestShow, 04_ShowMsg_Cluster) { void strSetDefault(SStrObj *pObj, int32_t index) {
test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", ""); memset(pObj, 0, sizeof(SStrObj));
EXPECT_EQ(test.GetShowRows(), 1); snprintf(pObj->key, sizeof(pObj->key), "k%d", index * 1000);
pObj->v8 = index;
pObj->v16 = index;
pObj->v32 = index * 1000;
pObj->v64 = index * 1000;
snprintf(pObj->vstr, sizeof(pObj->vstr), "v%d", index * 1000);
} }
int32_t strDefault(SMnode *pMnode) {
SStrObj strObj;
SSdbRaw *pRaw = NULL;
strSetDefault(&strObj, 1);
pRaw = strEncode(&strObj);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (sdbWrite(pMnode->pSdb, pRaw) != 0) return -1;
strSetDefault(&strObj, 2);
pRaw = strEncode(&strObj);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (sdbWriteWithoutFree(pMnode->pSdb, pRaw) != 0) return -1;
sdbFreeRaw(pRaw);
return 0;
}
bool sdbTraverseSucc1(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
if (pObj->v8 == 1) {
*p1 = *p2 + *p3 + pObj->v8;
}
return true;
}
bool sdbTraverseSucc2(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
*p1 = *p2 + *p3 + pObj->v8;
return true;
}
bool sdbTraverseFail(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) {
*p1 = *p2 + *p3;
return false;
}
TEST_F(MndTestSdb, 01_Write) {
void *pIter;
int32_t num;
SStrObj *pObj;
SMnode mnode;
SSdb *pSdb;
SSdbOpt opt = {0};
int32_t p1 = 0;
int32_t p2 = 111;
int32_t p3 = 222;
mnode.v100 = 100;
mnode.v200 = 200;
opt.pMnode = &mnode;
opt.path = "/tmp/mnode_test_sdb";
taosRemoveDir(opt.path);
SSdbTable strTable = {
.sdbType = SDB_USER,
.keyType = SDB_KEY_BINARY,
.deployFp = (SdbDeployFp)strDefault,
.encodeFp = (SdbEncodeFp)strEncode,
.decodeFp = (SdbDecodeFp)strDecode,
.insertFp = (SdbInsertFp)strInsert,
.updateFp = (SdbUpdateFp)strUpdate,
.deleteFp = (SdbDeleteFp)strDelete,
};
pSdb = sdbInit(&opt);
mnode.pSdb = pSdb;
ASSERT_NE(pSdb, nullptr);
ASSERT_EQ(sdbSetTable(pSdb, strTable), 0);
ASSERT_EQ(sdbDeploy(pSdb), 0);
#if 0
pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k1000");
ASSERT_NE(pObj, nullptr);
EXPECT_STREQ(pObj->key, "k1000");
EXPECT_STREQ(pObj->vstr, "v1000");
EXPECT_EQ(pObj->v8, 1);
EXPECT_EQ(pObj->v16, 1);
EXPECT_EQ(pObj->v32, 1000);
EXPECT_EQ(pObj->v64, 1000);
sdbRelease(pSdb, pObj);
pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k2000");
ASSERT_NE(pObj, nullptr);
EXPECT_STREQ(pObj->key, "k2000");
EXPECT_STREQ(pObj->vstr, "v2000");
EXPECT_EQ(pObj->v8, 2);
EXPECT_EQ(pObj->v16, 2);
EXPECT_EQ(pObj->v32, 2000);
EXPECT_EQ(pObj->v64, 2000);
sdbRelease(pSdb, pObj);
pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k200");
ASSERT_EQ(pObj, nullptr);
pIter = NULL;
num = 0;
do {
pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
if (pIter == NULL) break;
ASSERT_NE(pObj, nullptr);
num++;
sdbRelease(pSdb, pObj);
} while (1);
EXPECT_EQ(num, 2);
do {
pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
if (pIter == NULL) break;
if (strcmp(pObj->key, "k1000") == 0) {
sdbCancelFetch(pSdb, pIter);
break;
}
} while (1);
EXPECT_STREQ(pObj->key, "k1000");
p1 = 0;
p2 = 111;
p3 = 222;
sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3);
EXPECT_EQ(p1, 334);
p1 = 0;
p2 = 111;
p3 = 222;
sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3);
EXPECT_EQ(p1, 669);
p1 = 0;
p2 = 111;
p3 = 222;
sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseFail, &p1, &p2, &p3);
EXPECT_EQ(p1, 333);
EXPECT_EQ(sdbGetSize(pSdb, SDB_USER), 2);
EXPECT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1);
EXPECT_EQ(sdbGetTableVer(pSdb, SDB_USER), 2);
EXPECT_EQ(sdbUpdateVer(pSdb, 0), 2);
EXPECT_EQ(sdbUpdateVer(pSdb, 1), 3);
EXPECT_EQ(sdbUpdateVer(pSdb, -1), 2);
// insert, call func
// update, call func
// delete, call func 2
// write version
// sdb Write ver
// sdbRead
#endif
ASSERT_EQ(sdbWriteFile(pSdb), 0);
sdbCleanup(pSdb);
}
TEST_F(MndTestSdb, 01_Read) {
void *pIter;
int32_t num;
SStrObj *pObj;
SMnode mnode;
SSdb *pSdb;
SSdbOpt opt = {0};
int32_t p1 = 0;
int32_t p2 = 111;
int32_t p3 = 222;
mnode.v100 = 100;
mnode.v200 = 200;
opt.pMnode = &mnode;
opt.path = "/tmp/mnode_test_sdb";
taosRemoveDir(opt.path);
SSdbTable strTable = {
.sdbType = SDB_USER,
.keyType = SDB_KEY_BINARY,
.deployFp = (SdbDeployFp)strDefault,
.encodeFp = (SdbEncodeFp)strEncode,
.decodeFp = (SdbDecodeFp)strDecode,
.insertFp = (SdbInsertFp)strInsert,
.updateFp = (SdbUpdateFp)strDelete,
.deleteFp = (SdbDeleteFp)strUpdate,
};
pSdb = sdbInit(&opt);
mnode.pSdb = pSdb;
ASSERT_EQ(sdbReadFile(pSdb), 0);
sdbCleanup(pSdb);
}

2
source/dnode/mnode/sdb/CMakeLists.txt
View File

@ -6,5 +6,5 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries( target_link_libraries(
sdb os common util sdb os common util wal
) )

2
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -52,6 +52,8 @@ typedef struct SSdbRow {
const char *sdbTableName(ESdbType type); const char *sdbTableName(ESdbType type);
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/dnode/mnode/sdb/src/sdb.c
View File

@ -141,7 +141,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
} }
static int32_t sdbCreateDir(SSdb *pSdb) { static int32_t sdbCreateDir(SSdb *pSdb) {
if (taosMkDir(pSdb->currDir) != 0) { if (taosMulMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr()); mError("failed to create dir:%s since %s", pSdb->currDir, terrstr());
return -1; return -1;

3
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "sdbInt.h" #include "sdbInt.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "wal.h"
#define SDB_TABLE_SIZE 24 #define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512 #define SDB_RESERVE_SIZE 512
@ -137,7 +138,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
int32_t readLen = 0; int32_t readLen = 0;
int64_t ret = 0; int64_t ret = 0;
SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE); SSdbRaw *pRaw = taosMemoryMalloc(WAL_MAX_SIZE + 100);
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read file since %s", terrstr()); mError("failed read file since %s", terrstr());

8
source/dnode/qnode/src/qnode.c
View File

@ -17,7 +17,7 @@
#include "qndInt.h" #include "qndInt.h"
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"
//#include "tudf.h" #include "libs/function/function.h"
SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@ -26,7 +26,9 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
return NULL; return NULL;
} }
//udfcOpen(); if (udfcOpen() != 0) {
qError("qnode can not open udfc");
}
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
taosMemoryFreeClear(pQnode); taosMemoryFreeClear(pQnode);
@ -40,7 +42,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
void qndClose(SQnode *pQnode) { void qndClose(SQnode *pQnode) {
qWorkerDestroy((void **)&pQnode->pQuery); qWorkerDestroy((void **)&pQnode->pQuery);
//udfcClose(); udfcClose();
taosMemoryFree(pQnode); taosMemoryFree(pQnode);
} }

20
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -22,13 +22,13 @@
extern "C" { extern "C" {
#endif #endif
typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2); // typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
struct STbDdlH { // struct STbDdlH {
void *ahandle; // void *ahandle;
void *result; // void *result;
__tb_ddl_fn_t fp; // __tb_ddl_fn_t fp;
}; // };
static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
ASSERT(*pStore == NULL); ASSERT(*pStore == NULL);
@ -40,14 +40,6 @@ static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
void tsdbUidStoreDestory(STbUidStore *pStore);
void *tsdbUidStoreFree(STbUidStore *pStore);
int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq);
int32_t tsdbFetchTbUidList(void *pTsdb, void **result, void *suid, void *uid);
int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pUidStore);
int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType);
#ifdef __cplusplus #ifdef __cplusplus
} }

11
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -107,6 +107,15 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
// sma
int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq);
int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
void tsdbUidStoreDestory(STbUidStore* pStore);
void* tsdbUidStoreFree(STbUidStore* pStore);
int32_t tsdbTriggerRSma(STsdb* pTsdb, SMeta* pMeta, void* pMsg, int32_t inputType);
typedef struct { typedef struct {
int8_t streamType; // sma or other int8_t streamType; // sma or other
int8_t dstType; int8_t dstType;
@ -162,7 +171,7 @@ struct STbUidStore {
#define TD_VID(PVNODE) (PVNODE)->config.vgId #define TD_VID(PVNODE) (PVNODE)->config.vgId
typedef struct STbDdlH STbDdlH; // typedef struct STbDdlH STbDdlH;
// sma // sma
void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);

10
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1490,6 +1490,10 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
} }
#ifdef TD_DEBUG_PRINT_ROW
tdSRowPrint(row1, pSchema1, __func__);
#endif
if (isRow1DataRow) { if (isRow1DataRow) {
numOfColsOfRow1 = schemaNCols(pSchema1); numOfColsOfRow1 = schemaNCols(pSchema1);
} else { } else {
@ -1519,8 +1523,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
} else if (isRow1DataRow) { } else if (isRow1DataRow) {
colIdOfRow1 = pSchema1->columns[j].colId; colIdOfRow1 = pSchema1->columns[j].colId;
} else { } else {
SKvRowIdx* pColIdx = tdKvRowColIdxAt(row1, j); colIdOfRow1 = tdKvRowColIdAt(row1, j);
colIdOfRow1 = pColIdx->colId;
} }
int32_t colIdOfRow2; int32_t colIdOfRow2;
@ -1529,8 +1532,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
} else if (isRow2DataRow) { } else if (isRow2DataRow) {
colIdOfRow2 = pSchema2->columns[k].colId; colIdOfRow2 = pSchema2->columns[k].colId;
} else { } else {
SKvRowIdx* pColIdx = tdKvRowColIdxAt(row2, k); colIdOfRow2 = tdKvRowColIdAt(row2, k);
colIdOfRow2 = pColIdx->colId;
} }
if (colIdOfRow1 == colIdOfRow2) { if (colIdOfRow1 == colIdOfRow2) {

47
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -173,6 +173,7 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[])
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg); static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg);
static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg);
static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids);
// mgmt interface // mgmt interface
static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid);
@ -1692,18 +1693,16 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
* @param pReq * @param pReq
* @return int32_t * @return int32_t
*/ */
int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) { int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq) {
#if 0 if (!pReq->rollup) {
SRSmaParam *param = pReq->stbCfg.pRSmaParam; tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid);
if (!param) {
tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name,
pReq->stbCfg.suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SRSmaParam *param = &pReq->pRSmaParam;
if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) {
tsdbWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->stbCfg.suid); tsdbWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1716,9 +1715,9 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
SSmaStat *pStat = SMA_ENV_STAT(pEnv); SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL; SRSmaInfo *pRSmaInfo = NULL;
pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->stbCfg.suid, sizeof(tb_uid_t)); pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
if (pRSmaInfo) { if (pRSmaInfo) {
tsdbWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->stbCfg.suid); tsdbWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1758,14 +1757,13 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
} }
} }
if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->stbCfg.suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) !=
TSDB_CODE_SUCCESS) { TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} else { } else {
tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->stbCfg.suid); tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->suid);
} }
#endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1777,7 +1775,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
* @param uid * @param uid
* @return int32_t * @return int32_t
*/ */
int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { static int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) {
// prefer to store suid/uids in array // prefer to store suid/uids in array
if ((suid == pStore->suid) || (pStore->suid == 0)) { if ((suid == pStore->suid) || (pStore->suid == 0)) {
if (pStore->suid == 0) { if (pStore->suid == 0) {
@ -1833,6 +1831,7 @@ void tsdbUidStoreDestory(STbUidStore *pStore) {
if (pStore) { if (pStore) {
if (pStore->uidHash) { if (pStore->uidHash) {
if (pStore->tbUids) { if (pStore->tbUids) {
// When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
void *pIter = taosHashIterate(pStore->uidHash, NULL); void *pIter = taosHashIterate(pStore->uidHash, NULL);
while (pIter) { while (pIter) {
SArray *arr = *(SArray **)pIter; SArray *arr = *(SArray **)pIter;
@ -1847,8 +1846,10 @@ void tsdbUidStoreDestory(STbUidStore *pStore) {
} }
void *tsdbUidStoreFree(STbUidStore *pStore) { void *tsdbUidStoreFree(STbUidStore *pStore) {
tsdbUidStoreDestory(pStore); if (pStore) {
taosMemoryFree(pStore); tsdbUidStoreDestory(pStore);
taosMemoryFree(pStore);
}
return NULL; return NULL;
} }
@ -1861,7 +1862,7 @@ void *tsdbUidStoreFree(STbUidStore *pStore) {
* @param uid * @param uid
* @return int32_t * @return int32_t
*/ */
int32_t tsdbFetchTbUidList(void *pTsdb, void **ppStore, void *suid, void *uid) { int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) {
SSmaEnv *pEnv = REPO_RSMA_ENV((STsdb *)pTsdb); SSmaEnv *pEnv = REPO_RSMA_ENV((STsdb *)pTsdb);
// only applicable to rollup SMA ctables // only applicable to rollup SMA ctables
@ -1877,7 +1878,7 @@ int32_t tsdbFetchTbUidList(void *pTsdb, void **ppStore, void *suid, void *uid) {
} }
// info cached when create rsma stable and return directly for non-rsma ctables // info cached when create rsma stable and return directly for non-rsma ctables
if (!taosHashGet(infoHash, suid, sizeof(tb_uid_t))) { if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1887,7 +1888,7 @@ int32_t tsdbFetchTbUidList(void *pTsdb, void **ppStore, void *suid, void *uid) {
} }
} }
if (tsdbUidStorePut(*ppStore, *(tb_uid_t *)suid, (tb_uid_t *)uid) != 0) { if (tsdbUidStorePut(*ppStore, suid, &uid) != 0) {
*ppStore = tsdbUidStoreFree(*ppStore); *ppStore = tsdbUidStoreFree(*ppStore);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -1935,12 +1936,10 @@ static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid
int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) {
if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
tsdbDebug("vgId:%d no need to update tbUids since empty uidStore", REPO_ID(pTsdb)); tsdbDebug("vgId:%d no need to update tbUids since empty uidStore", REPO_ID(pTsdb));
tsdbUidStoreFree(pStore);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (tsdbUpdateTbUidListImpl(pTsdb, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { if (tsdbUpdateTbUidListImpl(pTsdb, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) {
tsdbUidStoreFree(pStore);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -1951,15 +1950,11 @@ int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) {
if (tsdbUpdateTbUidListImpl(pTsdb, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { if (tsdbUpdateTbUidListImpl(pTsdb, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) {
taosHashCancelIterate(pStore->uidHash, pIter); taosHashCancelIterate(pStore->uidHash, pIter);
tsdbUidStoreFree(pStore);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
pIter = taosHashIterate(pStore->uidHash, pIter); pIter = taosHashIterate(pStore->uidHash, pIter);
} }
tsdbUidStoreFree(pStore);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1971,8 +1966,6 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
STSRow *row = NULL; STSRow *row = NULL;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
// pMsg->length = htonl(pMsg->length);
// pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tInitSubmitMsgIterEx(pMsg, &msgIter) < 0) return -1; if (tInitSubmitMsgIterEx(pMsg, &msgIter) < 0) return -1;
while (true) { while (true) {

14
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -61,7 +61,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
len = pMsg->contLen - sizeof(SMsgHead); len = pMsg->contLen - sizeof(SMsgHead);
// todo: change the interface here
if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
return -1; return -1;
@ -300,13 +299,13 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq,
goto _err; goto _err;
} }
// tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) {
pRsp->code = terrno; pRsp->code = terrno;
goto _err; goto _err;
} }
tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req);
tCoderClear(&coder); tCoderClear(&coder);
return 0; return 0;
@ -323,6 +322,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
SVCreateTbBatchRsp rsp = {0}; SVCreateTbBatchRsp rsp = {0};
SVCreateTbRsp cRsp = {0}; SVCreateTbRsp cRsp = {0};
char tbName[TSDB_TABLE_FNAME_LEN]; char tbName[TSDB_TABLE_FNAME_LEN];
STbUidStore *pStore = NULL;
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
pRsp->code = TSDB_CODE_SUCCESS; pRsp->code = TSDB_CODE_SUCCESS;
@ -361,6 +361,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
cRsp.code = terrno; cRsp.code = terrno;
} else { } else {
cRsp.code = TSDB_CODE_SUCCESS; cRsp.code = TSDB_CODE_SUCCESS;
tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
} }
taosArrayPush(rsp.pArray, &cRsp); taosArrayPush(rsp.pArray, &cRsp);
@ -368,6 +369,9 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
tCoderClear(&coder); tCoderClear(&coder);
tsdbUpdateTbUidList(pVnode->pTsdb, pStore);
tsdbUidStoreFree(pStore);
// prepare rsp // prepare rsp
int32_t ret = 0; int32_t ret = 0;
tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen, ret); tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen, ret);
@ -426,7 +430,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
SSubmitRsp rsp = {0}; SSubmitRsp rsp = {0};
pRsp->code = 0; pRsp->code = 0;
tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
// handle the request // handle the request
if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) {
pRsp->code = terrno; pRsp->code = terrno;
@ -435,7 +439,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
// pRsp->msgType = TDMT_VND_SUBMIT_RSP; // pRsp->msgType = TDMT_VND_SUBMIT_RSP;
// vnodeProcessSubmitReq(pVnode, ptr, pRsp); // vnodeProcessSubmitReq(pVnode, ptr, pRsp);
// tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK); // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
// encode the response (TODO) // encode the response (TODO)
pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp));

9
source/libs/executor/src/executorimpl.c
View File

@ -1897,7 +1897,14 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) { if (fmIsAggFunc(pCtx->functionId) || fmIsNonstandardSQLFunc(pCtx->functionId)) {
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId);
if (!isUdaf) {
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
} else {
char *udfName = pExpr->pExpr->_function.pFunctNode->functionName;
strncpy(pCtx->udfName, udfName, strlen(udfName));
fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
}
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
} else { } else {
fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp); fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp);

15
source/libs/function/CMakeLists.txt
View File

@ -51,6 +51,21 @@ target_link_libraries(
udf1 PUBLIC os udf1 PUBLIC os
) )
add_library(udf2 MODULE test/udf2.c)
target_include_directories(
udf2
PUBLIC
"${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
udf2 PUBLIC os
)
#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
add_executable(udfd src/udfd.c) add_executable(udfd src/udfd.c)
target_include_directories( target_include_directories(

1
source/libs/function/inc/builtinsimpl.h
View File

@ -25,6 +25,7 @@ extern "C" {
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);

3
source/libs/function/inc/tudfInt.h
View File

@ -43,6 +43,9 @@ typedef struct SUdfSetupRequest {
typedef struct SUdfSetupResponse { typedef struct SUdfSetupResponse {
int64_t udfHandle; int64_t udfHandle;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
} SUdfSetupResponse; } SUdfSetupResponse;
typedef struct SUdfCallRequest { typedef struct SUdfCallRequest {

14
source/libs/function/src/builtinsimpl.c
View File

@ -139,6 +139,20 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return pResInfo->numOfRes; return pResInfo->numOfRes;
} }
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0;
cleanupResultRowEntry(pResInfo);
char* in = finalResult;
colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
return pResInfo->numOfRes;
}
EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) { if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) {

15
source/libs/function/src/functionMgt.c
View File

@ -21,6 +21,10 @@
#include "taos.h" #include "taos.h"
#include "taoserror.h" #include "taoserror.h"
#include "thash.h" #include "thash.h"
#include "builtins.h"
#include "catalog.h"
#include "tudf.h"
typedef struct SFuncMgtService { typedef struct SFuncMgtService {
SHashObj* pFuncNameHashTable; SHashObj* pFuncNameHashTable;
@ -120,6 +124,17 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
if (!fmIsUserDefinedFunc(funcId)) {
return TSDB_CODE_FAILED;
}
pFpSet->getEnv = udfAggGetEnv;
pFpSet->init = udfAggInit;
pFpSet->process = udfAggProcess;
pFpSet->finalize = udfAggFinalize;
return TSDB_CODE_SUCCESS;
}
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;

137
source/libs/function/src/tudf.c
View File

@ -19,6 +19,9 @@
#include "tudfInt.h" #include "tudfInt.h"
#include "tarray.h" #include "tarray.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "querynodes.h"
#include "builtinsimpl.h"
#include "functionMgt.h"
//TODO: network error processing. //TODO: network error processing.
//TODO: add unit test //TODO: add unit test
@ -147,6 +150,10 @@ typedef struct SUdfUvSession {
SUdfdProxy *udfc; SUdfdProxy *udfc;
int64_t severHandle; int64_t severHandle;
uv_pipe_t *udfSvcPipe; uv_pipe_t *udfSvcPipe;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
} SUdfUvSession; } SUdfUvSession;
typedef struct SClientUvTaskNode { typedef struct SClientUvTaskNode {
@ -235,12 +242,14 @@ void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) {
int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state) { int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state) {
int32_t len = 0; int32_t len = 0;
len += taosEncodeFixedI8(buf, state->numOfResult);
len += taosEncodeFixedI32(buf, state->bufLen); len += taosEncodeFixedI32(buf, state->bufLen);
len += taosEncodeBinary(buf, state->buf, state->bufLen); len += taosEncodeBinary(buf, state->buf, state->bufLen);
return len; return len;
} }
void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state) { void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state) {
buf = taosDecodeFixedI8(buf, &state->numOfResult);
buf = taosDecodeFixedI32(buf, &state->bufLen); buf = taosDecodeFixedI32(buf, &state->bufLen);
buf = taosDecodeBinary(buf, (void**)&state->buf, state->bufLen); buf = taosDecodeBinary(buf, (void**)&state->buf, state->bufLen);
return (void*)buf; return (void*)buf;
@ -342,11 +351,17 @@ void* decodeUdfRequest(const void* buf, SUdfRequest* request) {
int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) { int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) {
int32_t len = 0; int32_t len = 0;
len += taosEncodeFixedI64(buf, setupRsp->udfHandle); len += taosEncodeFixedI64(buf, setupRsp->udfHandle);
len += taosEncodeFixedI8(buf, setupRsp->outputType);
len += taosEncodeFixedI32(buf, setupRsp->outputLen);
len += taosEncodeFixedI32(buf, setupRsp->bufSize);
return len; return len;
} }
void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp) { void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp) {
buf = taosDecodeFixedI64(buf, &setupRsp->udfHandle); buf = taosDecodeFixedI64(buf, &setupRsp->udfHandle);
buf = taosDecodeFixedI8(buf, &setupRsp->outputType);
buf = taosDecodeFixedI32(buf, &setupRsp->outputLen);
buf = taosDecodeFixedI32(buf, &setupRsp->bufSize);
return (void*)buf; return (void*)buf;
} }
@ -1049,6 +1064,9 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
SUdfSetupResponse *rsp = &task->_setup.rsp; SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle; task->session->severHandle = rsp->udfHandle;
task->session->outputType = rsp->outputType;
task->session->outputLen = rsp->outputLen;
task->session->bufSize = rsp->bufSize;
if (task->errCode != 0) { if (task->errCode != 0) {
fnError("failed to setup udf. err: %d", task->errCode) fnError("failed to setup udf. err: %d", task->errCode)
} else { } else {
@ -1197,3 +1215,122 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
return err; return err;
} }
//memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
typedef struct SUdfAggRes {
SUdfUvSession *session;
int8_t finalResNum;
int8_t interResNum;
char* finalResBuf;
char* interResBuf;
} SUdfAggRes;
bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
if (fmIsScalarFunc(pFunc->funcId)) {
return false;
}
pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize;
return true;
}
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) {
if (functionSetup(pCtx, pResultCellInfo) != true) {
return false;
}
UdfcFuncHandle handle;
if (setupUdf((char*)pCtx->udfName, &handle) != 0) {
return false;
}
SUdfUvSession *session = (SUdfUvSession *)handle;
SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo);
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize;
memset(udfRes, 0, envSize);
udfRes->session = (SUdfUvSession *)handle;
SUdfInterBuf buf = {0};
if (callUdfAggInit(handle, &buf) != 0) {
return false;
}
udfRes->interResNum = buf.numOfResult;
memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
return true;
}
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t numOfCols = pInput->numOfInputCols;
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SUdfUvSession *session = udfRes->session;
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
SSDataBlock tempBlock = {0};
tempBlock.info.numOfCols = numOfCols;
tempBlock.info.rows = numOfRows;
tempBlock.info.uid = pInput->uid;
bool hasVarCol = false;
tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData *col = pInput->pData[i];
if (IS_VAR_DATA_TYPE(col->info.type)) {
hasVarCol = true;
}
taosArrayPush(tempBlock.pDataBlock, col);
}
tempBlock.info.hasVarCol = hasVarCol;
SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows);
SUdfInterBuf state = {.buf = udfRes->interResBuf,
.bufLen = session->bufSize,
.numOfResult = udfRes->interResNum};
SUdfInterBuf newState = {0};
callUdfAggProcess(session, inputBlock, &state, &newState);
udfRes->interResNum = newState.numOfResult;
memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
if (newState.numOfResult == 1 || state.numOfResult == 1) {
GET_RES_INFO(pCtx)->numOfRes = 1;
}
blockDataDestroy(inputBlock);
taosArrayDestroy(tempBlock.pDataBlock);
taosMemoryFree(newState.buf);
return 0;
}
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SUdfUvSession *session = udfRes->session;
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf,
.bufLen = session->outputLen,
.numOfResult = udfRes->finalResNum};
SUdfInterBuf state = {.buf = udfRes->interResBuf,
.bufLen = session->bufSize,
.numOfResult = udfRes->interResNum};
callUdfAggFinalize(session, &state, &resultBuf);
teardownUdf(session);
if (resultBuf.numOfResult == 1) {
GET_RES_INFO(pCtx)->numOfRes = 1;
}
return functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
}

109
source/libs/function/src/udfd.c
View File

@ -77,6 +77,10 @@ typedef struct SUdf {
uv_lib_t lib; uv_lib_t lib;
TUdfScalarProcFunc scalarProcFunc; TUdfScalarProcFunc scalarProcFunc;
TUdfFreeUdfColumnFunc freeUdfColumn; TUdfFreeUdfColumnFunc freeUdfColumn;
TUdfAggStartFunc aggStartFunc;
TUdfAggProcessFunc aggProcFunc;
TUdfAggFinishFunc aggFinishFunc;
} SUdf; } SUdf;
// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@ -97,15 +101,32 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
return UDFC_CODE_LOAD_UDF_FAILURE; return UDFC_CODE_LOAD_UDF_FAILURE;
} }
// TODO: find all the functions //TODO: init and destroy function
char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) {
strcpy(normalFuncName, udfName); char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); strcpy(processFuncName, udfName);
char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->scalarProcFunc));
char *freeSuffix = "_free"; char freeFuncName[TSDB_FUNC_NAME_LEN + 5] = {0};
strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); char *freeSuffix = "_free";
strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); strncpy(freeFuncName, processFuncName, strlen(processFuncName));
uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
} else if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
strcpy(processFuncName, udfName);
uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->aggProcFunc));
char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *startSuffix = "_start";
strncpy(startFuncName, processFuncName, strlen(processFuncName));
strncat(startFuncName, startSuffix, strlen(startSuffix));
uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc));
char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
char *finishSuffix = "_finish";
strncpy(finishFuncName, processFuncName, strlen(processFuncName));
strncat(finishFuncName, finishSuffix, strlen(finishSuffix));
uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc));
//TODO: merge
}
return 0; return 0;
} }
@ -160,6 +181,9 @@ void udfdProcessRequest(uv_work_t *req) {
rsp.type = request.type; rsp.type = request.type;
rsp.code = 0; rsp.code = 0;
rsp.setupRsp.udfHandle = (int64_t)(handle); rsp.setupRsp.udfHandle = (int64_t)(handle);
rsp.setupRsp.outputType = udf->outputType;
rsp.setupRsp.outputLen = udf->outputLen;
rsp.setupRsp.bufSize = udf->bufSize;
int32_t len = encodeUdfResponse(NULL, &rsp); int32_t len = encodeUdfResponse(NULL, &rsp);
rsp.msgLen = len; rsp.msgLen = len;
void *bufBegin = taosMemoryMalloc(len); void *bufBegin = taosMemoryMalloc(len);
@ -178,26 +202,58 @@ void udfdProcessRequest(uv_work_t *req) {
call->udfHandle); call->udfHandle);
SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
SUdf *udf = handle->udf; SUdf *udf = handle->udf;
SUdfDataBlock input = {0};
convertDataBlockToUdfDataBlock(&call->block, &input);
SUdfColumn output = {0};
// TODO: call different functions according to call type, for now just calar
if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
udf->scalarProcFunc(input, &output);
}
SUdfResponse response = {0}; SUdfResponse response = {0};
SUdfResponse *rsp = &response; SUdfResponse *rsp = &response;
if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { SUdfCallResponse *subRsp = &rsp->callRsp;
rsp->seqNum = request.seqNum;
rsp->type = request.type; switch(call->callType) {
rsp->code = 0; case TSDB_UDF_CALL_SCALA_PROC: {
SUdfCallResponse *subRsp = &rsp->callRsp; SUdfColumn output = {0};
subRsp->callType = call->callType;
convertUdfColumnToDataBlock(&output, &subRsp->resultData); SUdfDataBlock input = {0};
convertDataBlockToUdfDataBlock(&call->block, &input);
udf->scalarProcFunc(&input, &output);
convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
udf->freeUdfColumn(&output);
break;
}
case TSDB_UDF_CALL_AGG_INIT: {
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
.bufLen= udf->bufSize,
.numOfResult = 0};
udf->aggStartFunc(&outBuf);
subRsp->resultBuf = outBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
SUdfDataBlock input = {0};
convertDataBlockToUdfDataBlock(&call->block, &input);
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
.bufLen= udf->bufSize,
.numOfResult = 0};
udf->aggProcFunc(&input, &call->interBuf, &outBuf);
subRsp->resultBuf = outBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
.bufLen= udf->bufSize,
.numOfResult = 0};
udf->aggFinishFunc(&call->interBuf, &outBuf);
subRsp->resultBuf = outBuf;
break;
}
default:
break;
} }
rsp->seqNum = request.seqNum;
rsp->type = request.type;
rsp->code = 0;
subRsp->callType = call->callType;
int32_t len = encodeUdfResponse(NULL, rsp); int32_t len = encodeUdfResponse(NULL, rsp);
rsp->msgLen = len; rsp->msgLen = len;
void *bufBegin = taosMemoryMalloc(len); void *bufBegin = taosMemoryMalloc(len);
@ -205,9 +261,6 @@ void udfdProcessRequest(uv_work_t *req) {
encodeUdfResponse(&buf, rsp); encodeUdfResponse(&buf, rsp);
uvUdf->output = uv_buf_init(bufBegin, len); uvUdf->output = uv_buf_init(bufBegin, len);
// TODO: free udf column
udf->freeUdfColumn(&output);
taosMemoryFree(uvUdf->input.base); taosMemoryFree(uvUdf->input.base);
break; break;
} }

10
source/libs/function/test/udf1.c
View File

@ -9,18 +9,18 @@
#undef free #undef free
#define free free #define free free
int32_t udf1_setup() { int32_t udf1_init() {
return 0; return 0;
} }
int32_t udf1_teardown() { int32_t udf1_destroy() {
return 0; return 0;
} }
int32_t udf1(SUdfDataBlock block, SUdfColumn *resultCol) { int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
SUdfColumnData *resultData = &resultCol->colData; SUdfColumnData *resultData = &resultCol->colData;
resultData->numOfRows = block.numOfRows; resultData->numOfRows = block->numOfRows;
SUdfColumnData *srcData = &block.udfCols[0]->colData; SUdfColumnData *srcData = &block->udfCols[0]->colData;
resultData->varLengthColumn = srcData->varLengthColumn; resultData->varLengthColumn = srcData->varLengthColumn;
if (resultData->varLengthColumn) { if (resultData->varLengthColumn) {

52
source/libs/function/test/udf2.c Normal file
View File

@ -0,0 +1,52 @@
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include "tudf.h"
#undef malloc
#define malloc malloc
#undef free
#define free free
int32_t udf2_init() {
return 0;
}
int32_t udf2_destroy() {
return 0;
}
int32_t udf2_start(SUdfInterBuf *buf) {
*(int64_t*)(buf->buf) = 0;
buf->bufLen = sizeof(int64_t);
buf->numOfResult = 0;
return 0;
}
int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) {
int64_t sumSquares = *(int64_t*)interBuf->buf;
for (int32_t i = 0; i < block->numOfCols; ++i) {
for (int32_t j = 0; j < block->numOfRows; ++i) {
SUdfColumn* col = block->udfCols[i];
//TODO: check the bitmap for null value
int32_t* rows = (int32_t*)col->colData.fixLenCol.data;
sumSquares += rows[j] * rows[j];
}
}
*(int64_t*)newInterBuf = sumSquares;
newInterBuf->bufLen = sizeof(int64_t);
//TODO: if all null value, numOfResult = 0;
newInterBuf->numOfResult = 1;
return 0;
}
int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
//TODO: check numOfResults;
int64_t sumSquares = *(int64_t*)(buf->buf);
*(double*)(resultData->buf) = sqrt(sumSquares);
resultData->bufLen = sizeof(double);
resultData->numOfResult = 1;
return 0;
}

8
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -423,6 +423,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
EDealRes res = DEAL_RES_CONTINUE; EDealRes res = DEAL_RES_CONTINUE;
switch (nodeType(pNode)) { switch (nodeType(pNode)) {
case QUERY_NODE_NODE_LIST:
res = walkPhysiPlans(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext); res = walkScanPhysi((SScanPhysiNode*)pNode, order, walker, pContext);
break; break;
@ -534,10 +537,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
break; break;
case QUERY_NODE_PHYSICAL_SUBPLAN: { case QUERY_NODE_PHYSICAL_SUBPLAN: {
SSubplan* pSubplan = (SSubplan*)pNode; SSubplan* pSubplan = (SSubplan*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); res = walkPhysiPlans(pSubplan->pChildren, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlans(pSubplan->pChildren, order, walker, pContext);
}
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = walkPhysiPlan((SNode*)pSubplan->pNode, order, walker, pContext); res = walkPhysiPlan((SNode*)pSubplan->pNode, order, walker, pContext);
} }

24
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -389,21 +389,25 @@ void nodesDestroyNode(SNodeptr pNode) {
case QUERY_NODE_COLUMN_DEF: // no pointer field case QUERY_NODE_COLUMN_DEF: // no pointer field
case QUERY_NODE_DOWNSTREAM_SOURCE: // no pointer field case QUERY_NODE_DOWNSTREAM_SOURCE: // no pointer field
break; break;
case QUERY_NODE_DATABASE_OPTIONS: case QUERY_NODE_DATABASE_OPTIONS: {
nodesDestroyList(((SDatabaseOptions*)pNode)->pRetentions); SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode;
nodesDestroyNode(pOptions->pDaysPerFile);
nodesDestroyList(pOptions->pKeep);
nodesDestroyList(pOptions->pRetentions);
break; break;
}
case QUERY_NODE_TABLE_OPTIONS: { case QUERY_NODE_TABLE_OPTIONS: {
STableOptions* pStmt = (STableOptions*)pNode; STableOptions* pOptions = (STableOptions*)pNode;
nodesDestroyList(pStmt->pSma); nodesDestroyList(pOptions->pSma);
nodesDestroyList(pStmt->pFuncs); nodesDestroyList(pOptions->pRollupFuncs);
break; break;
} }
case QUERY_NODE_INDEX_OPTIONS: { case QUERY_NODE_INDEX_OPTIONS: {
SIndexOptions* pStmt = (SIndexOptions*)pNode; SIndexOptions* pOptions = (SIndexOptions*)pNode;
nodesDestroyList(pStmt->pFuncs); nodesDestroyList(pOptions->pFuncs);
nodesDestroyNode(pStmt->pInterval); nodesDestroyNode(pOptions->pInterval);
nodesDestroyNode(pStmt->pOffset); nodesDestroyNode(pOptions->pOffset);
nodesDestroyNode(pStmt->pSliding); nodesDestroyNode(pOptions->pSliding);
break; break;
} }
case QUERY_NODE_SET_OPERATOR: { case QUERY_NODE_SET_OPERATOR: {

54
source/libs/parser/inc/parAst.h
View File

@ -36,8 +36,7 @@ typedef struct SAstCreateContext {
} SAstCreateContext; } SAstCreateContext;
typedef enum EDatabaseOptionType { typedef enum EDatabaseOptionType {
DB_OPTION_BLOCKS = 1, DB_OPTION_BUFFER = 1,
DB_OPTION_CACHE,
DB_OPTION_CACHELAST, DB_OPTION_CACHELAST,
DB_OPTION_COMP, DB_OPTION_COMP,
DB_OPTION_DAYS, DB_OPTION_DAYS,
@ -45,31 +44,30 @@ typedef enum EDatabaseOptionType {
DB_OPTION_MAXROWS, DB_OPTION_MAXROWS,
DB_OPTION_MINROWS, DB_OPTION_MINROWS,
DB_OPTION_KEEP, DB_OPTION_KEEP,
DB_OPTION_PAGES,
DB_OPTION_PAGESIZE,
DB_OPTION_PRECISION, DB_OPTION_PRECISION,
DB_OPTION_QUORUM,
DB_OPTION_REPLICA, DB_OPTION_REPLICA,
DB_OPTION_TTL, DB_OPTION_STRICT,
DB_OPTION_WAL, DB_OPTION_WAL,
DB_OPTION_VGROUPS, DB_OPTION_VGROUPS,
DB_OPTION_SINGLE_STABLE, DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAM_MODE,
DB_OPTION_STRICT,
DB_OPTION_RETENTIONS DB_OPTION_RETENTIONS
} EDatabaseOptionType; } EDatabaseOptionType;
typedef enum ETableOptionType { typedef enum ETableOptionType {
TABLE_OPTION_KEEP = 1, TABLE_OPTION_COMMENT = 1,
TABLE_OPTION_TTL, TABLE_OPTION_DELAY,
TABLE_OPTION_COMMENT,
TABLE_OPTION_SMA,
TABLE_OPTION_FILE_FACTOR, TABLE_OPTION_FILE_FACTOR,
TABLE_OPTION_DELAY TABLE_OPTION_ROLLUP,
TABLE_OPTION_TTL,
TABLE_OPTION_SMA
} ETableOptionType; } ETableOptionType;
typedef struct SAlterOption { typedef struct SAlterOption {
int32_t type; int32_t type;
SValueNode* pVal; SToken val;
SNodeList* pList; SNodeList* pList;
} SAlterOption; } SAlterOption;
extern SToken nil_token; extern SToken nil_token;
@ -121,25 +119,29 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createTableOptions(SAstCreateContext* pCxt);
SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type); SDataType createDataType(uint8_t type);
SDataType createVarLenDataType(uint8_t type, const SToken* pLen); SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols,
SNodeList* pTags, SNode* pOptions); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal);
SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal);
SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment);
SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols,
SNodeList* pTags, SNode* pOptions);
SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable, SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable,
SNodeList* pSpecificTags, SNodeList* pValsOfTags); SNodeList* pSpecificTags, SNodeList* pValsOfTags, SNode* pOptions);
SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables); SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables);
SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables); SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables);
SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions); SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
const SToken* pColName, SDataType dataType); const SToken* pColName, SDataType dataType);
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName); SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName);

90
source/libs/parser/inc/sql.y
View File

@ -137,43 +137,41 @@ not_exists_opt(A) ::= .
exists_opt(A) ::= IF EXISTS. { A = true; } exists_opt(A) ::= IF EXISTS. { A = true; }
exists_opt(A) ::= . { A = false; } exists_opt(A) ::= . { A = false; }
db_options(A) ::= . { A = createDatabaseOptions(pCxt); } db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); }
db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); }
db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); }
db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); }
db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) DAYS NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
db_options(A) ::= db_options(B) DAYS NK_VARIABLE(C). { ((SDatabaseOptions*)B)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &C); A = B; } db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); }
db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { ((SDatabaseOptions*)B)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); }
db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); }
db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) KEEP integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, C); }
db_options(A) ::= db_options(B) KEEP integer_list(C). { ((SDatabaseOptions*)B)->pKeep = C; A = B; } db_options(A) ::= db_options(B) KEEP variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, C); }
db_options(A) ::= db_options(B) KEEP variable_list(C). { ((SDatabaseOptions*)B)->pKeep = C; A = B; } db_options(A) ::= db_options(B) PAGES NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGES, &C); }
db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { ((SDatabaseOptions*)B)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C); A = B; } db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGESIZE, &C); }
db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { ((SDatabaseOptions*)B)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); }
db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { ((SDatabaseOptions*)B)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); }
db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { ((SDatabaseOptions*)B)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) STRICT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); }
db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { ((SDatabaseOptions*)B)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, C); }
db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { ((SDatabaseOptions*)B)->pRetentions = C; A = B; }
db_options(A) ::= db_options(B) STRICT NK_INTEGER(C). { ((SDatabaseOptions*)B)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
alter_db_options(A) ::= alter_db_option(B). { A = createDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseAlterOption(pCxt, B, &C); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }
%type alter_db_option { SAlterOption } %type alter_db_option { SAlterOption }
%destructor alter_db_option { } %destructor alter_db_option { }
alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= PAGES NK_INTEGER(B). { A.type = DB_OPTION_PAGES; A.val = B; }
alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= STRICT NK_INTEGER(B). { A.type = DB_OPTION_STRICT; A.val = B; }
alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; }
alter_db_option(A) ::= STRICT NK_INTEGER(B). { A.type = DB_OPTION_STRICT; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
%type integer_list { SNodeList* } %type integer_list { SNodeList* }
%destructor integer_list { nodesDestroyList($$); } %destructor integer_list { nodesDestroyList($$); }
@ -204,7 +202,7 @@ cmd ::= DROP STABLE exists_opt(A) full_table_name(B).
cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; } cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; }
cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; } cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; }
alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); } alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableModifyOptions(pCxt, B, C); }
alter_table_clause(A) ::= alter_table_clause(A) ::=
full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); } full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); } alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
@ -229,7 +227,7 @@ multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C).
create_subtable_clause(A) ::= create_subtable_clause(A) ::=
not_exists_opt(B) full_table_name(C) USING full_table_name(D) not_exists_opt(B) full_table_name(C) USING full_table_name(D)
specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP. { A = createCreateSubTableClause(pCxt, B, C, D, E, F); } specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
%type multi_drop_clause { SNodeList* } %type multi_drop_clause { SNodeList* }
%destructor multi_drop_clause { nodesDestroyList($$); } %destructor multi_drop_clause { nodesDestroyList($$); }
@ -289,25 +287,21 @@ tags_def_opt(A) ::= tags_def(B).
%destructor tags_def { nodesDestroyList($$); } %destructor tags_def { nodesDestroyList($$); }
tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; }
table_options(A) ::= . { A = createTableOptions(pCxt); } table_options(A) ::= . { A = createDefaultTableOptions(pCxt); }
table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { ((STableOptions*)B)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C); A = B; } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); }
table_options(A) ::= table_options(B) KEEP integer_list(C). { ((STableOptions*)B)->pKeep = C; A = B; } table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); }
table_options(A) ::= table_options(B) KEEP variable_list(C). { ((STableOptions*)B)->pKeep = C; A = B; } table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { ((STableOptions*)B)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_ROLLUP, C); }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { ((STableOptions*)B)->pSma = C; A = B; } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { ((STableOptions*)B)->pFuncs = C; A = B; } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_SMA, C); }
table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { ((STableOptions*)B)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &C); A = B; }
table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { ((STableOptions*)B)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
alter_table_options(A) ::= alter_table_option(B). { A = createTableOptions(pCxt); A = setTableAlterOption(pCxt, A, &B); } alter_table_options(A) ::= alter_table_option(B). { A = createAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableAlterOption(pCxt, B, &C); } alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); }
%type alter_table_option { SAlterOption } %type alter_table_option { SAlterOption }
%destructor alter_table_option { } %destructor alter_table_option { }
alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; }
alter_table_option(A) ::= KEEP integer_list(B). { A.type = TABLE_OPTION_KEEP; A.pList = B; } alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; }
alter_table_option(A) ::= KEEP variable_list(B). { A.type = TABLE_OPTION_KEEP; A.pList = B; }
alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
%type col_name_list { SNodeList* } %type col_name_list { SNodeList* }
%destructor col_name_list { nodesDestroyList($$); } %destructor col_name_list { nodesDestroyList($$); }

180
source/libs/parser/src/parAstCreater.c
View File

@ -44,7 +44,13 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
pCxt->notSupport = false; pCxt->notSupport = false;
pCxt->valid = true; pCxt->valid = true;
pCxt->pRootNode = NULL; pCxt->pRootNode = NULL;
pCxt->placeholderNo = 1; pCxt->placeholderNo = 0;
}
static void copyStringFormStringToken(SToken* pToken, char* pBuf, int32_t len) {
if (pToken->n > 2) {
strncpy(pBuf, pToken->z + 1, TMIN(pToken->n - 2, len - 1));
}
} }
static void trimEscape(SToken* pName) { static void trimEscape(SToken* pName) {
@ -309,7 +315,7 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera
CHECK_OUT_OF_MEM(val); CHECK_OUT_OF_MEM(val);
val->literal = strndup(pLiteral->z, pLiteral->n); val->literal = strndup(pLiteral->z, pLiteral->n);
CHECK_OUT_OF_MEM(val->literal); CHECK_OUT_OF_MEM(val->literal);
val->placeholderNo = pCxt->placeholderNo++; val->placeholderNo = ++pCxt->placeholderNo;
return (SNode*)val; return (SNode*)val;
} }
@ -374,6 +380,11 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
CHECK_OUT_OF_MEM(func); CHECK_OUT_OF_MEM(func);
strcpy(func->functionName, "cast"); strcpy(func->functionName, "cast");
func->node.resType = dt; func->node.resType = dt;
if (TSDB_DATA_TYPE_BINARY == dt.type) {
func->node.resType.bytes += 2;
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2;
}
nodesListMakeAppend(&func->pParameterList, pExpr); nodesListMakeAppend(&func->pParameterList, pExpr);
return (SNode*)func; return (SNode*)func;
} }
@ -604,64 +615,113 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
return (SNode*)setOp; return (SNode*)setOp;
} }
SNode* createDatabaseOptions(SAstCreateContext* pCxt) { SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
pOptions->cachelast = TSDB_DEFAULT_CACHE_LAST_ROW;
pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL;
pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
pOptions->maxRowsPerBlock = TSDB_DEFAULT_MAXROWS_FBLOCK;
pOptions->minRowsPerBlock = TSDB_DEFAULT_MINROWS_FBLOCK;
pOptions->keep[0] = TSDB_DEFAULT_KEEP;
pOptions->keep[1] = TSDB_DEFAULT_KEEP;
pOptions->keep[2] = TSDB_DEFAULT_KEEP;
pOptions->pages = TSDB_DEFAULT_PAGES_PER_VNODE;
pOptions->pagesize = TSDB_DEFAULT_PAGESIZE_PER_VNODE;
pOptions->precision = TSDB_DEFAULT_PRECISION;
pOptions->replica = TSDB_DEFAULT_DB_REPLICA;
pOptions->strict = TSDB_DEFAULT_DB_STRICT;
pOptions->walLevel = TSDB_DEFAULT_WAL_LEVEL;
pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) { SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
switch (pAlterOption->type) { SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
case DB_OPTION_BLOCKS: CHECK_OUT_OF_MEM(pOptions);
((SDatabaseOptions*)pOptions)->pNumOfBlocks = pAlterOption->pVal; pOptions->buffer = -1;
break; pOptions->cachelast = -1;
case DB_OPTION_CACHE: pOptions->compressionLevel = -1;
((SDatabaseOptions*)pOptions)->pCacheBlockSize = pAlterOption->pVal; pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
pOptions->maxRowsPerBlock = -1;
pOptions->minRowsPerBlock = -1;
pOptions->keep[0] = -1;
pOptions->keep[1] = -1;
pOptions->keep[2] = -1;
pOptions->pages = -1;
pOptions->pagesize = -1;
pOptions->precision = -1;
pOptions->replica = -1;
pOptions->strict = -1;
pOptions->walLevel = -1;
pOptions->numOfVgroups = -1;
pOptions->singleStable = -1;
return (SNode*)pOptions;
}
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) {
switch (type) {
case DB_OPTION_BUFFER:
((SDatabaseOptions*)pOptions)->buffer = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_CACHELAST: case DB_OPTION_CACHELAST:
((SDatabaseOptions*)pOptions)->pCachelast = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->cachelast = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_COMP: case DB_OPTION_COMP:
((SDatabaseOptions*)pOptions)->pCompressionLevel = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->compressionLevel = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_DAYS: case DB_OPTION_DAYS: {
((SDatabaseOptions*)pOptions)->pDaysPerFile = pAlterOption->pVal; SToken* pToken = pVal;
if (TK_NK_INTEGER == pToken->type) {
((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10);
} else {
((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken);
}
break; break;
}
case DB_OPTION_FSYNC: case DB_OPTION_FSYNC:
((SDatabaseOptions*)pOptions)->pFsyncPeriod = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->fsyncPeriod = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_MAXROWS: case DB_OPTION_MAXROWS:
((SDatabaseOptions*)pOptions)->pMaxRowsPerBlock = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_MINROWS: case DB_OPTION_MINROWS:
((SDatabaseOptions*)pOptions)->pMinRowsPerBlock = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->minRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_KEEP: case DB_OPTION_KEEP:
((SDatabaseOptions*)pOptions)->pKeep = pAlterOption->pList; ((SDatabaseOptions*)pOptions)->pKeep = pVal;
break;
case DB_OPTION_PAGES:
((SDatabaseOptions*)pOptions)->pages = strtol(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_PAGESIZE:
((SDatabaseOptions*)pOptions)->pagesize = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_PRECISION: case DB_OPTION_PRECISION:
((SDatabaseOptions*)pOptions)->pPrecision = pAlterOption->pVal; copyStringFormStringToken((SToken*)pVal, ((SDatabaseOptions*)pOptions)->precisionStr,
sizeof(((SDatabaseOptions*)pOptions)->precisionStr));
break; break;
case DB_OPTION_REPLICA: case DB_OPTION_REPLICA:
((SDatabaseOptions*)pOptions)->pReplica = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->replica = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_TTL: case DB_OPTION_STRICT:
((SDatabaseOptions*)pOptions)->pTtl = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->strict = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_WAL: case DB_OPTION_WAL:
((SDatabaseOptions*)pOptions)->pWalLevel = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->walLevel = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_VGROUPS: case DB_OPTION_VGROUPS:
((SDatabaseOptions*)pOptions)->pNumOfVgroups = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->numOfVgroups = strtol(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_SINGLE_STABLE: case DB_OPTION_SINGLE_STABLE:
((SDatabaseOptions*)pOptions)->pSingleStable = pAlterOption->pVal; ((SDatabaseOptions*)pOptions)->singleStable = strtol(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_STREAM_MODE:
((SDatabaseOptions*)pOptions)->pStreamMode = pAlterOption->pVal;
break; break;
case DB_OPTION_RETENTIONS: case DB_OPTION_RETENTIONS:
((SDatabaseOptions*)pOptions)->pRetentions = pAlterOption->pList; ((SDatabaseOptions*)pOptions)->pRetentions = pVal;
break; break;
default: default:
break; break;
@ -669,6 +729,17 @@ SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp
return pOptions; return pOptions;
} }
SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
switch (pAlterOption->type) {
case DB_OPTION_KEEP:
case DB_OPTION_RETENTIONS:
return setDatabaseOption(pCxt, pOptions, pAlterOption->type, pAlterOption->pList);
default:
break;
}
return setDatabaseOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val);
}
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) { SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) {
if (!checkDbName(pCxt, pDbName, false)) { if (!checkDbName(pCxt, pDbName, false)) {
return NULL; return NULL;
@ -703,31 +774,44 @@ SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode*
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createTableOptions(SAstCreateContext* pCxt) { SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->delay = TSDB_DEFAULT_ROLLUP_DELAY;
pOptions->filesFactor = TSDB_DEFAULT_ROLLUP_FILE_FACTOR;
pOptions->ttl = TSDB_DEFAULT_TABLE_TTL;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) { SNode* createAlterTableOptions(SAstCreateContext* pCxt) {
switch (pAlterOption->type) { STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
case TABLE_OPTION_KEEP: CHECK_OUT_OF_MEM(pOptions);
((STableOptions*)pOptions)->pKeep = pAlterOption->pList; pOptions->delay = -1;
break; pOptions->filesFactor = -1;
case TABLE_OPTION_TTL: pOptions->ttl = -1;
((STableOptions*)pOptions)->pTtl = pAlterOption->pVal; return (SNode*)pOptions;
break; }
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal) {
switch (type) {
case TABLE_OPTION_COMMENT: case TABLE_OPTION_COMMENT:
((STableOptions*)pOptions)->pComments = pAlterOption->pVal; copyStringFormStringToken((SToken*)pVal, ((STableOptions*)pOptions)->comment,
break; sizeof(((STableOptions*)pOptions)->comment));
case TABLE_OPTION_SMA:
((STableOptions*)pOptions)->pSma = pAlterOption->pList;
break;
case TABLE_OPTION_FILE_FACTOR:
((STableOptions*)pOptions)->pFilesFactor = pAlterOption->pVal;
break; break;
case TABLE_OPTION_DELAY: case TABLE_OPTION_DELAY:
((STableOptions*)pOptions)->pDelay = pAlterOption->pVal; ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10);
break;
case TABLE_OPTION_FILE_FACTOR:
((STableOptions*)pOptions)->filesFactor = strtod(((SToken*)pVal)->z, NULL);
break;
case TABLE_OPTION_ROLLUP:
((STableOptions*)pOptions)->pRollupFuncs = pVal;
break;
case TABLE_OPTION_TTL:
((STableOptions*)pOptions)->ttl = strtol(((SToken*)pVal)->z, NULL, 10);
break;
case TABLE_OPTION_SMA:
((STableOptions*)pOptions)->pSma = pVal;
break; break;
default: default:
break; break;
@ -778,7 +862,7 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode*
} }
SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable, SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable,
SNodeList* pSpecificTags, SNodeList* pValsOfTags) { SNodeList* pSpecificTags, SNodeList* pValsOfTags, SNode* pOptions) {
if (NULL == pRealTable) { if (NULL == pRealTable) {
return NULL; return NULL;
} }
@ -833,7 +917,7 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) { SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) {
if (NULL == pRealTable) { if (NULL == pRealTable) {
return NULL; return NULL;
} }

1
source/libs/parser/src/parAstParser.c
View File

@ -80,6 +80,7 @@ abort_parse:
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
(*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->pRoot = cxt.pRootNode;
(*pQuery)->placeholderNum = cxt.placeholderNo;
} }
return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED;
} }

27
source/libs/parser/src/parInsert.c
View File

@ -937,6 +937,11 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
} }
*gotRow = true; *gotRow = true;
#ifdef TD_DEBUG_PRINT_ROW
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&schema, spd->numOfCols);
tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema);
#endif
} }
// *len = pBuilder->extendedRowSize; // *len = pBuilder->extendedRowSize;
@ -1328,10 +1333,6 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
for (int c = 0; c < spd->numOfBound; ++c) { for (int c = 0; c < spd->numOfBound; ++c) {
SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1];
if (bind[c].buffer_type != pColSchema->type) {
return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
}
if (bind[c].num != rowNum) { if (bind[c].num != rowNum) {
return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
} }
@ -1346,6 +1347,10 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, &param)); CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, &param));
} else { } else {
if (bind[c].buffer_type != pColSchema->type) {
return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
}
int32_t colLen = pColSchema->bytes; int32_t colLen = pColSchema->bytes;
if (IS_VAR_DATA_TYPE(pColSchema->type)) { if (IS_VAR_DATA_TYPE(pColSchema->type)) {
colLen = bind[c].length[r]; colLen = bind[c].length[r];
@ -1359,7 +1364,6 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
checkTimestamp(pDataBlock, (const char*)&tsKey); checkTimestamp(pDataBlock, (const char*)&tsKey);
} }
} }
// set the null value for the columns that do not assign values // set the null value for the columns that do not assign values
if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
for (int32_t i = 0; i < spd->numOfCols; ++i) { for (int32_t i = 0; i < spd->numOfCols; ++i) {
@ -1369,6 +1373,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in
} }
} }
} }
#ifdef TD_DEBUG_PRINT_ROW
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema);
#endif
pDataBlock->size += extendedRowSize; pDataBlock->size += extendedRowSize;
} }
@ -1447,6 +1456,14 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu
} }
} }
} }
#ifdef TD_DEBUG_PRINT_ROW
if(rowEnd) {
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema);
}
#endif
} }
if (rowEnd) { if (rowEnd) {

6
source/libs/parser/src/parTokenizer.c
View File

@ -43,7 +43,7 @@ static SKeyword keywordTable[] = {
{"BETWEEN", TK_BETWEEN}, {"BETWEEN", TK_BETWEEN},
{"BINARY", TK_BINARY}, {"BINARY", TK_BINARY},
{"BIGINT", TK_BIGINT}, {"BIGINT", TK_BIGINT},
{"BLOCKS", TK_BLOCKS}, // {"BLOCKS", TK_BLOCKS},
{"BNODE", TK_BNODE}, {"BNODE", TK_BNODE},
{"BNODES", TK_BNODES}, {"BNODES", TK_BNODES},
{"BOOL", TK_BOOL}, {"BOOL", TK_BOOL},
@ -142,7 +142,7 @@ static SKeyword keywordTable[] = {
{"QTIME", TK_QTIME}, {"QTIME", TK_QTIME},
{"QUERIES", TK_QUERIES}, {"QUERIES", TK_QUERIES},
{"QUERY", TK_QUERY}, {"QUERY", TK_QUERY},
{"QUORUM", TK_QUORUM}, // {"QUORUM", TK_QUORUM},
{"RATIO", TK_RATIO}, {"RATIO", TK_RATIO},
{"REPLICA", TK_REPLICA}, {"REPLICA", TK_REPLICA},
{"RESET", TK_RESET}, {"RESET", TK_RESET},
@ -169,7 +169,7 @@ static SKeyword keywordTable[] = {
{"STORAGE", TK_STORAGE}, {"STORAGE", TK_STORAGE},
{"STREAM", TK_STREAM}, {"STREAM", TK_STREAM},
{"STREAMS", TK_STREAMS}, {"STREAMS", TK_STREAMS},
{"STREAM_MODE", TK_STREAM_MODE}, // {"STREAM_MODE", TK_STREAM_MODE},
{"STRICT", TK_STRICT}, {"STRICT", TK_STRICT},
{"SYNCDB", TK_SYNCDB}, {"SYNCDB", TK_SYNCDB},
{"TABLE", TK_TABLE}, {"TABLE", TK_TABLE},

356
source/libs/parser/src/parTranslater.c
View File

@ -24,7 +24,8 @@
#include "ttime.h" #include "ttime.h"
#include "systable.h" #include "systable.h"
#define GET_OPTION_VAL(pVal, defaultVal) (NULL == (pVal) ? (defaultVal) : getBigintFromValueNode((SValueNode*)(pVal))) #define generateDealNodeErrMsg(pCxt, code, ...) \
(pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__) ? DEAL_RES_ERROR : DEAL_RES_ERROR)
typedef struct STranslateContext { typedef struct STranslateContext {
SParseContext* pParseCxt; SParseContext* pParseCxt;
@ -51,15 +52,6 @@ static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_B
static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; }
enum EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t code, ...) {
va_list ap;
va_start(ap, code);
generateSyntaxErrMsg(&pCxt->msgBuf, code, ap);
va_end(ap);
pCxt->errCode = code;
return DEAL_RES_ERROR;
}
static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel);
if (currTotalLevel > pCxt->currLevel) { if (currTotalLevel > pCxt->currLevel) {
@ -549,8 +541,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) ||
(TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_VAR_DATA_TYPE(rdt.type)) || (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_VAR_DATA_TYPE(ldt.type))) { (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
@ -1547,160 +1539,123 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db); tNameGetFullDbName(&name, pReq->db);
pReq->numOfVgroups = GET_OPTION_VAL(pStmt->pOptions->pNumOfVgroups, TSDB_DEFAULT_VN_PER_DB); pReq->numOfVgroups = pStmt->pOptions->numOfVgroups;
pReq->cacheBlockSize = GET_OPTION_VAL(pStmt->pOptions->pCacheBlockSize, TSDB_DEFAULT_CACHE_BLOCK_SIZE); pReq->daysPerFile = pStmt->pOptions->daysPerFile;
pReq->totalBlocks = GET_OPTION_VAL(pStmt->pOptions->pNumOfBlocks, TSDB_DEFAULT_TOTAL_BLOCKS); pReq->daysToKeep0 = pStmt->pOptions->keep[0];
pReq->daysPerFile = GET_OPTION_VAL(pStmt->pOptions->pDaysPerFile, TSDB_DEFAULT_DAYS_PER_FILE); pReq->daysToKeep1 = pStmt->pOptions->keep[1];
pReq->daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 0), TSDB_DEFAULT_KEEP); pReq->daysToKeep2 = pStmt->pOptions->keep[2];
pReq->daysToKeep1 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 1), TSDB_DEFAULT_KEEP); pReq->minRows = pStmt->pOptions->minRowsPerBlock;
pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), TSDB_DEFAULT_KEEP); pReq->maxRows = pStmt->pOptions->maxRowsPerBlock;
pReq->minRows = GET_OPTION_VAL(pStmt->pOptions->pMinRowsPerBlock, TSDB_DEFAULT_MINROWS_FBLOCK);
pReq->maxRows = GET_OPTION_VAL(pStmt->pOptions->pMaxRowsPerBlock, TSDB_DEFAULT_MAXROWS_FBLOCK);
pReq->commitTime = -1; pReq->commitTime = -1;
pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, TSDB_DEFAULT_FSYNC_PERIOD); pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, TSDB_DEFAULT_WAL_LEVEL); pReq->walLevel = pStmt->pOptions->walLevel;
pReq->precision = GET_OPTION_VAL(pStmt->pOptions->pPrecision, TSDB_TIME_PRECISION_MILLI); pReq->precision = pStmt->pOptions->precision;
pReq->compression = GET_OPTION_VAL(pStmt->pOptions->pCompressionLevel, TSDB_DEFAULT_COMP_LEVEL); pReq->compression = pStmt->pOptions->compressionLevel;
pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, TSDB_DEFAULT_DB_REPLICA); pReq->replications = pStmt->pOptions->replica;
pReq->update = -1; pReq->update = -1;
pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, TSDB_DEFAULT_CACHE_LAST_ROW); pReq->cacheLastRow = pStmt->pOptions->cachelast;
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
pReq->streamMode = GET_OPTION_VAL(pStmt->pOptions->pStreamMode, TSDB_DEFAULT_DB_STREAM_MODE); pReq->singleSTable = pStmt->pOptions->singleStable;
pReq->ttl = GET_OPTION_VAL(pStmt->pOptions->pTtl, TSDB_DEFAULT_DB_TTL); pReq->strict = pStmt->pOptions->strict;
pReq->singleSTable = GET_OPTION_VAL(pStmt->pOptions->pSingleStable, TSDB_DEFAULT_DB_SINGLE_STABLE); // pStmt->pOptions->buffer;
pReq->strict = GET_OPTION_VAL(pStmt->pOptions->pStrict, TSDB_DEFAULT_DB_STRICT); // pStmt->pOptions->pages;
// pStmt->pOptions->pagesize;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
} }
static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, SValueNode* pVal, int32_t minVal, static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t minVal,
int32_t maxVal) { int32_t maxVal) {
if (NULL != pVal) { if (val >= 0 && (val < minVal || val > maxVal)) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal);
return pCxt->errCode;
}
if (pVal->isDuration &&
(TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit && TIME_UNIT_DAY != pVal->unit)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_OPTION_UNIT, pName, pVal->unit);
}
int64_t val = getBigintFromValueNode(pVal);
if (val < minVal || val > maxVal) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void convertValueFromStrToInt(SValueNode* pVal, int64_t val) { static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
taosMemoryFreeClear(pVal->datum.p); if (NULL != pOptions->pDaysPerFile) {
pVal->datum.i = val; if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) {
pVal->node.resType.type = TSDB_DATA_TYPE_BIGINT;
pVal->node.resType.bytes = tDataTypes[pVal->node.resType.type].bytes;
}
static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode; return pCxt->errCode;
} }
char* pRrecision = varDataVal(pVal->datum.p); if (TIME_UNIT_MINUTE != pOptions->pDaysPerFile->unit && TIME_UNIT_HOUR != pOptions->pDaysPerFile->unit &&
if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_MILLI_STR)) { TIME_UNIT_DAY != pOptions->pDaysPerFile->unit) {
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_MILLI); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_OPTION_UNIT, "daysPerFile",
} else if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_MICRO_STR)) { pOptions->pDaysPerFile->unit);
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_MICRO);
} else if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_NANO_STR)) {
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_NANO);
} else {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pVal->datum.p);
} }
pOptions->daysPerFile = getBigintFromValueNode(pOptions->pDaysPerFile);
} }
return TSDB_CODE_SUCCESS; return checkRangeOption(pCxt, "daysPerFile", pOptions->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
} }
static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, SValueNode* pVal, int32_t v1, int32_t v2) { static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (NULL != pVal) { if (NULL == pOptions->pKeep) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
int64_t val = pVal->datum.i;
if (val != v1 && val != v2) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkTtlOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
int64_t val = pVal->datum.i;
if (val < TSDB_MIN_DB_TTL) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) {
if (NULL == pKeep) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t numOfKeep = LIST_LENGTH(pKeep); int32_t numOfKeep = LIST_LENGTH(pOptions->pKeep);
if (numOfKeep > 3 || numOfKeep < 1) { if (numOfKeep > 3 || numOfKeep < 1) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_NUM); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_NUM);
} }
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pKeep) { FOREACH(pNode, pOptions->pKeep) {
if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode)) { SValueNode* pVal = (SValueNode*)pNode;
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode; return pCxt->errCode;
} }
} if (pVal->isDuration && TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit &&
TIME_UNIT_DAY != pVal->unit) {
if (1 == numOfKeep) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pVal->unit);
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pKeep, nodesCloneNode(nodesListGetNode(pKeep, 0)))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
++numOfKeep;
}
if (2 == numOfKeep) {
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pKeep, nodesCloneNode(nodesListGetNode(pKeep, 1)))) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
} }
SValueNode* pKeep0 = (SValueNode*)nodesListGetNode(pKeep, 0); pOptions->keep[0] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 0));
SValueNode* pKeep1 = (SValueNode*)nodesListGetNode(pKeep, 1);
SValueNode* pKeep2 = (SValueNode*)nodesListGetNode(pKeep, 2); if (numOfKeep < 2) {
if ((pKeep0->isDuration && pOptions->keep[1] = pOptions->keep[0];
(TIME_UNIT_MINUTE != pKeep0->unit && TIME_UNIT_HOUR != pKeep0->unit && TIME_UNIT_DAY != pKeep0->unit)) || } else {
(pKeep1->isDuration && pOptions->keep[1] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 1));
(TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || }
(pKeep2->isDuration && if (numOfKeep < 3) {
(TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { pOptions->keep[2] = pOptions->keep[1];
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, } else {
pKeep2->unit); pOptions->keep[2] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 2));
} }
int32_t daysToKeep0 = getBigintFromValueNode(pKeep0); if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP ||
int32_t daysToKeep1 = getBigintFromValueNode(pKeep1); pOptions->keep[0] > TSDB_MAX_KEEP || pOptions->keep[1] > TSDB_MAX_KEEP || pOptions->keep[2] > TSDB_MAX_KEEP) {
int32_t daysToKeep2 = getBigintFromValueNode(pKeep2); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, pOptions->keep[0], pOptions->keep[1],
if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP || pOptions->keep[2], TSDB_MIN_KEEP, TSDB_MAX_KEEP);
daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2,
TSDB_MIN_KEEP, TSDB_MAX_KEEP);
} }
if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) { if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_ORDER); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_ORDER);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->precisionStr[0]) {
if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_MILLI;
} else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MICRO_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_MICRO;
} else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_NANO_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_NANO;
} else {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pOptions->precisionStr);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) {
if (val >= 0 && val != v1 && val != v2) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2);
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) { static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) {
if (NULL == pRetentions) { if (NULL == pRetentions) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1725,11 +1680,8 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions,
bool alter) { bool alter) {
if (NULL == pOptions->pDaysPerFile && NULL == pOptions->pKeep) { int32_t daysPerFile = pOptions->daysPerFile;
return TSDB_CODE_SUCCESS; int32_t daysToKeep0 = pOptions->keep[0];
}
int64_t daysPerFile = GET_OPTION_VAL(pOptions->pDaysPerFile, alter ? -1 : TSDB_DEFAULT_DAYS_PER_FILE);
int64_t daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pOptions->pKeep, 0), alter ? -1 : TSDB_DEFAULT_KEEP);
if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) {
SDbCfgInfo dbCfg; SDbCfgInfo dbCfg;
int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); int32_t code = getDBCfg(pCxt, pDbName, &dbCfg);
@ -1737,7 +1689,7 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
return code; return code;
} }
daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile); daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile);
daysToKeep0 = (-1 == daysPerFile ? dbCfg.daysToKeep0 : daysToKeep0); daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0);
} }
if (daysPerFile > daysToKeep0) { if (daysPerFile > daysToKeep0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DAYS_VALUE); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DAYS_VALUE);
@ -1747,66 +1699,60 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions,
bool alter) { bool alter) {
int32_t code = int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, INT32_MAX);
checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheBlockSize", pOptions->pCacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, code = checkRangeOption(pCxt, "cacheLast", pOptions->cachelast, TSDB_MIN_DB_CACHE_LAST_ROW,
TSDB_MAX_CACHE_BLOCK_SIZE);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLast", pOptions->pCachelast, TSDB_MIN_DB_CACHE_LAST_ROW,
TSDB_MAX_DB_CACHE_LAST_ROW); TSDB_MAX_DB_CACHE_LAST_ROW);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "compression", pOptions->pCompressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = code = checkDbDaysOption(pCxt, pOptions);
checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->pFsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "maxRowsPerBlock", pOptions->pMaxRowsPerBlock, TSDB_MIN_MAXROWS_FBLOCK, code = checkRangeOption(pCxt, "maxRowsPerBlock", pOptions->maxRowsPerBlock, TSDB_MIN_MAXROWS_FBLOCK,
TSDB_MAX_MAXROWS_FBLOCK); TSDB_MAX_MAXROWS_FBLOCK);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "minRowsPerBlock", pOptions->pMinRowsPerBlock, TSDB_MIN_MINROWS_FBLOCK, code = checkRangeOption(pCxt, "minRowsPerBlock", pOptions->minRowsPerBlock, TSDB_MIN_MINROWS_FBLOCK,
TSDB_MAX_MINROWS_FBLOCK); TSDB_MAX_MINROWS_FBLOCK);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkKeepOption(pCxt, pOptions->pKeep); code = checkDbKeepOption(pCxt, pOptions);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbPrecisionOption(pCxt, pOptions->pPrecision); code = checkRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, INT32_MAX);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "replications", pOptions->pReplica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); code = checkRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE,
TSDB_MAX_PAGESIZE_PER_VNODE);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkTtlOption(pCxt, pOptions->pTtl); code = checkDbPrecisionOption(pCxt, pOptions);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "walLevel", pOptions->pWalLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL); code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "vgroups", pOptions->pNumOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); code = checkDbEnumOption(pCxt, "strict", pOptions->strict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "singleStable", pOptions->pSingleStable, TSDB_DB_SINGLE_STABLE_ON, code = checkDbEnumOption(pCxt, "walLevel", pOptions->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "singleStable", pOptions->singleStable, TSDB_DB_SINGLE_STABLE_ON,
TSDB_DB_SINGLE_STABLE_OFF); TSDB_DB_SINGLE_STABLE_OFF);
} }
if (TSDB_CODE_SUCCESS == code) {
code =
checkDbEnumOption(pCxt, "streamMode", pOptions->pStreamMode, TSDB_DB_STREAM_MODE_OFF, TSDB_DB_STREAM_MODE_ON);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbRetentionsOption(pCxt, pOptions->pRetentions); code = checkDbRetentionsOption(pCxt, pOptions->pRetentions);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "strict", pOptions->pStrict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions, alter); code = checkOptionsDependency(pCxt, pDbName, pOptions, alter);
} }
@ -1865,15 +1811,16 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db); tNameGetFullDbName(&name, pReq->db);
pReq->totalBlocks = GET_OPTION_VAL(pStmt->pOptions->pNumOfBlocks, -1); // pStmt->pOptions->buffer
pReq->daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 0), -1); pReq->cacheLastRow = pStmt->pOptions->cachelast;
pReq->daysToKeep1 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 1), -1); pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), -1); pReq->daysToKeep0 = pStmt->pOptions->keep[0];
pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, -1); pReq->daysToKeep1 = pStmt->pOptions->keep[1];
pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, -1); pReq->daysToKeep2 = pStmt->pOptions->keep[2];
pReq->strict = GET_OPTION_VAL(pStmt->pOptions->pQuorum, -1); // pStmt->pOptions->pages
pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, -1); pReq->replications = pStmt->pOptions->replica;
pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, -1); pReq->strict = pStmt->pOptions->strict;
pReq->walLevel = pStmt->pOptions->walLevel;
return; return;
} }
@ -1924,27 +1871,10 @@ static SColumnDefNode* findColDef(SNodeList* pCols, const SColumnNode* pCol) {
return NULL; return NULL;
} }
static int32_t checkTableCommentOption(STranslateContext* pCxt, SValueNode* pVal) { static int32_t checTableFactorOption(STranslateContext* pCxt, float val) {
if (NULL != pVal) { if (val < TSDB_MIN_ROLLUP_FILE_FACTOR || val > TSDB_MAX_ROLLUP_FILE_FACTOR) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", val,
return pCxt->errCode; TSDB_MIN_ROLLUP_FILE_FACTOR, TSDB_MAX_ROLLUP_FILE_FACTOR);
}
if (pVal->node.resType.bytes >= TSDB_STB_COMMENT_LEN) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COMMENT_OPTION, TSDB_STB_COMMENT_LEN - 1);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d,
TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1989,25 +1919,19 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
} }
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
int32_t code = checkKeepOption(pCxt, pStmt->pOptions->pKeep); int32_t code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkTtlOption(pCxt, pStmt->pOptions->pTtl); code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkTableCommentOption(pCxt, pStmt->pOptions->pComments); code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "ttl", pStmt->pOptions->ttl, TSDB_MIN_TABLE_TTL, INT32_MAX);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkTableSmaOption(pCxt, pStmt); code = checkTableSmaOption(pCxt, pStmt);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkTableRollupOption(pCxt, pStmt->pOptions->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = checTableFactorOption(pCxt, pStmt->pOptions->pFilesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkTableTags(pCxt, pStmt); code = checkTableTags(pCxt, pStmt);
} }
@ -2144,7 +2068,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) {
} }
SNode* pFunc = NULL; SNode* pFunc = NULL;
FOREACH(pFunc, pStmt->pOptions->pFuncs) { FOREACH(pFunc, pStmt->pOptions->pRollupFuncs) {
SNode* pCol = NULL; SNode* pCol = NULL;
bool primaryKey = true; bool primaryKey = true;
FOREACH(pCol, pStmt->pCols) { FOREACH(pCol, pStmt->pCols) {
@ -2235,8 +2159,8 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt,
static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) {
pReq->igExists = pStmt->ignoreExists; pReq->igExists = pStmt->ignoreExists;
pReq->xFilesFactor = GET_OPTION_VAL(pStmt->pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); pReq->xFilesFactor = pStmt->pOptions->filesFactor;
pReq->delay = GET_OPTION_VAL(pStmt->pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); pReq->delay = pStmt->pOptions->delay;
columnDefNodeToField(pStmt->pCols, &pReq->pColumns); columnDefNodeToField(pStmt->pCols, &pReq->pColumns);
columnDefNodeToField(pStmt->pTags, &pReq->pTags); columnDefNodeToField(pStmt->pTags, &pReq->pTags);
pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); pReq->numOfColumns = LIST_LENGTH(pStmt->pCols);
@ -2735,7 +2659,7 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt*
if ('\0' != pStmt->targetTabName[0]) { if ('\0' != pStmt->targetTabName[0]) {
strcpy(name.dbname, pStmt->targetDbName); strcpy(name.dbname, pStmt->targetDbName);
strcpy(name.tname, pStmt->targetTabName); strcpy(name.tname, pStmt->targetTabName);
tNameExtractFullName(&name, createReq.outputSTbName); tNameExtractFullName(&name, createReq.targetStbFullName);
} }
int32_t code = translateQuery(pCxt, pStmt->pQuery); int32_t code = translateQuery(pCxt, pStmt->pQuery);
@ -3222,23 +3146,6 @@ static void destroyCreateTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->ntb.schema.pSchema); taosMemoryFreeClear(pReq->ntb.schema.pSchema);
} }
static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) {
if (0 == LIST_LENGTH(pOptions->pFuncs)) {
return TSDB_CODE_SUCCESS;
}
#if 0
pReq->ntbCfg.pRSmaParam = taosMemoryCalloc(1, sizeof(SRSmaParam));
if (NULL == pReq->ntbCfg.pRSmaParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReq->ntbCfg.pRSmaParam->delay = GET_OPTION_VAL(pOptions->pDelay, TSDB_DEFAULT_DB_DELAY);
pReq->ntbCfg.pRSmaParam->xFilesFactor = GET_OPTION_VAL(pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR);
#endif
return TSDB_CODE_SUCCESS;
}
static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo, static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo,
SVgroupTablesBatch* pBatch) { SVgroupTablesBatch* pBatch) {
char dbFName[TSDB_DB_FNAME_LEN] = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0};
@ -3262,11 +3169,6 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
toSchema((SColumnDefNode*)pCol, index + 1, req.ntb.schema.pSchema + index); toSchema((SColumnDefNode*)pCol, index + 1, req.ntb.schema.pSchema + index);
++index; ++index;
} }
if (TSDB_CODE_SUCCESS != buildSmaParam(pStmt->pOptions, &req)) {
destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
pBatch->info = *pVgroupInfo; pBatch->info = *pVgroupInfo;
strcpy(pBatch->dbName, pStmt->dbName); strcpy(pBatch->dbName, pStmt->dbName);
pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));

4
source/libs/parser/src/parUtil.c
View File

@ -68,8 +68,6 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Invalid option %s: %s"; return "Invalid option %s: %s";
case TSDB_CODE_PAR_INVALID_ENUM_OPTION: case TSDB_CODE_PAR_INVALID_ENUM_OPTION:
return "Invalid option %s: %" PRId64 ", only %d, %d allowed"; return "Invalid option %s: %" PRId64 ", only %d, %d allowed";
case TSDB_CODE_PAR_INVALID_TTL_OPTION:
return "Invalid option ttl: %" PRId64 ", should be greater than or equal to %d";
case TSDB_CODE_PAR_INVALID_KEEP_NUM: case TSDB_CODE_PAR_INVALID_KEEP_NUM:
return "Invalid number of keep options"; return "Invalid number of keep options";
case TSDB_CODE_PAR_INVALID_KEEP_ORDER: case TSDB_CODE_PAR_INVALID_KEEP_ORDER:
@ -89,7 +87,7 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_INVALID_OPTION_UNIT: case TSDB_CODE_PAR_INVALID_OPTION_UNIT:
return "Invalid option %s unit: %c, only m, h, d allowed"; return "Invalid option %s unit: %c, only m, h, d allowed";
case TSDB_CODE_PAR_INVALID_KEEP_UNIT: case TSDB_CODE_PAR_INVALID_KEEP_UNIT:
return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed"; return "Invalid option keep unit: %c, only m, h, d allowed";
case TSDB_CODE_PAR_AGG_FUNC_NESTING: case TSDB_CODE_PAR_AGG_FUNC_NESTING:
return "Aggregate functions do not support nesting"; return "Aggregate functions do not support nesting";
case TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE: case TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE:

3702
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

13
source/libs/parser/test/parserTestUtil.h → source/libs/parser/test/parAlterTest.cpp
View File

@ -13,9 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef PARSER_TEST_UTIL_H #include "parTestUtil.h"
#define PARSER_TEST_UTIL_H
extern bool g_isDump; using namespace std;
#endif // PARSER_TEST_UTIL_H class ParserAlterTest : public ParserTestBase {};
TEST_F(ParserAlterTest, stmt) {
useDb("root", "test");
run("create database db1");
}

0
source/libs/parser/test/parserInsertTest.cpp → source/libs/parser/test/parInsertTest.cpp
View File

7
source/libs/parser/test/parserTestMain.cpp → source/libs/parser/test/parTestMain.cpp
View File

@ -13,23 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <getopt.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string> #include <string>
#include <getopt.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#ifdef WINDOWS #ifdef WINDOWS
#define TD_USE_WINSOCK #define TD_USE_WINSOCK
#endif #endif
#include "functionMgt.h" #include "functionMgt.h"
#include "mockCatalog.h" #include "mockCatalog.h"
#include "os.h" #include "os.h"
#include "parTestUtil.h"
#include "parToken.h" #include "parToken.h"
#include "parserTestUtil.h"
bool g_isDump = false;
class ParserEnv : public testing::Environment { class ParserEnv : public testing::Environment {
public: public:

149
source/libs/parser/test/parTestUtil.cpp Normal file
View File

@ -0,0 +1,149 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "parTestUtil.h"
#include <algorithm>
#include <array>
#include "parInt.h"
using namespace std;
using namespace testing;
#define DO_WITH_THROW(func, ...) \
do { \
int32_t code__ = func(__VA_ARGS__); \
if (TSDB_CODE_SUCCESS != code__) { \
throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " code:" + to_string(code__) + \
", strerror:" + string(tstrerror(code__)) + ", msg:" + string(stmtEnv_.msgBuf_.data())); \
} \
} while (0);
bool g_isDump = false;
class ParserTestBaseImpl {
public:
void useDb(const string& acctId, const string& db) {
caseEnv_.acctId_ = acctId;
caseEnv_.db_ = db;
}
void run(const string& sql) {
reset();
try {
SParseContext cxt = {0};
setParseContext(sql, &cxt);
SQuery* pQuery = nullptr;
doParse(&cxt, &pQuery);
doTranslate(&cxt, pQuery);
doCalculateConstant(&cxt, pQuery);
if (g_isDump) {
dump();
}
} catch (...) {
dump();
throw;
}
}
private:
struct caseEnv {
string acctId_;
string db_;
};
struct stmtEnv {
string sql_;
array<char, 1024> msgBuf_;
};
struct stmtRes {
string parsedAst_;
string translatedAst_;
string calcConstAst_;
};
void reset() {
stmtEnv_.sql_.clear();
stmtEnv_.msgBuf_.fill(0);
res_.parsedAst_.clear();
res_.translatedAst_.clear();
res_.calcConstAst_.clear();
}
void dump() {
cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl;
cout << "raw syntax tree : " << endl;
cout << res_.parsedAst_ << endl;
cout << "translated syntax tree : " << endl;
cout << res_.translatedAst_ << endl;
cout << "optimized syntax tree : " << endl;
cout << res_.calcConstAst_ << endl;
}
void setParseContext(const string& sql, SParseContext* pCxt) {
stmtEnv_.sql_ = sql;
transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower);
pCxt->acctId = atoi(caseEnv_.acctId_.c_str());
pCxt->db = caseEnv_.db_.c_str();
pCxt->pSql = stmtEnv_.sql_.c_str();
pCxt->sqlLen = stmtEnv_.sql_.length();
pCxt->pMsg = stmtEnv_.msgBuf_.data();
pCxt->msgLen = stmtEnv_.msgBuf_.max_size();
}
void doParse(SParseContext* pCxt, SQuery** pQuery) {
DO_WITH_THROW(parse, pCxt, pQuery);
res_.parsedAst_ = toString((*pQuery)->pRoot);
}
void doTranslate(SParseContext* pCxt, SQuery* pQuery) {
DO_WITH_THROW(translate, pCxt, pQuery);
res_.translatedAst_ = toString(pQuery->pRoot);
}
void doCalculateConstant(SParseContext* pCxt, SQuery* pQuery) {
DO_WITH_THROW(calculateConstant, pCxt, pQuery);
res_.calcConstAst_ = toString(pQuery->pRoot);
}
string toString(const SNode* pRoot) {
char* pStr = NULL;
int32_t len = 0;
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
string str(pStr);
taosMemoryFreeClear(pStr);
return str;
}
caseEnv caseEnv_;
stmtEnv stmtEnv_;
stmtRes res_;
};
ParserTestBase::ParserTestBase() : impl_(new ParserTestBaseImpl()) {}
ParserTestBase::~ParserTestBase() {}
void ParserTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); }
void ParserTestBase::run(const std::string& sql) { return impl_->run(sql); }

37
source/libs/parser/test/parTestUtil.h Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef PARSER_TEST_UTIL_H
#define PARSER_TEST_UTIL_H
#include <gtest/gtest.h>
class ParserTestBaseImpl;
class ParserTestBase : public testing::Test {
public:
ParserTestBase();
virtual ~ParserTestBase();
void useDb(const std::string& acctId, const std::string& db);
void run(const std::string& sql);
private:
std::unique_ptr<ParserTestBaseImpl> impl_;
};
extern bool g_isDump;
#endif // PARSER_TEST_UTIL_H

2
source/libs/parser/test/parserAstTest.cpp
View File

@ -19,7 +19,7 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "parInt.h" #include "parInt.h"
#include "parserTestUtil.h" #include "parTestUtil.h"
using namespace std; using namespace std;
using namespace testing; using namespace testing;

52
source/libs/planner/src/planner.c
View File

@ -19,26 +19,23 @@
typedef struct SCollectPlaceholderValuesCxt { typedef struct SCollectPlaceholderValuesCxt {
int32_t errCode; int32_t errCode;
SNodeList* pValues; SArray* pValues;
} SCollectPlaceholderValuesCxt; } SCollectPlaceholderValuesCxt;
static EDealRes collectPlaceholderValuesImpl(SNode* pNode, void* pContext) { static EDealRes collectPlaceholderValuesImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) && ((SValueNode*)pNode)->placeholderNo > 0) { if (QUERY_NODE_VALUE == nodeType(pNode) && ((SValueNode*)pNode)->placeholderNo > 0) {
SCollectPlaceholderValuesCxt* pCxt = pContext; SCollectPlaceholderValuesCxt* pCxt = pContext;
pCxt->errCode = nodesListMakeAppend(&pCxt->pValues, pNode); taosArrayInsert(pCxt->pValues, ((SValueNode*)pNode)->placeholderNo - 1, &pNode);
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR;
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static int32_t collectPlaceholderValues(SPlanContext* pCxt, SQueryPlan* pPlan) { static int32_t collectPlaceholderValues(SPlanContext* pCxt, SQueryPlan* pPlan) {
SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = NULL}; pPlan->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = pPlan->pPlaceholderValues};
nodesWalkPhysiPlan((SNode*)pPlan, collectPlaceholderValuesImpl, &cxt); nodesWalkPhysiPlan((SNode*)pPlan, collectPlaceholderValuesImpl, &cxt);
if (TSDB_CODE_SUCCESS == cxt.errCode) {
pPlan->pPlaceholderValues = cxt.pValues;
} else {
nodesDestroyList(cxt.pValues);
}
return cxt.errCode; return cxt.errCode;
} }
@ -60,7 +57,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList);
} }
if (TSDB_CODE_SUCCESS == code && pCxt->isStmtQuery) { if (TSDB_CODE_SUCCESS == code && pCxt->placeholderNum > 0) {
code = collectPlaceholderValues(pCxt, *pPlan); code = collectPlaceholderValues(pCxt, *pPlan);
} }
@ -108,7 +105,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.type = pParam->buffer_type;
pVal->node.resType.bytes = *(pParam->length); pVal->node.resType.bytes = NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes;
switch (pParam->buffer_type) { switch (pParam->buffer_type) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
pVal->datum.b = *((bool*)pParam->buffer); pVal->datum.b = *((bool*)pParam->buffer);
@ -133,6 +130,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
break; break;
case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
if (NULL == pVal->datum.p) { if (NULL == pVal->datum.p) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -155,7 +153,6 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
pVal->datum.u = *((uint64_t*)pParam->buffer); pVal->datum.u = *((uint64_t*)pParam->buffer);
break; break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_BLOB:
@ -168,18 +165,35 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx) { static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) {
if (colIdx < 0) { int64_t queryId = *(uint64_t *)pContext;
int32_t index = 0;
SNode* pNode = NULL; if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pNode)) {
SQueryPlan* planNode = (SQueryPlan*)pNode;
planNode->queryId = queryId;
} else if (QUERY_NODE_PHYSICAL_SUBPLAN == nodeType(pNode)) {
SSubplan* subplanNode = (SSubplan*)pNode;
subplanNode->id.queryId = queryId;
}
FOREACH(pNode, pPlan->pPlaceholderValues) { return DEAL_RES_CONTINUE;
setValueByBindParam((SValueNode*)pNode, pParams + index); }
++index;
int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) {
int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues);
if (colIdx < 0) {
for (int32_t i = 0; i < size; ++i) {
setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i);
} }
} else { } else {
setValueByBindParam((SValueNode*)nodesListGetNode(pPlan->pPlaceholderValues, colIdx), pParams); setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams);
} }
if (colIdx < 0 || ((colIdx + 1) == size)) {
nodesWalkPhysiPlan((SNode*)pPlan, updatePlanQueryId, &queryId);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

2
source/libs/planner/test/planTestUtil.cpp
View File

@ -14,9 +14,9 @@
*/ */
#include "planTestUtil.h" #include "planTestUtil.h"
#include <array>
#include <algorithm> #include <algorithm>
#include <array>
#include "cmdnodes.h" #include "cmdnodes.h"
#include "parser.h" #include "parser.h"

2
source/libs/scalar/inc/sclInt.h
View File

@ -34,7 +34,7 @@ typedef struct SScalarCtx {
#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)
#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) #define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0))
#define sclFatal(...) qFatal(__VA_ARGS__) #define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__)

9
source/libs/scalar/src/filter.c
View File

@ -3541,11 +3541,16 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
} }
if (QUERY_NODE_VALUE == nodeType(*pNode)) { if (QUERY_NODE_VALUE == nodeType(*pNode)) {
SValueNode *valueNode = (SValueNode *)*pNode;
if (valueNode->placeholderNo >= 1) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
SValueNode *valueNode = (SValueNode *)*pNode;
if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) { if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -3587,7 +3592,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL) { if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL || node->opType == OP_TYPE_NOT_EQUAL) {
stat->scalarMode = true; stat->scalarMode = true;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }

13
source/libs/scalar/src/scalar.c
View File

@ -7,6 +7,7 @@
#include "tcommon.h" #include "tcommon.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "scalar.h" #include "scalar.h"
#include "tudf.h"
int32_t scalarGetOperatorParamNum(EOperatorType type) { int32_t scalarGetOperatorParamNum(EOperatorType type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type
@ -336,14 +337,12 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum)); SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
if (fmIsUserDefinedFunc(node->funcId)) { if (fmIsUserDefinedFunc(node->funcId)) {
#if 0
UdfcFuncHandle udfHandle = NULL; UdfcFuncHandle udfHandle = NULL;
SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle)); SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle));
code = callUdfScalarFunc(udfHandle, params, paramNum, output); code = callUdfScalarFunc(udfHandle, params, paramNum, output);
teardownUdf(udfHandle); teardownUdf(udfHandle);
SCL_ERR_JRET(code); SCL_ERR_JRET(code);
#endif
} else { } else {
SScalarFuncExecFuncs ffpSet = {0}; SScalarFuncExecFuncs ffpSet = {0};
code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
@ -505,6 +504,7 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT
} }
res->node.resType.type = TSDB_DATA_TYPE_BOOL; res->node.resType.type = TSDB_DATA_TYPE_BOOL;
res->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
res->datum.b = false; res->datum.b = false;
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
@ -520,14 +520,14 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) {
if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) { if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) {
SValueNode *valueNode = (SValueNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pLeft;
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
return sclRewriteBasedOnOptr(pNode, ctx, node->opType); return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
} }
} }
if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) {
SValueNode *valueNode = (SValueNode *)node->pRight; SValueNode *valueNode = (SValueNode *)node->pRight;
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) {
return sclRewriteBasedOnOptr(pNode, ctx, node->opType); return sclRewriteBasedOnOptr(pNode, ctx, node->opType);
} }
} }
@ -589,7 +589,10 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
if (colDataIsNull_s(output.columnData, 0)) { if (colDataIsNull_s(output.columnData, 0)) {
res->node.resType.type = TSDB_DATA_TYPE_NULL; res->node.resType.type = TSDB_DATA_TYPE_NULL;
} else { } else {
res->node.resType = node->node.resType; res->node.resType.type = output.columnData->info.type;
res->node.resType.bytes = output.columnData->info.bytes;
res->node.resType.scale = output.columnData->info.scale;
res->node.resType.precision = output.columnData->info.precision;
int32_t type = output.columnData->info.type; int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1);

19
source/libs/scalar/src/sclfunc.c
View File

@ -133,7 +133,12 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS
colDataAppendNULL(pOutputData, i); colDataAppendNULL(pOutputData, i);
continue; continue;
} }
out[i] = valFn(getValueFn(pInputData->pData, i)); double result = valFn(getValueFn(pInputData->pData, i));
if (isinf(result) || isnan(result)) {
colDataAppendNULL(pOutputData, i);
} else {
out[i] = result;
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -162,7 +167,12 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
colDataAppendNULL(pOutputData, i); colDataAppendNULL(pOutputData, i);
continue; continue;
} }
out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); double result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
if (isinf(result) || isnan(result)) {
colDataAppendNULL(pOutputData, i);
} else {
out[i] = result;
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -648,11 +658,6 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]);
if (IS_VAR_DATA_TYPE(outputType)) {
int32_t factor = (TSDB_DATA_TYPE_NCHAR == outputType) ? TSDB_NCHAR_SIZE : 1;
outputLen = outputLen * factor + VARSTR_HEADER_SIZE;
}
char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1);
char *output = outputBuf; char *output = outputBuf;

21
source/libs/scalar/src/sclvector.c
View File

@ -1058,36 +1058,36 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
_getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type);
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore continue;
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/getVectorDoubleValueFnRight(RIGHT_COL, i); / getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else { } else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pRightCol, i)) { if (colDataIsNull_s(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore continue;
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, 0) *output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
/ getVectorDoubleValueFnRight(RIGHT_COL, i); / getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check)
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else { } else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) { if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore continue;
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/ getVectorDoubleValueFnRight(RIGHT_COL, 0); / getVectorDoubleValueFnRight(RIGHT_COL, 0);
@ -1195,9 +1195,10 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore continue;
} }
*output = - getVectorDoubleValueFnLeft(LEFT_COL, i); double result = getVectorDoubleValueFnLeft(LEFT_COL, i);
*output = (result == 0) ? 0 : -result;
} }
doReleaseVec(pLeftCol, leftConvert); doReleaseVec(pLeftCol, leftConvert);

172
source/libs/stream/src/tstreamUpdate.c Normal file
View File

@ -0,0 +1,172 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tstreamUpdate.h"
#include "ttime.h"
#define DEFAULT_FALSE_POSITIVE 0.01
#define DEFAULT_BUCKET_SIZE 1024
#define ROWS_PER_MILLISECOND 1
#define MAX_NUM_SCALABLE_BF 120
#define MIN_NUM_SCALABLE_BF 10
#define DEFAULT_PREADD_BUCKET 1
#define MAX_INTERVAL MILLISECOND_PER_MINUTE
#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10)
static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
if (pInfo->numSBFs < count ) {
count = pInfo->numSBFs;
}
for (uint64_t i = 0; i < count; ++i) {
SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND,
DEFAULT_FALSE_POSITIVE);
taosArrayPush(pInfo->pTsSBFs, &tsSBF);
}
}
static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
if (count < pInfo->numSBFs - 1) {
for (uint64_t i = 0; i < count; ++i) {
SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, i);
tScalableBfDestroy(pTsSBFs);
taosArrayRemove(pInfo->pTsSBFs, i);
}
} else {
taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy);
}
pInfo->minTS += pInfo->interval * count;
}
static int64_t adjustInterval(int64_t interval, int32_t precision) {
int64_t val = interval;
if (precision != TSDB_TIME_PRECISION_MILLI) {
val = convertTimePrecision(interval, precision, TSDB_TIME_PRECISION_MILLI);
}
if (val < MIN_INTERVAL) {
val = MIN_INTERVAL;
} else if (val > MAX_INTERVAL) {
val = MAX_INTERVAL;
}
val = convertTimePrecision(val, TSDB_TIME_PRECISION_MILLI, precision);
return val;
}
SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark) {
return updateInfoInit(pInterval->interval, pInterval->precision, watermark);
}
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark) {
SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
if (pInfo == NULL) {
return NULL;
}
pInfo->pTsBuckets = NULL;
pInfo->pTsSBFs = NULL;
pInfo->minTS = -1;
pInfo->interval = adjustInterval(interval, precision);
pInfo->watermark = watermark;
uint64_t bfSize = (uint64_t)(watermark / pInfo->interval);
if (bfSize < MIN_NUM_SCALABLE_BF) {
bfSize = MIN_NUM_SCALABLE_BF;
} else if (bfSize > MAX_NUM_SCALABLE_BF) {
bfSize = MAX_NUM_SCALABLE_BF;
}
pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(SScalableBf));
if (pInfo->pTsSBFs == NULL) {
updateInfoDestroy(pInfo);
return NULL;
}
pInfo->numSBFs = bfSize;
windowSBfAdd(pInfo, bfSize);
pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY));
if (pInfo->pTsBuckets == NULL) {
updateInfoDestroy(pInfo);
return NULL;
}
TSKEY dumy = 0;
for(uint64_t i=0; i < DEFAULT_BUCKET_SIZE; ++i) {
taosArrayPush(pInfo->pTsBuckets, &dumy);
}
pInfo->numBuckets = DEFAULT_BUCKET_SIZE;
return pInfo;
}
static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) {
if (ts <= 0) {
return NULL;
}
if (pInfo->minTS < 0) {
pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval);
}
uint64_t index = (uint64_t)((ts - pInfo->minTS) / pInfo->interval);
if (index >= pInfo->numSBFs) {
uint64_t count = index + 1 - pInfo->numSBFs;
windowSBfDelete(pInfo, count);
windowSBfAdd(pInfo, count);
index = pInfo->numSBFs - 1;
}
SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index);
if (res == NULL) {
res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND,
DEFAULT_FALSE_POSITIVE);
taosArrayPush(pInfo->pTsSBFs, &res);
}
return res;
}
bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) {
int32_t res = TSDB_CODE_FAILED;
uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets;
SScalableBf* pSBf = getSBf(pInfo, ts);
// pSBf may be a null pointer
if (pSBf) {
res = tScalableBfPut(pSBf, &ts, sizeof(TSKEY));
}
TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
if (maxTs < ts ) {
taosArraySet(pInfo->pTsBuckets, index, &ts);
return false;
}
if (ts < pInfo->minTS) {
return true;
} else if (res == TSDB_CODE_SUCCESS) {
return false;
}
//check from tsdb api
return true;
}
void updateInfoDestroy(SUpdateInfo *pInfo) {
if (pInfo == NULL) {
return;
}
taosArrayDestroy(pInfo->pTsBuckets);
uint64_t size = taosArrayGetSize(pInfo->pTsSBFs);
for (uint64_t i = 0; i < size; i++) {
SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
tScalableBfDestroy(pSBF);
}
taosArrayDestroy(pInfo->pTsSBFs);
taosMemoryFree(pInfo);
}

20
source/libs/stream/test/CMakeLists.txt
View File

@ -0,0 +1,20 @@
MESSAGE(STATUS "build stream unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
# bloomFilterTest
ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp")
TARGET_LINK_LIBRARIES(
streamUpdateTest
PUBLIC os util common gtest stream
)
TARGET_INCLUDE_DIRECTORIES(
streamUpdateTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
)

103
source/libs/stream/test/tstreamUpdateTest.cpp Normal file
View File

@ -0,0 +1,103 @@
#include <gtest/gtest.h>
#include "tstreamUpdate.h"
#include "ttime.h"
using namespace std;
TEST(TD_STREAM_UPDATE_TEST, update) {
int64_t interval = 20 * 1000;
int64_t watermark = 10 * 60 * 1000;
SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
GTEST_ASSERT_EQ(isUpdated(pSU,1, 0), true);
GTEST_ASSERT_EQ(isUpdated(pSU,1, -1), true);
for(int i=0; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), false);
}
for(int i=0; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), true);
}
for(int i=0; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,i, 2), false);
}
for(int i=0; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,i, 2), true);
}
for(int i=0; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,i, 1), true);
}
for(int i=3; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,0, i), false);
}
GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023);
for(int i=3; i < 1024; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU,0, i), true);
}
GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023);
SUpdateInfo *pSU1 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
for(int i=1; i <= watermark / interval; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
GTEST_ASSERT_EQ(pSU1->minTS, interval);
GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
}
for(int i=0; i < pSU1->numSBFs; i++) {
SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU1->pTsSBFs, i);
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
GTEST_ASSERT_EQ(pBF->size, 1);
}
for(int i= watermark / interval + 1, j = 2 ; i <= watermark / interval + 10; i++,j++) {
GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
GTEST_ASSERT_EQ(pSU1->minTS, interval*j);
GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU1->pTsSBFs, pSU1->numSBFs - 1);
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
GTEST_ASSERT_EQ(pBF->size, 1);
}
for(int i= watermark / interval * 100, j = 0; j < 10; i+= (watermark / interval * 2), j++) {
GTEST_ASSERT_EQ(isUpdated(pSU1, 1, i * interval + 5), false);
GTEST_ASSERT_EQ(pSU1->minTS, (i-(pSU1->numSBFs-1))*interval);
GTEST_ASSERT_EQ(pSU1->numSBFs, watermark / interval);
}
SUpdateInfo *pSU2 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
GTEST_ASSERT_EQ(isUpdated(pSU2, 1, 1 * interval + 5), false);
GTEST_ASSERT_EQ(pSU2->minTS, interval);
for(int i= watermark / interval * 100, j = 0; j < 10; i+= (watermark / interval * 10), j++) {
GTEST_ASSERT_EQ(isUpdated(pSU2, 1, i * interval + 5), false);
GTEST_ASSERT_EQ(pSU2->minTS, (i-(pSU2->numSBFs-1))*interval);
GTEST_ASSERT_EQ(pSU2->numSBFs, watermark / interval);
GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU2->pTsBuckets,1), i * interval + 5);
}
SUpdateInfo *pSU3 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);
for(int j = 1; j < 100; j++) {
for(int i = 0; i < pSU3->numSBFs; i++) {
GTEST_ASSERT_EQ(isUpdated(pSU3, i, i * interval + 5 * j), false);
GTEST_ASSERT_EQ(pSU3->minTS, 0);
GTEST_ASSERT_EQ(pSU3->numSBFs, watermark / interval);
GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU3->pTsBuckets, i), i * interval + 5 * j);
SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU3->pTsSBFs, i);
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0);
GTEST_ASSERT_EQ(pBF->size, j);
}
}
updateInfoDestroy(pSU);
updateInfoDestroy(pSU1);
updateInfoDestroy(pSU2);
updateInfoDestroy(pSU3);
}
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

29
source/libs/transport/inc/transComm.h
View File

@ -28,6 +28,7 @@ extern "C" {
#include "taoserror.h" #include "taoserror.h"
#include "tglobal.h" #include "tglobal.h"
#include "thash.h" #include "thash.h"
#include "theap.h"
#include "tidpool.h" #include "tidpool.h"
#include "tmd5.h" #include "tmd5.h"
#include "tmempool.h" #include "tmempool.h"
@ -328,10 +329,38 @@ void transQueueClear(STransQueue* queue);
*/ */
void transQueueDestroy(STransQueue* queue); void transQueueDestroy(STransQueue* queue);
/*
* delay queue based on uv loop and uv timer, and only used in retry
*/
typedef struct STaskArg {
void* param1;
void* param2;
} STaskArg;
typedef struct SDelayTask {
void (*func)(void* arg);
void* arg;
uint64_t execTime;
HeapNode node;
} SDelayTask;
typedef struct SDelayQueue {
uv_timer_t* timer;
Heap* heap;
uv_loop_t* loop;
} SDelayQueue;
int transDQCreate(uv_loop_t* loop, SDelayQueue** queue);
void transDQDestroy(SDelayQueue* queue);
int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
/* /*
* init global func * init global func
*/ */
void transThreadOnce(); void transThreadOnce();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

43
source/libs/transport/src/transCli.c
View File

@ -60,10 +60,10 @@ typedef struct SCliThrdObj {
// msg queue // msg queue
queue msg; queue msg;
TdThreadMutex msgMtx; TdThreadMutex msgMtx;
SDelayQueue* delayQueue;
uint64_t nextTimeout; // next timeout uint64_t nextTimeout; // next timeout
void* pTransInst; // void* pTransInst; //
bool quit; bool quit;
} SCliThrdObj; } SCliThrdObj;
typedef struct SCliObj { typedef struct SCliObj {
@ -161,8 +161,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
transUnrefCliHandle(conn); \ transUnrefCliHandle(conn); \
} \ } \
if (T_REF_VAL_GET(conn) == 1) { \ if (T_REF_VAL_GET(conn) == 1) { \
SCliThrdObj* thrd = conn->hostThrd; \ transUnrefCliHandle(conn); \
addConnToPool(thrd->pool, conn); \
} \ } \
destroyCmsg(pMsg); \ destroyCmsg(pMsg); \
return; \ return; \
@ -838,12 +837,13 @@ static SCliThrdObj* createThrdObj() {
uv_loop_init(pThrd->loop); uv_loop_init(pThrd->loop);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb); pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb);
uv_timer_init(pThrd->loop, &pThrd->timer); uv_timer_init(pThrd->loop, &pThrd->timer);
pThrd->timer.data = pThrd; pThrd->timer.data = pThrd;
pThrd->pool = createConnPool(4); pThrd->pool = createConnPool(4);
transDQCreate(pThrd->loop, &pThrd->delayQueue);
pThrd->quit = false; pThrd->quit = false;
return pThrd; return pThrd;
} }
@ -851,12 +851,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) {
if (pThrd == NULL) { if (pThrd == NULL) {
return; return;
} }
taosThreadJoin(pThrd->thread, NULL); taosThreadJoin(pThrd->thread, NULL);
CLI_RELEASE_UV(pThrd->loop); CLI_RELEASE_UV(pThrd->loop);
taosThreadMutexDestroy(&pThrd->msgMtx); taosThreadMutexDestroy(&pThrd->msgMtx);
transDestroyAsyncPool(pThrd->asyncPool); transDestroyAsyncPool(pThrd->asyncPool);
uv_timer_stop(&pThrd->timer); transDQDestroy(pThrd->delayQueue);
taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd->loop);
taosMemoryFree(pThrd); taosMemoryFree(pThrd);
} }
@ -885,6 +886,16 @@ int cliRBChoseIdx(STrans* pTransInst) {
} }
return index % pTransInst->numOfThreads; return index % pTransInst->numOfThreads;
} }
static void doDelayTask(void* param) {
STaskArg* arg = param;
SCliMsg* pMsg = arg->param1;
SCliThrdObj* pThrd = arg->param2;
cliHandleReq(pMsg, pThrd);
taosMemoryFree(arg);
}
int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
SCliThrdObj* pThrd = pConn->hostThrd; SCliThrdObj* pThrd = pConn->hostThrd;
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
@ -908,20 +919,28 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
if (pCtx->retryCount < pEpSet->numOfEps) { if (pCtx->retryCount < pEpSet->numOfEps) {
pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
cliHandleReq(pMsg, pThrd);
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
cliDestroy((uv_handle_t*)pConn->stream); cliDestroy((uv_handle_t*)pConn->stream);
return -1; return -1;
} }
} else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
if (pResp->contLen == 0) { if (pResp->contLen == 0) {
pEpSet->inUse = (pEpSet->inUse++) % pEpSet->numOfEps; pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
} else { } else {
SMEpSet emsg = {0}; SMEpSet emsg = {0};
tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg);
pCtx->epSet = emsg.epSet; pCtx->epSet = emsg.epSet;
} }
cliHandleReq(pMsg, pThrd); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
// release pConn arg->param1 = pMsg;
arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
addConnToPool(pThrd, pConn); addConnToPool(pThrd, pConn);
return -1; return -1;
} }

77
source/libs/transport/src/transComm.c
View File

@ -358,4 +358,81 @@ void transQueueDestroy(STransQueue* queue) {
transQueueClear(queue); transQueueClear(queue);
taosArrayDestroy(queue->q); taosArrayDestroy(queue->q);
} }
static int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
SDelayTask* arg1 = container_of(a, SDelayTask, node);
SDelayTask* arg2 = container_of(b, SDelayTask, node);
if (arg1->execTime > arg2->execTime) {
return 0;
} else {
return 1;
}
}
static void transDQTimeout(uv_timer_t* timer) {
SDelayQueue* queue = timer->data;
tTrace("timer %p timeout", timer);
uint64_t timeout = 0;
do {
HeapNode* minNode = heapMin(queue->heap);
if (minNode == NULL) break;
SDelayTask* task = container_of(minNode, SDelayTask, node);
if (task->execTime <= taosGetTimestampMs()) {
heapRemove(queue->heap, minNode);
task->func(task->arg);
taosMemoryFree(task);
timeout = 0;
} else {
timeout = task->execTime - taosGetTimestampMs();
break;
}
} while (1);
if (timeout != 0) {
uv_timer_start(queue->timer, transDQTimeout, timeout, 0);
}
}
int transDQCreate(uv_loop_t* loop, SDelayQueue** queue) {
uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t));
uv_timer_init(loop, timer);
Heap* heap = heapCreate(timeCompare);
SDelayQueue* q = taosMemoryCalloc(1, sizeof(SDelayQueue));
q->heap = heap;
q->timer = timer;
q->loop = loop;
q->timer->data = q;
*queue = q;
return 0;
}
void transDQDestroy(SDelayQueue* queue) {
taosMemoryFree(queue->timer);
while (heapSize(queue->heap) > 0) {
HeapNode* minNode = heapMin(queue->heap);
if (minNode == NULL) {
return;
}
heapRemove(queue->heap, minNode);
SDelayTask* task = container_of(minNode, SDelayTask, node);
taosMemoryFree(task);
}
heapDestroy(queue->heap);
taosMemoryFree(queue);
}
int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) {
SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask));
task->func = func;
task->arg = arg;
task->execTime = taosGetTimestampMs() + timeoutMs;
tTrace("timer %p put task into queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs);
heapInsert(queue->heap, &task->node);
uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0);
return 0;
}
#endif #endif

4
source/os/src/osDir.c
View File

@ -72,8 +72,8 @@ void taosRemoveDir(const char *dirname) {
while ((de = taosReadDir(pDir)) != NULL) { while ((de = taosReadDir(pDir)) != NULL) {
if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
char filename[1024]; char filename[1024] = {0};
snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de));
if (taosDirEntryIsDir(de)) { if (taosDirEntryIsDir(de)) {
taosRemoveDir(filename); taosRemoveDir(filename);
} else { } else {

14
source/util/src/tarray.c
View File

@ -318,6 +318,20 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
pArray->size = 0; pArray->size = 0;
} }
void taosArrayClearP(SArray* pArray, FDelete fp) {
if (pArray == NULL) return;
if (fp == NULL) {
pArray->size = 0;
return;
}
for (int32_t i = 0; i < pArray->size; ++i) {
fp(*(void**)TARRAY_GET_ELEM(pArray, i));
}
pArray->size = 0;
}
void* taosArrayDestroy(SArray* pArray) { void* taosArrayDestroy(SArray* pArray) {
if (pArray) { if (pArray) {
taosMemoryFree(pArray->pData); taosMemoryFree(pArray->pData);

117
source/util/src/tbloomfilter.c Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tbloomfilter.h"
#include "taos.h"
#include "taoserror.h"
#define UNIT_NUM_BITS 64
#define UNIT_ADDR_NUM_BITS 6
static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t mask = 1 << (index % UNIT_NUM_BITS);
uint64_t old = buf[unitIndex];
buf[unitIndex] |= mask;
return buf[unitIndex] != old;
}
static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t mask = 1 << (index % UNIT_NUM_BITS);
return buf[unitIndex] & mask;
}
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) {
if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
return NULL;
}
SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
if (pBF == NULL) {
return NULL;
}
pBF->expectedEntries = expectedEntries;
pBF->errorRate = errorRate;
double lnRate = fabs(log(errorRate));
// ln(2)^2 = 0.480453013918201
// m = - n * ln(P) / ( ln(2) )^2
// m is the size of bloom filter, n is expected entries, P is false positive probability
pBF->numUnits = (uint64_t) ceil(expectedEntries * lnRate / 0.480453013918201 / UNIT_NUM_BITS);
pBF->numBits = pBF->numUnits * 64;
pBF->size = 0;
// ln(2) = 0.693147180559945
pBF->hashFunctions = (uint32_t) ceil(lnRate / 0.693147180559945);
pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR);
pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t));
if (pBF->buffer == NULL) {
tBloomFilterDestroy(pBF);
return NULL;
}
return pBF;
}
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
ASSERT(!tBloomFilterIsFull(pBF));
uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
bool hasChange = false;
const register uint64_t size = pBF->numBits;
uint64_t cbHash = h1;
for (uint64_t i = 0; i < pBF->hashFunctions; ++i) {
hasChange |= setBit(pBF->buffer, cbHash % size);
cbHash += h2;
}
if (hasChange) {
pBF->size++;
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_FAILED;
}
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
uint32_t len) {
uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
const register uint64_t size = pBF->numBits;
uint64_t cbHash = h1;
for (uint64_t i = 0; i < pBF->hashFunctions; ++i) {
if (!getBit(pBF->buffer, cbHash % size)) {
return TSDB_CODE_SUCCESS;
}
cbHash += h2;
}
return TSDB_CODE_FAILED;
}
void tBloomFilterDestroy(SBloomFilter *pBF) {
if (pBF == NULL) {
return;
}
taosMemoryFree(pBF->buffer);
taosMemoryFree(pBF);
}
void tBloomFilterDump(const struct SBloomFilter *pBF) {
// ToDo
}
bool tBloomFilterIsFull(const SBloomFilter *pBF) {
return pBF->size >= pBF->expectedEntries;
}

106
source/util/src/tscalablebf.c Normal file
View File

@ -0,0 +1,106 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tscalablebf.h"
#include "taoserror.h"
#define DEFAULT_GROWTH 2
#define DEFAULT_TIGHTENING_RATIO 0.5
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries,
double errorRate);
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
const uint32_t defaultSize = 8;
if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
return NULL;
}
SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
if (pSBf == NULL) {
return NULL;
}
pSBf->numBits = 0;
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL ) {
tScalableBfDestroy(pSBf);
return NULL;
}
pSBf->growth = DEFAULT_GROWTH;
return pSBf;
}
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t size = taosArrayGetSize(pSBf->bfArray);
for (int32_t i = size - 2; i >= 0; --i) {
if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i),
keyBuf, len) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
ASSERT(pNormalBf);
if (tBloomFilterIsFull(pNormalBf)) {
pNormalBf = tScalableBfAddFilter(pSBf,
pNormalBf->expectedEntries * pSBf->growth,
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
if (pNormalBf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return tBloomFilterPut(pNormalBf, keyBuf, len);
}
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
uint32_t len) {
int32_t size = taosArrayGetSize(pSBf->bfArray);
for (int32_t i = size - 1; i >= 0; --i) {
if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i),
keyBuf, len) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
return TSDB_CODE_SUCCESS;
}
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries,
double errorRate) {
SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate);
if (pNormalBf == NULL) {
return NULL;
}
if(taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) {
tBloomFilterDestroy(pNormalBf);
return NULL;
}
pSBf->numBits += pNormalBf->numBits;
return pNormalBf;
}
void tScalableBfDestroy(SScalableBf *pSBf) {
if (pSBf == NULL) {
return;
}
if (pSBf->bfArray != NULL) {
taosArrayDestroyP(pSBf->bfArray, (FDelete)tBloomFilterDestroy);
}
taosMemoryFree(pSBf);
}
void tScalableBfDump(const SScalableBf *pSBf) {
// Todo;
}

8
source/util/test/CMakeLists.txt
View File

@ -59,4 +59,12 @@ target_link_libraries(cfgTest os util gtest_main)
add_test( add_test(
NAME cfgTest NAME cfgTest
COMMAND cfgTest COMMAND cfgTest
)
# bloomFilterTest
add_executable(bloomFilterTest "bloomFilterTest.cpp")
target_link_libraries(bloomFilterTest os util gtest_main)
add_test(
NAME bloomFilterTest
COMMAND bloomFilterTest
) )

140
source/util/test/bloomFilterTest.cpp Normal file
View File

@ -0,0 +1,140 @@
#include <gtest/gtest.h>
#include "tscalablebf.h"
#include "taoserror.h"
using namespace std;
TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
int64_t ts1 = 1650803518000;
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01));
SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005);
GTEST_ASSERT_EQ(pBF1->numBits, 1152);
GTEST_ASSERT_EQ(pBF1->numUnits, 1152/64);
int64_t count = 0;
for(int64_t i = 0; count < 100; i++) {
int64_t ts = i + ts1;
if(tBloomFilterPut(pBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
count++;
}
}
ASSERT_TRUE(tBloomFilterIsFull(pBF1));
SBloomFilter *pBF2 = tBloomFilterInit(1000*10000, 0.1);
GTEST_ASSERT_EQ(pBF2->numBits, 47925312);
GTEST_ASSERT_EQ(pBF2->numUnits, 47925312/64);
SBloomFilter *pBF3 = tBloomFilterInit(10000*10000, 0.001);
GTEST_ASSERT_EQ(pBF3->numBits, 1437758784);
GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784/64);
int64_t size = 10000;
SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001);
for(int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
}
ASSERT_TRUE(!tBloomFilterIsFull(pBF4));
for(int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
}
for(int64_t i = 2000; i < 3000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
}
tBloomFilterDestroy(pBF1);
tBloomFilterDestroy(pBF2);
tBloomFilterDestroy(pBF3);
tBloomFilterDestroy(pBF4);
}
TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int64_t ts1 = 1650803518000;
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01));
SScalableBf *pSBF1 = tScalableBfInit(100, 0.01);
GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
int64_t count = 0;
int64_t index = 0;
for( ; count < 100; index++) {
int64_t ts = index + ts1;
if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
count++;
}
}
GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
for( ; count < 300; index++) {
int64_t ts = index + ts1;
if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
count++;
}
}
GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496);
for( ; count < 700; index++) {
int64_t ts = index + ts1;
if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
count++;
}
}
GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496+5568);
for( ; count < 1500; index++) {
int64_t ts = index + ts1;
if(tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS ) {
count++;
}
}
GTEST_ASSERT_EQ(pSBF1->numBits, 1152+2496+5568+12288);
int32_t aSize = taosArrayGetSize(pSBF1->bfArray);
int64_t totalBits = 0;
for(int64_t i = 0; i < aSize; i++) {
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i);
ASSERT_TRUE(tBloomFilterIsFull(pBF));
totalBits += pBF->numBits;
}
GTEST_ASSERT_EQ(pSBF1->numBits, totalBits);
for(int64_t i = 0; i < index; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF1, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
}
int64_t size = 10000;
SScalableBf *pSBF4 = tScalableBfInit(size, 0.001);
for(int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
}
for(int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_FAILED);
}
for(int64_t i = 2000; i < 3000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfNoContain(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
}
tScalableBfDestroy(pSBF1);
tScalableBfDestroy(pSBF4);
}

2
tests/pytest/util/sql.py
View File

@ -16,6 +16,7 @@ import os
import time import time
import datetime import datetime
import inspect import inspect
import traceback
import psutil import psutil
import shutil import shutil
import pandas as pd import pandas as pd
@ -88,6 +89,7 @@ class TDSql:
caller = inspect.getframeinfo(inspect.stack()[1][0]) caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e)) args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.notice("%s(%d) failed: sql:%s, %s" % args) tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
traceback.print_exc()
raise Exception(repr(e)) raise Exception(repr(e))
if row_tag: if row_tag:
return self.queryResult return self.queryResult

841
tests/script/api/batchprepare.c
View File

File diff suppressed because it is too large Load Diff

2
tests/script/jenkins/basic.txt
View File

@ -81,7 +81,7 @@
./test.sh -f tsim/insert/backquote.sim -m ./test.sh -f tsim/insert/backquote.sim -m
./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m
./test.sh -f tsim/query/interval-offset.sim -m ./test.sh -f tsim/query/interval-offset.sim -m
#./test.sh -f tsim/tmq/basic1.sim -m ./test.sh -f tsim/tmq/basic3.sim -m
./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/stable/vnode3.sim -m
./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/qnode/basic1.sim -m
./test.sh -f tsim/mnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m

Some files were not shown because too many files have changed in this diff Show More