xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add language clause for create function

This commit is contained in:
Xiaoyu Wang 2023-03-04 11:08:50 +08:00
parent 42d2c15f19
commit 599967d39f
10 changed files with 3118 additions and 2995 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
include/common/ttokendef.h
View File

@ -205,144 +205,145 @@
#define TK_OUTPUTTYPE 187
#define TK_AGGREGATE 188
#define TK_BUFSIZE 189
#define TK_STREAM 190
#define TK_INTO 191
#define TK_TRIGGER 192
#define TK_AT_ONCE 193
#define TK_WINDOW_CLOSE 194
#define TK_IGNORE 195
#define TK_EXPIRED 196
#define TK_FILL_HISTORY 197
#define TK_UPDATE 198
#define TK_SUBTABLE 199
#define TK_KILL 200
#define TK_CONNECTION 201
#define TK_TRANSACTION 202
#define TK_BALANCE 203
#define TK_VGROUP 204
#define TK_MERGE 205
#define TK_REDISTRIBUTE 206
#define TK_SPLIT 207
#define TK_DELETE 208
#define TK_INSERT 209
#define TK_NULL 210
#define TK_NK_QUESTION 211
#define TK_NK_ARROW 212
#define TK_ROWTS 213
#define TK_QSTART 214
#define TK_QEND 215
#define TK_QDURATION 216
#define TK_WSTART 217
#define TK_WEND 218
#define TK_WDURATION 219
#define TK_IROWTS 220
#define TK_ISFILLED 221
#define TK_CAST 222
#define TK_NOW 223
#define TK_TODAY 224
#define TK_TIMEZONE 225
#define TK_CLIENT_VERSION 226
#define TK_SERVER_VERSION 227
#define TK_SERVER_STATUS 228
#define TK_CURRENT_USER 229
#define TK_CASE 230
#define TK_END 231
#define TK_WHEN 232
#define TK_THEN 233
#define TK_ELSE 234
#define TK_BETWEEN 235
#define TK_IS 236
#define TK_NK_LT 237
#define TK_NK_GT 238
#define TK_NK_LE 239
#define TK_NK_GE 240
#define TK_NK_NE 241
#define TK_MATCH 242
#define TK_NMATCH 243
#define TK_CONTAINS 244
#define TK_IN 245
#define TK_JOIN 246
#define TK_INNER 247
#define TK_SELECT 248
#define TK_DISTINCT 249
#define TK_WHERE 250
#define TK_PARTITION 251
#define TK_BY 252
#define TK_SESSION 253
#define TK_STATE_WINDOW 254
#define TK_EVENT_WINDOW 255
#define TK_START 256
#define TK_SLIDING 257
#define TK_FILL 258
#define TK_VALUE 259
#define TK_VALUE_F 260
#define TK_NONE 261
#define TK_PREV 262
#define TK_NULL_F 263
#define TK_LINEAR 264
#define TK_NEXT 265
#define TK_HAVING 266
#define TK_RANGE 267
#define TK_EVERY 268
#define TK_ORDER 269
#define TK_SLIMIT 270
#define TK_SOFFSET 271
#define TK_LIMIT 272
#define TK_OFFSET 273
#define TK_ASC 274
#define TK_NULLS 275
#define TK_ABORT 276
#define TK_AFTER 277
#define TK_ATTACH 278
#define TK_BEFORE 279
#define TK_BEGIN 280
#define TK_BITAND 281
#define TK_BITNOT 282
#define TK_BITOR 283
#define TK_BLOCKS 284
#define TK_CHANGE 285
#define TK_COMMA 286
#define TK_CONCAT 287
#define TK_CONFLICT 288
#define TK_COPY 289
#define TK_DEFERRED 290
#define TK_DELIMITERS 291
#define TK_DETACH 292
#define TK_DIVIDE 293
#define TK_DOT 294
#define TK_EACH 295
#define TK_FAIL 296
#define TK_FILE 297
#define TK_FOR 298
#define TK_GLOB 299
#define TK_ID 300
#define TK_IMMEDIATE 301
#define TK_IMPORT 302
#define TK_INITIALLY 303
#define TK_INSTEAD 304
#define TK_ISNULL 305
#define TK_KEY 306
#define TK_MODULES 307
#define TK_NK_BITNOT 308
#define TK_NK_SEMI 309
#define TK_NOTNULL 310
#define TK_OF 311
#define TK_PLUS 312
#define TK_PRIVILEGE 313
#define TK_RAISE 314
#define TK_REPLACE 315
#define TK_RESTRICT 316
#define TK_ROW 317
#define TK_SEMI 318
#define TK_STAR 319
#define TK_STATEMENT 320
#define TK_STRICT 321
#define TK_STRING 322
#define TK_TIMES 323
#define TK_VALUES 324
#define TK_VARIABLE 325
#define TK_VIEW 326
#define TK_WAL 327
#define TK_LANGUAGE 190
#define TK_STREAM 191
#define TK_INTO 192
#define TK_TRIGGER 193
#define TK_AT_ONCE 194
#define TK_WINDOW_CLOSE 195
#define TK_IGNORE 196
#define TK_EXPIRED 197
#define TK_FILL_HISTORY 198
#define TK_UPDATE 199
#define TK_SUBTABLE 200
#define TK_KILL 201
#define TK_CONNECTION 202
#define TK_TRANSACTION 203
#define TK_BALANCE 204
#define TK_VGROUP 205
#define TK_MERGE 206
#define TK_REDISTRIBUTE 207
#define TK_SPLIT 208
#define TK_DELETE 209
#define TK_INSERT 210
#define TK_NULL 211
#define TK_NK_QUESTION 212
#define TK_NK_ARROW 213
#define TK_ROWTS 214
#define TK_QSTART 215
#define TK_QEND 216
#define TK_QDURATION 217
#define TK_WSTART 218
#define TK_WEND 219
#define TK_WDURATION 220
#define TK_IROWTS 221
#define TK_ISFILLED 222
#define TK_CAST 223
#define TK_NOW 224
#define TK_TODAY 225
#define TK_TIMEZONE 226
#define TK_CLIENT_VERSION 227
#define TK_SERVER_VERSION 228
#define TK_SERVER_STATUS 229
#define TK_CURRENT_USER 230
#define TK_CASE 231
#define TK_END 232
#define TK_WHEN 233
#define TK_THEN 234
#define TK_ELSE 235
#define TK_BETWEEN 236
#define TK_IS 237
#define TK_NK_LT 238
#define TK_NK_GT 239
#define TK_NK_LE 240
#define TK_NK_GE 241
#define TK_NK_NE 242
#define TK_MATCH 243
#define TK_NMATCH 244
#define TK_CONTAINS 245
#define TK_IN 246
#define TK_JOIN 247
#define TK_INNER 248
#define TK_SELECT 249
#define TK_DISTINCT 250
#define TK_WHERE 251
#define TK_PARTITION 252
#define TK_BY 253
#define TK_SESSION 254
#define TK_STATE_WINDOW 255
#define TK_EVENT_WINDOW 256
#define TK_START 257
#define TK_SLIDING 258
#define TK_FILL 259
#define TK_VALUE 260
#define TK_VALUE_F 261
#define TK_NONE 262
#define TK_PREV 263
#define TK_NULL_F 264
#define TK_LINEAR 265
#define TK_NEXT 266
#define TK_HAVING 267
#define TK_RANGE 268
#define TK_EVERY 269
#define TK_ORDER 270
#define TK_SLIMIT 271
#define TK_SOFFSET 272
#define TK_LIMIT 273
#define TK_OFFSET 274
#define TK_ASC 275
#define TK_NULLS 276
#define TK_ABORT 277
#define TK_AFTER 278
#define TK_ATTACH 279
#define TK_BEFORE 280
#define TK_BEGIN 281
#define TK_BITAND 282
#define TK_BITNOT 283
#define TK_BITOR 284
#define TK_BLOCKS 285
#define TK_CHANGE 286
#define TK_COMMA 287
#define TK_CONCAT 288
#define TK_CONFLICT 289
#define TK_COPY 290
#define TK_DEFERRED 291
#define TK_DELIMITERS 292
#define TK_DETACH 293
#define TK_DIVIDE 294
#define TK_DOT 295
#define TK_EACH 296
#define TK_FAIL 297
#define TK_FILE 298
#define TK_FOR 299
#define TK_GLOB 300
#define TK_ID 301
#define TK_IMMEDIATE 302
#define TK_IMPORT 303
#define TK_INITIALLY 304
#define TK_INSTEAD 305
#define TK_ISNULL 306
#define TK_KEY 307
#define TK_MODULES 308
#define TK_NK_BITNOT 309
#define TK_NK_SEMI 310
#define TK_NOTNULL 311
#define TK_OF 312
#define TK_PLUS 313
#define TK_PRIVILEGE 314
#define TK_RAISE 315
#define TK_REPLACE 316
#define TK_RESTRICT 317
#define TK_ROW 318
#define TK_SEMI 319
#define TK_STAR 320
#define TK_STATEMENT 321
#define TK_STRICT 322
#define TK_STRING 323
#define TK_TIMES 324
#define TK_VALUES 325
#define TK_VARIABLE 326
#define TK_VIEW 327
#define TK_WAL 328
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

1
include/libs/nodes/cmdnodes.h
View File

@ -430,6 +430,7 @@ typedef struct SCreateFunctionStmt {
char libraryPath[PATH_MAX];
SDataType outputDt;
int32_t bufSize;
int8_t language;
} SCreateFunctionStmt;
typedef struct SDropFunctionStmt {

2
include/util/tdef.h
View File

@ -206,7 +206,7 @@ typedef enum ELogicConditionType {
#define TSDB_FUNC_TYPE_SCALAR 1
#define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_SCRIPT_BIN_LIB 0
#define TSDB_FUNC_SCRIPT_LUA 1
#define TSDB_FUNC_SCRIPT_PYTHON 1
#define TSDB_FUNC_MAX_RETRIEVE 1024
#define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0'

2
source/libs/parser/inc/parAst.h
View File

@ -212,7 +212,7 @@ SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions,
SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName,
const SToken* pLibPath, SDataType dataType, int32_t bufSize);
const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage);
SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);
SNode* createStreamOptions(SAstCreateContext* pCxt);
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,

7
source/libs/parser/inc/sql.y
View File

@ -531,7 +531,7 @@ explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C).
/************************************************ create/drop function ************************************************/
cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B)
AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E); }
AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E) language_opt(G). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E, &G); }
cmd ::= DROP FUNCTION exists_opt(B) function_name(A). { pCxt->pRootNode = createDropFunctionStmt(pCxt, B, &A); }
%type agg_func_opt { bool }
@ -544,6 +544,11 @@ agg_func_opt(A) ::= AGGREGATE.
bufsize_opt(A) ::= . { A = 0; }
bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); }
%type language_opt { SToken }
%destructor language_opt { }
language_opt(A) ::= . { A = nil_token; }
language_opt(A) ::= LANGUAGE NK_STRING(B). { A = B; }
/************************************************ create/drop stream **************************************************/
cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO
full_table_name(C) col_list_opt(H) tag_def_or_ref_opt(F) subtable_opt(G)

19
source/libs/parser/src/parAstCreater.c
View File

@ -1779,13 +1779,29 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) {
return pStmt;
}
static int32_t convertUdfLanguageType(SAstCreateContext* pCxt, const SToken* pLanguageToken, int8_t* pLanguage) {
if (TK_NK_NIL == pLanguageToken->type || 0 == strncasecmp(pLanguageToken->z + 1, "c", pLanguageToken->n - 2)) {
*pLanguage = TSDB_FUNC_SCRIPT_BIN_LIB;
} else if (0 == strncasecmp(pLanguageToken->z + 1, "python", pLanguageToken->n - 2)) {
*pLanguage = TSDB_FUNC_SCRIPT_PYTHON;
} else {
pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
"udf programming language supports c and python");
}
return pCxt->errCode;
}
SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName,
const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage) {
CHECK_PARSER_STATUS(pCxt);
if (pLibPath->n <= 2) {
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
return NULL;
}
int8_t language = 0;
if (TSDB_CODE_SUCCESS != convertUdfLanguageType(pCxt, pLanguage, &language)) {
return NULL;
}
SCreateFunctionStmt* pStmt = (SCreateFunctionStmt*)nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreExists = ignoreExists;
@ -1794,6 +1810,7 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool
COPY_STRING_FORM_STR_TOKEN(pStmt->libraryPath, pLibPath);
pStmt->outputDt = dataType;
pStmt->bufSize = bufSize;
pStmt->language = language;
return (SNode*)pStmt;
}

1
source/libs/parser/src/parTokenizer.c
View File

@ -124,6 +124,7 @@ static SKeyword keywordTable[] = {
{"JSON", TK_JSON},
{"KEEP", TK_KEEP},
{"KILL", TK_KILL},
{"LANGUAGE", TK_LANGUAGE},
{"LAST", TK_LAST},
{"LAST_ROW", TK_LAST_ROW},
{"LICENCES", TK_LICENCES},

2
source/libs/parser/src/parTranslater.c
View File

@ -6358,7 +6358,7 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS
strcpy(req.name, pStmt->funcName);
req.igExists = pStmt->ignoreExists;
req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR;
req.scriptType = TSDB_FUNC_SCRIPT_BIN_LIB;
req.scriptType = pStmt->language;
req.outputType = pStmt->outputDt.type;
req.outputLen = pStmt->outputDt.bytes;
req.bufSize = pStmt->bufSize;

5772
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

30
source/libs/parser/test/parInitialCTest.cpp
View File

@ -13,6 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <fstream>
#include "parTestUtil.h"
using namespace std;
@ -381,7 +383,8 @@ TEST_F(ParserInitialCTest, createDnode) {
}
/*
* CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value]
* CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name
* AS library_path OUTPUTTYPE type_name [BUFSIZE value] [LANGUAGE value]
*/
TEST_F(ParserInitialCTest, createFunction) {
useDb("root", "test");
@ -389,12 +392,13 @@ TEST_F(ParserInitialCTest, createFunction) {
SCreateFuncReq expect = {0};
auto setCreateFuncReq = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0,
int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0) {
int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0,
int8_t language = TSDB_FUNC_SCRIPT_BIN_LIB) {
memset(&expect, 0, sizeof(SCreateFuncReq));
strcpy(expect.name, pUdfName);
expect.igExists = igExists;
expect.funcType = funcType;
expect.scriptType = TSDB_FUNC_SCRIPT_BIN_LIB;
expect.scriptType = language;
expect.outputType = outputType;
expect.outputLen = outputBytes > 0 ? outputBytes : tDataTypes[outputType].bytes;
expect.bufSize = bufSize;
@ -412,13 +416,25 @@ TEST_F(ParserInitialCTest, createFunction) {
ASSERT_EQ(req.outputType, expect.outputType);
ASSERT_EQ(req.outputLen, expect.outputLen);
ASSERT_EQ(req.bufSize, expect.bufSize);
tFreeSCreateFuncReq(&req);
});
setCreateFuncReq("udf1", TSDB_DATA_TYPE_INT);
// run("CREATE FUNCTION udf1 AS './build/lib/libudf1.so' OUTPUTTYPE INT");
struct udfFile {
udfFile(const std::string& filename) : path_(filename) {
std::ofstream file(filename, std::ios::binary);
file << 123 << "abc" << '\n';
file.close();
}
~udfFile() { remove(path_.c_str()); }
std::string path_;
} udffile("udf");
setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8);
// run("CREATE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS './build/lib/libudf2.so' OUTPUTTYPE DOUBLE BUFSIZE 8");
setCreateFuncReq("udf1", TSDB_DATA_TYPE_INT);
run("CREATE FUNCTION udf1 AS 'udf' OUTPUTTYPE INT");
setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8, TSDB_FUNC_SCRIPT_PYTHON);
run("CREATE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS 'udf' OUTPUTTYPE DOUBLE BUFSIZE 8 LANGUAGE 'python'");
}
/*