xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #7523 from taosdata/feature/szhou/tagregex 

[TD-6145]<feature>:tagfilter-add parser support
This commit is contained in:
dapan1121 2021-09-06 15:49:24 +08:00 committed by GitHub
parent a47f664cf2 f842633b17
commit fb4c7bfd11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 2522 additions and 1495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/client/src/tscSQLParser.c
View File

@ -22,6 +22,7 @@
#include <qSqlparser.h> #include <qSqlparser.h>
#include "os.h" #include "os.h"
#include "regex.h"
#include "qPlan.h" #include "qPlan.h"
#include "qSqlparser.h" #include "qSqlparser.h"
#include "qTableMeta.h" #include "qTableMeta.h"
@ -278,6 +279,8 @@ static uint8_t convertRelationalOperator(SStrToken *pToken) {
return TSDB_BINARY_OP_REMAINDER; return TSDB_BINARY_OP_REMAINDER;
case TK_LIKE: case TK_LIKE:
return TSDB_RELATION_LIKE; return TSDB_RELATION_LIKE;
case TK_MATCH:
return TSDB_RELATION_MATCH;
case TK_ISNULL: case TK_ISNULL:
return TSDB_RELATION_ISNULL; return TSDB_RELATION_ISNULL;
case TK_NOTNULL: case TK_NOTNULL:
@ -3776,6 +3779,9 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
case TK_LIKE: case TK_LIKE:
pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE;
break; break;
case TK_MATCH:
pColumnFilter->lowerRelOptr = TSDB_RELATION_MATCH;
break;
case TK_ISNULL: case TK_ISNULL:
pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL; pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL;
break; break;
@ -3839,9 +3845,15 @@ static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) { static int32_t tablenameCondToString(tSqlExpr* pExpr, uint32_t opToken, SStringBuilder* sb) {
assert(opToken == TK_LIKE || opToken == TK_MATCH);
if (opToken == TK_LIKE) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
taosStringBuilderAppendString(sb, pExpr->value.pz); taosStringBuilderAppendString(sb, pExpr->value.pz);
} else if (opToken == TK_MATCH) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN);
taosStringBuilderAppendString(sb, pExpr->value.pz);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3862,7 +3874,7 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
int32_t ret = 0; int32_t ret = 0;
const char* msg1 = "non binary column not support like operator"; const char* msg1 = "non binary column not support like/match operator";
const char* msg2 = "binary column not support this operator"; const char* msg2 = "binary column not support this operator";
const char* msg3 = "bool column not support this operator"; const char* msg3 = "bool column not support this operator";
const char* msg4 = "primary key not support this operator"; const char* msg4 = "primary key not support this operator";
@ -3890,12 +3902,13 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
&& pExpr->tokenId != TK_ISNULL && pExpr->tokenId != TK_ISNULL
&& pExpr->tokenId != TK_NOTNULL && pExpr->tokenId != TK_NOTNULL
&& pExpr->tokenId != TK_LIKE && pExpr->tokenId != TK_LIKE
&& pExpr->tokenId != TK_MATCH
&& pExpr->tokenId != TK_IN) { && pExpr->tokenId != TK_IN) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
goto _err_ret; goto _err_ret;
} }
} else { } else {
if (pExpr->tokenId == TK_LIKE) { if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
goto _err_ret; goto _err_ret;
} }
@ -3943,12 +3956,12 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr*
if (pTableCond->tokenId == TK_IN) { if (pTableCond->tokenId == TK_IN) {
ret = tablenameListToString(pRight, sb); ret = tablenameListToString(pRight, sb);
} else if (pTableCond->tokenId == TK_LIKE) { } else if (pTableCond->tokenId == TK_LIKE || pTableCond->tokenId == TK_MATCH) {
if (pRight->tokenId != TK_STRING) { if (pRight->tokenId != TK_STRING) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }
ret = tablenameCondToString(pRight, sb); ret = tablenameCondToString(pRight, pTableCond->tokenId, sb);
} }
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
@ -4397,7 +4410,7 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr
} }
static bool validTableNameOptr(tSqlExpr* pExpr) { static bool validTableNameOptr(tSqlExpr* pExpr) {
const char nameFilterOptr[] = {TK_IN, TK_LIKE}; const char nameFilterOptr[] = {TK_IN, TK_LIKE, TK_MATCH};
for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) {
if (pExpr->tokenId == nameFilterOptr[i]) { if (pExpr->tokenId == nameFilterOptr[i]) {
@ -4489,6 +4502,45 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// check for match expression
static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
const char* msg1 = "regular expression string should be less than %d characters";
const char* msg2 = "illegal column type for match";
const char* msg3 = "invalid regular expression";
tSqlExpr* pLeft = pExpr->pLeft;
tSqlExpr* pRight = pExpr->pRight;
if (pExpr->tokenId == TK_MATCH) {
if (pRight->value.nLen > tsMaxRegexStringLen) {
char tmp[64] = {0};
sprintf(tmp, msg1, tsMaxRegexStringLen);
return invalidOperationMsg(msgBuf, tmp);
}
SSchema* pSchema = tscGetTableSchema(pTableMeta);
if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) {
return invalidOperationMsg(msgBuf, msg2);
}
int errCode = 0;
regex_t regex;
char regErrBuf[256] = {0};
const char* pattern = pRight->value.pz;
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, regErrBuf, sizeof(regErrBuf));
tscError("Failed to compile regex pattern %s. reason %s", pattern, regErrBuf);
return invalidOperationMsg(msgBuf, msg3);
}
regfree(&regex);
}
return TSDB_CODE_SUCCESS;
}
int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) { int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) {
tSqlExpr* left = tSqlExprClone(expr); tSqlExpr* left = tSqlExprClone(expr);
tSqlExpr* right = expr; tSqlExpr* right = expr;
@ -4540,6 +4592,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
return code; return code;
} }
// validate the match expression
code = validateMatchExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range
if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
@ -4867,16 +4925,16 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
STagCond* pTagCond = &pQueryInfo->tagCond; STagCond* pTagCond = &pQueryInfo->tagCond;
pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid;
assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_IN); assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_IN);
if (pExpr->tokenId == TK_LIKE) { if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
char* str = taosStringBuilderGetResult(sb, NULL); char* str = taosStringBuilderGetResult(sb, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str); pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} } else {
SStringBuilder sb1;
SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); memset(&sb1, 0, sizeof(sb1));
taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
// remove the duplicated input table names // remove the duplicated input table names
@ -4896,7 +4954,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
char name[TSDB_DB_NAME_LEN] = {0}; char name[TSDB_DB_NAME_LEN] = {0};
tNameGetDbName(&pTableMetaInfo->name, name); tNameGetDbName(&pTableMetaInfo->name, name);
SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; SStrToken dbToken = {.type = TK_STRING, .z = name, .n = (uint32_t)strlen(name)};
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
if (i >= 1) { if (i >= 1) {
@ -4921,11 +4979,12 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
char* str = taosStringBuilderGetResult(&sb1, NULL); char* str = taosStringBuilderGetResult(&sb1, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str); pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); pQueryInfo->tagCond.tbnameCond.len = (int32_t)strlen(str);
taosStringBuilderDestroy(&sb1); taosStringBuilderDestroy(&sb1);
tfree(segments); tfree(segments);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
}
} }
int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) { int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) {
@ -8112,7 +8171,7 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect
} }
static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) { static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) {
const char* msg1 = "non binary column not support like operator"; const char* msg1 = "non binary column not support like/match operator";
const char* msg2 = "invalid operator for binary column in having clause"; const char* msg2 = "invalid operator for binary column in having clause";
const char* msg3 = "invalid operator for bool column in having clause"; const char* msg3 = "invalid operator for bool column in having clause";
@ -8164,11 +8223,12 @@ static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, S
&& pExpr->tokenId != TK_ISNULL && pExpr->tokenId != TK_ISNULL
&& pExpr->tokenId != TK_NOTNULL && pExpr->tokenId != TK_NOTNULL
&& pExpr->tokenId != TK_LIKE && pExpr->tokenId != TK_LIKE
&& pExpr->tokenId != TK_MATCH
) { ) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
} }
} else { } else {
if (pExpr->tokenId == TK_LIKE) { if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }

2
src/common/inc/texpr.h
View File

@ -33,9 +33,11 @@ struct SSchema;
#define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_IN "IN|"
#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|"
#define QUERY_COND_REL_PREFIX_MATCH "MATCH|"
#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_IN_LEN 3
#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5
#define QUERY_COND_REL_PREFIX_MATCH_LEN 6
typedef bool (*__result_filter_fn_t)(const void *, void *); typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *); typedef void (*__do_filter_suppl_fn_t)(void *, void *);

1
src/common/inc/tglobal.h
View File

@ -74,6 +74,7 @@ extern int8_t tsKeepOriginalColumnName;
// client // client
extern int32_t tsMaxSQLStringLen; extern int32_t tsMaxSQLStringLen;
extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;
extern int8_t tsTscEnableRecordSql; extern int8_t tsTscEnableRecordSql;
extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;

11
src/common/src/texpr.c
View File

@ -430,6 +430,17 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY; pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len; pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_MATCH;
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1;
pVal->pz = exception_malloc(len);
memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN, len);
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE; right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_IN; expr->_node.optr = TSDB_RELATION_IN;

2
src/common/src/tglobal.c
View File

@ -85,6 +85,8 @@ int32_t tsCompressColData = -1;
// client // client
int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN;
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
int8_t tsTscEnableRecordSql = 0; int8_t tsTscEnableRecordSql = 0;
// the maximum number of results for projection query on super table that are returned from // the maximum number of results for projection query on super table that are returned from

2
src/inc/taosdef.h
View File

@ -164,6 +164,8 @@ do { \
#define TSDB_RELATION_OR 12 #define TSDB_RELATION_OR 12
#define TSDB_RELATION_NOT 13 #define TSDB_RELATION_NOT 13
#define TSDB_RELATION_MATCH 14
#define TSDB_BINARY_OP_ADD 30 #define TSDB_BINARY_OP_ADD 30
#define TSDB_BINARY_OP_SUBTRACT 31 #define TSDB_BINARY_OP_SUBTRACT 31
#define TSDB_BINARY_OP_MULTIPLY 32 #define TSDB_BINARY_OP_MULTIPLY 32

308
src/inc/ttokendef.h
View File

@ -37,160 +37,160 @@
#define TK_NOTNULL 19 #define TK_NOTNULL 19
#define TK_IS 20 #define TK_IS 20
#define TK_LIKE 21 #define TK_LIKE 21
#define TK_GLOB 22 #define TK_MATCH 22
#define TK_BETWEEN 23 #define TK_GLOB 23
#define TK_IN 24 #define TK_BETWEEN 24
#define TK_GT 25 #define TK_IN 25
#define TK_GE 26 #define TK_GT 26
#define TK_LT 27 #define TK_GE 27
#define TK_LE 28 #define TK_LT 28
#define TK_BITAND 29 #define TK_LE 29
#define TK_BITOR 30 #define TK_BITAND 30
#define TK_LSHIFT 31 #define TK_BITOR 31
#define TK_RSHIFT 32 #define TK_LSHIFT 32
#define TK_PLUS 33 #define TK_RSHIFT 33
#define TK_MINUS 34 #define TK_PLUS 34
#define TK_DIVIDE 35 #define TK_MINUS 35
#define TK_TIMES 36 #define TK_DIVIDE 36
#define TK_STAR 37 #define TK_TIMES 37
#define TK_SLASH 38 #define TK_STAR 38
#define TK_REM 39 #define TK_SLASH 39
#define TK_CONCAT 40 #define TK_REM 40
#define TK_UMINUS 41 #define TK_CONCAT 41
#define TK_UPLUS 42 #define TK_UMINUS 42
#define TK_BITNOT 43 #define TK_UPLUS 43
#define TK_SHOW 44 #define TK_BITNOT 44
#define TK_DATABASES 45 #define TK_SHOW 45
#define TK_TOPICS 46 #define TK_DATABASES 46
#define TK_FUNCTIONS 47 #define TK_TOPICS 47
#define TK_MNODES 48 #define TK_FUNCTIONS 48
#define TK_DNODES 49 #define TK_MNODES 49
#define TK_ACCOUNTS 50 #define TK_DNODES 50
#define TK_USERS 51 #define TK_ACCOUNTS 51
#define TK_MODULES 52 #define TK_USERS 52
#define TK_QUERIES 53 #define TK_MODULES 53
#define TK_CONNECTIONS 54 #define TK_QUERIES 54
#define TK_STREAMS 55 #define TK_CONNECTIONS 55
#define TK_VARIABLES 56 #define TK_STREAMS 56
#define TK_SCORES 57 #define TK_VARIABLES 57
#define TK_GRANTS 58 #define TK_SCORES 58
#define TK_VNODES 59 #define TK_GRANTS 59
#define TK_DOT 60 #define TK_VNODES 60
#define TK_CREATE 61 #define TK_DOT 61
#define TK_TABLE 62 #define TK_CREATE 62
#define TK_STABLE 63 #define TK_TABLE 63
#define TK_DATABASE 64 #define TK_STABLE 64
#define TK_TABLES 65 #define TK_DATABASE 65
#define TK_STABLES 66 #define TK_TABLES 66
#define TK_VGROUPS 67 #define TK_STABLES 67
#define TK_DROP 68 #define TK_VGROUPS 68
#define TK_TOPIC 69 #define TK_DROP 69
#define TK_FUNCTION 70 #define TK_TOPIC 70
#define TK_DNODE 71 #define TK_FUNCTION 71
#define TK_USER 72 #define TK_DNODE 72
#define TK_ACCOUNT 73 #define TK_USER 73
#define TK_USE 74 #define TK_ACCOUNT 74
#define TK_DESCRIBE 75 #define TK_USE 75
#define TK_DESC 76 #define TK_DESCRIBE 76
#define TK_ALTER 77 #define TK_DESC 77
#define TK_PASS 78 #define TK_ALTER 78
#define TK_PRIVILEGE 79 #define TK_PASS 79
#define TK_LOCAL 80 #define TK_PRIVILEGE 80
#define TK_COMPACT 81 #define TK_LOCAL 81
#define TK_LP 82 #define TK_COMPACT 82
#define TK_RP 83 #define TK_LP 83
#define TK_IF 84 #define TK_RP 84
#define TK_EXISTS 85 #define TK_IF 85
#define TK_AS 86 #define TK_EXISTS 86
#define TK_OUTPUTTYPE 87 #define TK_AS 87
#define TK_AGGREGATE 88 #define TK_OUTPUTTYPE 88
#define TK_BUFSIZE 89 #define TK_AGGREGATE 89
#define TK_PPS 90 #define TK_BUFSIZE 90
#define TK_TSERIES 91 #define TK_PPS 91
#define TK_DBS 92 #define TK_TSERIES 92
#define TK_STORAGE 93 #define TK_DBS 93
#define TK_QTIME 94 #define TK_STORAGE 94
#define TK_CONNS 95 #define TK_QTIME 95
#define TK_STATE 96 #define TK_CONNS 96
#define TK_COMMA 97 #define TK_STATE 97
#define TK_KEEP 98 #define TK_COMMA 98
#define TK_CACHE 99 #define TK_KEEP 99
#define TK_REPLICA 100 #define TK_CACHE 100
#define TK_QUORUM 101 #define TK_REPLICA 101
#define TK_DAYS 102 #define TK_QUORUM 102
#define TK_MINROWS 103 #define TK_DAYS 103
#define TK_MAXROWS 104 #define TK_MINROWS 104
#define TK_BLOCKS 105 #define TK_MAXROWS 105
#define TK_CTIME 106 #define TK_BLOCKS 106
#define TK_WAL 107 #define TK_CTIME 107
#define TK_FSYNC 108 #define TK_WAL 108
#define TK_COMP 109 #define TK_FSYNC 109
#define TK_PRECISION 110 #define TK_COMP 110
#define TK_UPDATE 111 #define TK_PRECISION 111
#define TK_CACHELAST 112 #define TK_UPDATE 112
#define TK_PARTITIONS 113 #define TK_CACHELAST 113
#define TK_UNSIGNED 114 #define TK_PARTITIONS 114
#define TK_TAGS 115 #define TK_UNSIGNED 115
#define TK_USING 116 #define TK_TAGS 116
#define TK_NULL 117 #define TK_USING 117
#define TK_NOW 118 #define TK_NULL 118
#define TK_SELECT 119 #define TK_NOW 119
#define TK_UNION 120 #define TK_SELECT 120
#define TK_ALL 121 #define TK_UNION 121
#define TK_DISTINCT 122 #define TK_ALL 122
#define TK_FROM 123 #define TK_DISTINCT 123
#define TK_VARIABLE 124 #define TK_FROM 124
#define TK_INTERVAL 125 #define TK_VARIABLE 125
#define TK_EVERY 126 #define TK_INTERVAL 126
#define TK_SESSION 127 #define TK_EVERY 127
#define TK_STATE_WINDOW 128 #define TK_SESSION 128
#define TK_FILL 129 #define TK_STATE_WINDOW 129
#define TK_SLIDING 130 #define TK_FILL 130
#define TK_ORDER 131 #define TK_SLIDING 131
#define TK_BY 132 #define TK_ORDER 132
#define TK_ASC 133 #define TK_BY 133
#define TK_GROUP 134 #define TK_ASC 134
#define TK_HAVING 135 #define TK_GROUP 135
#define TK_LIMIT 136 #define TK_HAVING 136
#define TK_OFFSET 137 #define TK_LIMIT 137
#define TK_SLIMIT 138 #define TK_OFFSET 138
#define TK_SOFFSET 139 #define TK_SLIMIT 139
#define TK_WHERE 140 #define TK_SOFFSET 140
#define TK_RESET 141 #define TK_WHERE 141
#define TK_QUERY 142 #define TK_RESET 142
#define TK_SYNCDB 143 #define TK_QUERY 143
#define TK_ADD 144 #define TK_SYNCDB 144
#define TK_COLUMN 145 #define TK_ADD 145
#define TK_MODIFY 146 #define TK_COLUMN 146
#define TK_TAG 147 #define TK_MODIFY 147
#define TK_CHANGE 148 #define TK_TAG 148
#define TK_SET 149 #define TK_CHANGE 149
#define TK_KILL 150 #define TK_SET 150
#define TK_CONNECTION 151 #define TK_KILL 151
#define TK_STREAM 152 #define TK_CONNECTION 152
#define TK_COLON 153 #define TK_STREAM 153
#define TK_ABORT 154 #define TK_COLON 154
#define TK_AFTER 155 #define TK_ABORT 155
#define TK_ATTACH 156 #define TK_AFTER 156
#define TK_BEFORE 157 #define TK_ATTACH 157
#define TK_BEGIN 158 #define TK_BEFORE 158
#define TK_CASCADE 159 #define TK_BEGIN 159
#define TK_CLUSTER 160 #define TK_CASCADE 160
#define TK_CONFLICT 161 #define TK_CLUSTER 161
#define TK_COPY 162 #define TK_CONFLICT 162
#define TK_DEFERRED 163 #define TK_COPY 163
#define TK_DELIMITERS 164 #define TK_DEFERRED 164
#define TK_DETACH 165 #define TK_DELIMITERS 165
#define TK_EACH 166 #define TK_DETACH 166
#define TK_END 167 #define TK_EACH 167
#define TK_EXPLAIN 168 #define TK_END 168
#define TK_FAIL 169 #define TK_EXPLAIN 169
#define TK_FOR 170 #define TK_FAIL 170
#define TK_IGNORE 171 #define TK_FOR 171
#define TK_IMMEDIATE 172 #define TK_IGNORE 172
#define TK_INITIALLY 173 #define TK_IMMEDIATE 173
#define TK_INSTEAD 174 #define TK_INITIALLY 174
#define TK_MATCH 175 #define TK_INSTEAD 175
#define TK_KEY 176 #define TK_KEY 176
#define TK_OF 177 #define TK_OF 177
#define TK_RAISE 178 #define TK_RAISE 178

5
src/query/inc/sql.y
View File

@ -11,7 +11,7 @@
%left OR. %left OR.
%left AND. %left AND.
%right NOT. %right NOT.
%left EQ NE ISNULL NOTNULL IS LIKE GLOB BETWEEN IN. %left EQ NE ISNULL NOTNULL IS LIKE MATCH GLOB BETWEEN IN.
%left GT GE LT LE. %left GT GE LT LE.
%left BITAND BITOR LSHIFT RSHIFT. %left BITAND BITOR LSHIFT RSHIFT.
%left PLUS MINUS. %left PLUS MINUS.
@ -751,6 +751,9 @@ expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); }
// like expression // like expression
expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); }
// match expression
expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); }
//in expression //in expression
expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); }

16
src/query/src/qFilter.c
View File

@ -28,6 +28,7 @@ OptrStr gOptrStr[] = {
{TSDB_RELATION_GREATER_EQUAL, ">="}, {TSDB_RELATION_GREATER_EQUAL, ">="},
{TSDB_RELATION_NOT_EQUAL, "!="}, {TSDB_RELATION_NOT_EQUAL, "!="},
{TSDB_RELATION_LIKE, "like"}, {TSDB_RELATION_LIKE, "like"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_ISNULL, "is null"}, {TSDB_RELATION_ISNULL, "is null"},
{TSDB_RELATION_NOTNULL, "not null"}, {TSDB_RELATION_NOTNULL, "not null"},
{TSDB_RELATION_IN, "in"}, {TSDB_RELATION_IN, "in"},
@ -156,7 +157,7 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) {
__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal,
compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp,
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8 setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexComp,
}; };
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@ -195,7 +196,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break;
case TSDB_DATA_TYPE_BINARY: { case TSDB_DATA_TYPE_BINARY: {
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparFn = 7; comparFn = 7;
} else if (optr == TSDB_RELATION_IN) { } else if (optr == TSDB_RELATION_IN) {
comparFn = 8; comparFn = 8;
@ -207,7 +210,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_LIKE) { if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_LIKE) {
comparFn = 9; comparFn = 9;
} else if (optr == TSDB_RELATION_IN) { } else if (optr == TSDB_RELATION_IN) {
comparFn = 8; comparFn = 8;
@ -1871,6 +1876,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
case TSDB_RELATION_LIKE: { case TSDB_RELATION_LIKE: {
return ret == 0; return ret == 0;
} }
case TSDB_RELATION_MATCH: {
return ret == 0;
}
case TSDB_RELATION_IN: { case TSDB_RELATION_IN: {
return ret == 1; return ret == 1;
} }
@ -2641,7 +2649,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t
} }
if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
|| cunit->optr == TSDB_RELATION_NOT_EQUAL) { || cunit->optr == TSDB_RELATION_NOT_EQUAL) {
continue; continue;
} }

3359
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

10
src/tsdb/src/tsdbRead.c
View File

@ -3709,6 +3709,9 @@ static bool tableFilterFp(const void* pNode, void* param) {
case TSDB_RELATION_LIKE: { case TSDB_RELATION_LIKE: {
return ret == 0; return ret == 0;
} }
case TSDB_RELATION_MATCH: {
return ret == 0;
}
case TSDB_RELATION_IN: { case TSDB_RELATION_IN: {
return ret == 1; return ret == 1;
} }
@ -4042,6 +4045,8 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
pCond->start->v = queryColInfo->q; pCond->start->v = queryColInfo->q;
} else if (optr == TSDB_RELATION_LIKE) { } else if (optr == TSDB_RELATION_LIKE) {
assert(0); assert(0);
} else if (optr == TSDB_RELATION_MATCH) {
assert(0);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -4199,7 +4204,7 @@ static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, S
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) { if (pQueryInfo->optr == TSDB_RELATION_IN) {
addToResult = pQueryInfo->compare(name, pQueryInfo->q); addToResult = pQueryInfo->compare(name, pQueryInfo->q);
} else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { } else if (pQueryInfo->optr == TSDB_RELATION_LIKE || pQueryInfo->optr == TSDB_RELATION_MATCH) {
addToResult = !pQueryInfo->compare(name, pQueryInfo->q); addToResult = !pQueryInfo->compare(name, pQueryInfo->q);
} }
} else { } else {
@ -4231,7 +4236,8 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re
param->setupInfoFn(pExpr, param->pExtInfo); param->setupInfoFn(pExpr, param->pExtInfo);
tQueryInfo *pQueryInfo = pExpr->_node.info; tQueryInfo *pQueryInfo = pExpr->_node.info;
if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr != TSDB_RELATION_IN)) { if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr != TSDB_RELATION_MATCH
&& pQueryInfo->optr != TSDB_RELATION_IN)) {
queryIndexedColumn(pSkipList, pQueryInfo, result); queryIndexedColumn(pSkipList, pQueryInfo, result);
} else { } else {
queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn);

2
src/util/inc/tcompare.h
View File

@ -26,6 +26,7 @@ extern "C" {
#define TSDB_PATTERN_NOMATCH 1 #define TSDB_PATTERN_NOMATCH 1
#define TSDB_PATTERN_NOWILDCARDMATCH 2 #define TSDB_PATTERN_NOWILDCARDMATCH 2
#define TSDB_PATTERN_STRING_DEFAULT_LEN 100 #define TSDB_PATTERN_STRING_DEFAULT_LEN 100
#define TSDB_REGEX_STRING_DEFAULT_LEN 128
#define FLT_COMPAR_TOL_FACTOR 4 #define FLT_COMPAR_TOL_FACTOR 4
#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) #define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
@ -82,6 +83,7 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrPatternComp(const void* pLeft, const void* pRight); int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight); int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);

55
src/util/src/tcompare.c
View File

@ -12,11 +12,17 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "tcompare.h"
#include "tulog.h"
#include "hash.h"
#include "regex.h"
#include "os.h" #include "os.h"
#include "ttype.h" #include "ttype.h"
#include "tcompare.h"
#include "hash.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) { int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
@ -344,6 +350,43 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
} }
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) { int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a; const char* x = *(const char**)a;
const char* y = *(const char**)b; const char* y = *(const char**)b;
@ -405,7 +448,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break;
case TSDB_DATA_TYPE_BINARY: { case TSDB_DATA_TYPE_BINARY: {
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ if (optr == TSDB_RELATION_MATCH) {
comparFn = compareStrRegexComp;
} else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternComp; comparFn = compareStrPatternComp;
} else if (optr == TSDB_RELATION_IN) { } else if (optr == TSDB_RELATION_IN) {
comparFn = compareFindItemInSet; comparFn = compareFindItemInSet;
@ -417,7 +462,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_LIKE) { if (optr == TSDB_RELATION_MATCH) {
comparFn = compareStrRegexComp;
} else if (optr == TSDB_RELATION_LIKE) {
comparFn = compareWStrPatternComp; comparFn = compareWStrPatternComp;
} else if (optr == TSDB_RELATION_IN) { } else if (optr == TSDB_RELATION_IN) {
comparFn = compareFindItemInSet; comparFn = compareFindItemInSet;

1
src/util/src/ttokenizer.c
View File

@ -53,6 +53,7 @@ static SKeyword keywordTable[] = {
{"NOTNULL", TK_NOTNULL}, {"NOTNULL", TK_NOTNULL},
{"IS", TK_IS}, {"IS", TK_IS},
{"LIKE", TK_LIKE}, {"LIKE", TK_LIKE},
{"MATCH", TK_MATCH},
{"GLOB", TK_GLOB}, {"GLOB", TK_GLOB},
{"BETWEEN", TK_BETWEEN}, {"BETWEEN", TK_BETWEEN},
{"IN", TK_IN}, {"IN", TK_IN},

1
tests/script/fullGeneralSuite.sim
View File

@ -222,3 +222,4 @@ run general/stream/metrics_replica1_vnoden.sim
run general/db/show_create_db.sim run general/db/show_create_db.sim
run general/db/show_create_table.sim run general/db/show_create_table.sim
run general/parser/like.sim run general/parser/like.sim
run general/parser/regex.sim

62
tests/script/general/parser/regex.sim Normal file
View File

@ -0,0 +1,62 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
system sh/exec.sh -n dnode1 -s start
sleep 100
sql connect
$db = testdb
sql drop database if exists $db
sql create database $db
sql use $db
print ======================== regular expression match test
$st_name = st
$ct1_name = ct1
$ct2_name = ct2
sql create table $st_name (ts timestamp, c1b binary(20)) tags(t1b binary(20));
sql create table $ct1_name using $st_name tags('taosdata1')
sql create table $ct2_name using $st_name tags('taosdata2')
sql create table not_match using $st_name tags('NOTMATCH')
sql select tbname from $st_name where tbname match '.*'
if $rows != 3 then
return -1
endi
sql select tbname from $st_name where tbname match '^ct[[:digit:]]'
if $rows != 2 then
return -1
endi
sql select tbname from $st_name where tbname match '.*'
if $rows !=3 then
return -1
endi
sql select tbname from $st_name where t1b match '[[:lower:]]+'
if $rows != 2 then
return -1
endi
sql insert into $ct1_name values(now, 'this is engine')
sql insert into $ct2_name values(now, 'this is app egnine')
sql select c1b from $st_name where c1b match 'engine'
if $data00 != @this is engine@ then
return -1
endi
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT