xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-6145]<feature>:support tag filtering with regex

This commit is contained in:
shenglian zhou 2021-08-23 13:59:17 +08:00
parent 03c9253d9a
commit 1f54d6aad9
14 changed files with 889 additions and 1206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

168
src/client/src/tscSQLParser.c
View File

@ -279,6 +279,8 @@ static uint8_t convertRelationalOperator(SStrToken *pToken) {
return TSDB_BINARY_OP_REMAINDER;
case TK_LIKE:
return TSDB_RELATION_LIKE;
case TK_MATCH:
return TSDB_RELATION_MATCH;
case TK_ISNULL:
return TSDB_RELATION_ISNULL;
case TK_NOTNULL:
@ -3788,6 +3790,9 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
case TK_LIKE:
pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE;
break;
case TK_MATCH:
pColumnFilter->lowerRelOptr = TSDB_RELATION_MATCH;
break;
case TK_ISNULL:
pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL;
break;
@ -3851,9 +3856,15 @@ static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) {
return TSDB_CODE_SUCCESS;
}
static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
taosStringBuilderAppendString(sb, pExpr->value.pz);
static int32_t tablenameCondToString(tSqlExpr* pExpr, uint32_t opToken, SStringBuilder* sb) {
assert(opToken == TK_LIKE || opToken == TK_MATCH);
if (opToken == TK_LIKE) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
taosStringBuilderAppendString(sb, pExpr->value.pz);
} else if (opToken == TK_MATCH) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN);
taosStringBuilderAppendString(sb, pExpr->value.pz);
}
return TSDB_CODE_SUCCESS;
}
@ -3874,7 +3885,7 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
int32_t ret = 0;
const char* msg1 = "non binary column not support like operator";
const char* msg1 = "non binary column not support like/match operator";
const char* msg2 = "binary column not support this operator";
const char* msg3 = "bool column not support this operator";
const char* msg4 = "primary key not support this operator";
@ -3902,12 +3913,13 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
&& pExpr->tokenId != TK_ISNULL
&& pExpr->tokenId != TK_NOTNULL
&& pExpr->tokenId != TK_LIKE
&& pExpr->tokenId != TK_MATCH
&& pExpr->tokenId != TK_IN) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
goto _err_ret;
}
} else {
if (pExpr->tokenId == TK_LIKE) {
if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
goto _err_ret;
}
@ -3955,12 +3967,12 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr*
if (pTableCond->tokenId == TK_IN) {
ret = tablenameListToString(pRight, sb);
} else if (pTableCond->tokenId == TK_LIKE) {
} else if (pTableCond->tokenId == TK_LIKE || pTableCond->tokenId == TK_MATCH) {
if (pRight->tokenId != TK_STRING) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
ret = tablenameCondToString(pRight, sb);
ret = tablenameCondToString(pRight, pTableCond->tokenId, sb);
}
if (ret != TSDB_CODE_SUCCESS) {
@ -4409,7 +4421,7 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr
}
static bool validTableNameOptr(tSqlExpr* pExpr) {
const char nameFilterOptr[] = {TK_IN, TK_LIKE};
const char nameFilterOptr[] = {TK_IN, TK_LIKE, TK_MATCH};
for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) {
if (pExpr->tokenId == nameFilterOptr[i]) {
@ -4501,6 +4513,44 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
return TSDB_CODE_SUCCESS;
}
// check for match expression
static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
const char* msg1 = "regular expression string should be less than %d characters";
const char* msg2 = "illegal column type for match";
const char* msg3 = "invalid regular expression";
tSqlExpr* pLeft = pExpr->pLeft;
tSqlExpr* pRight = pExpr->pRight;
if (pExpr->tokenId == TK_MATCH) {
if (pRight->value.nLen > tsMaxRegexStringLen) {
char tmp[64] = {0};
sprintf(tmp, msg1, tsMaxRegexStringLen);
return invalidOperationMsg(msgBuf, tmp);
}
SSchema* pSchema = tscGetTableSchema(pTableMeta);
if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) {
return invalidOperationMsg(msgBuf, msg2);
}
int errCode = 0;
regex_t regex;
char regErrBuf[256] = {0};
const char* pattern = pRight->value.pz;
int cflags = REG_EXTENDED | REG_ICASE;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, regErrBuf, sizeof(regErrBuf));
tscError("Failed to compile regex pattern %s. reason %s", pattern, regErrBuf);
return invalidOperationMsg(msgBuf, msg3);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) {
tSqlExpr* left = tSqlExprClone(expr);
tSqlExpr* right = expr;
@ -4552,6 +4602,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
return code;
}
// validate the match expression
code = validateMatchExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range
if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
@ -4879,65 +4935,66 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
STagCond* pTagCond = &pQueryInfo->tagCond;
pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid;
assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_IN);
assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_IN);
if (pExpr->tokenId == TK_LIKE) {
if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
char* str = taosStringBuilderGetResult(sb, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
return TSDB_CODE_SUCCESS;
}
} else {
SStringBuilder sb1;
memset(&sb1, 0, sizeof(sb1));
taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1));
taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
// remove the duplicated input table names
int32_t num = 0;
char* tableNameString = taosStringBuilderGetResult(sb, NULL);
// remove the duplicated input table names
int32_t num = 0;
char* tableNameString = taosStringBuilderGetResult(sb, NULL);
char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num);
qsort(segments, num, POINTER_BYTES, tableNameCompar);
char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num);
qsort(segments, num, POINTER_BYTES, tableNameCompar);
int32_t j = 1;
for (int32_t i = 1; i < num; ++i) {
if (strcmp(segments[i], segments[i - 1]) != 0) {
segments[j++] = segments[i];
int32_t j = 1;
for (int32_t i = 1; i < num; ++i) {
if (strcmp(segments[i], segments[i - 1]) != 0) {
segments[j++] = segments[i];
}
}
}
num = j;
num = j;
char name[TSDB_DB_NAME_LEN] = {0};
tNameGetDbName(&pTableMetaInfo->name, name);
SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) };
for (int32_t i = 0; i < num; ++i) {
if (i >= 1) {
taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);
char name[TSDB_DB_NAME_LEN] = {0};
tNameGetDbName(&pTableMetaInfo->name, name);
SStrToken dbToken = {.type = TK_STRING, .z = name, .n = (uint32_t)strlen(name)};
for (int32_t i = 0; i < num; ++i) {
if (i >= 1) {
taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);
}
char idBuf[TSDB_TABLE_FNAME_LEN] = {0};
int32_t xlen = (int32_t)strlen(segments[i]);
SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING};
int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
if (ret != TSDB_CODE_SUCCESS) {
taosStringBuilderDestroy(&sb1);
tfree(segments);
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
return ret;
}
taosStringBuilderAppendString(&sb1, idBuf);
}
char idBuf[TSDB_TABLE_FNAME_LEN] = {0};
int32_t xlen = (int32_t)strlen(segments[i]);
SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING};
char* str = taosStringBuilderGetResult(&sb1, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t)strlen(str);
int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
if (ret != TSDB_CODE_SUCCESS) {
taosStringBuilderDestroy(&sb1);
tfree(segments);
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
return ret;
}
taosStringBuilderAppendString(&sb1, idBuf);
taosStringBuilderDestroy(&sb1);
tfree(segments);
return TSDB_CODE_SUCCESS;
}
char* str = taosStringBuilderGetResult(&sb1, NULL);
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
taosStringBuilderDestroy(&sb1);
tfree(segments);
return TSDB_CODE_SUCCESS;
}
int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) {
@ -8114,7 +8171,7 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect
}
static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) {
const char* msg1 = "non binary column not support like operator";
const char* msg1 = "non binary column not support like/match operator";
const char* msg2 = "invalid operator for binary column in having clause";
const char* msg3 = "invalid operator for bool column in having clause";
@ -8166,11 +8223,12 @@ static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, S
&& pExpr->tokenId != TK_ISNULL
&& pExpr->tokenId != TK_NOTNULL
&& pExpr->tokenId != TK_LIKE
&& pExpr->tokenId != TK_MATCH
) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
} else {
if (pExpr->tokenId == TK_LIKE) {
if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}

2
src/common/inc/texpr.h
View File

@ -33,9 +33,11 @@ struct SSchema;
#define QUERY_COND_REL_PREFIX_IN "IN|"
#define QUERY_COND_REL_PREFIX_LIKE "LIKE|"
#define QUERY_COND_REL_PREFIX_MATCH "MATCH|"
#define QUERY_COND_REL_PREFIX_IN_LEN 3
#define QUERY_COND_REL_PREFIX_LIKE_LEN 5
#define QUERY_COND_REL_PREFIX_MATCH_LEN 6
typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *);

1
src/common/inc/tglobal.h
View File

@ -74,6 +74,7 @@ extern int8_t tsKeepOriginalColumnName;
// client
extern int32_t tsMaxSQLStringLen;
extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;
extern int8_t tsTscEnableRecordSql;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime;

11
src/common/src/texpr.c
View File

@ -430,6 +430,17 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_MATCH;
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1;
pVal->pz = exception_malloc(len);
memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN, len);
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_IN;

1
src/common/src/tglobal.c
View File

@ -85,6 +85,7 @@ int32_t tsCompressColData = -1;
// client
int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN;
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_MAX_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_MAX_LEN;
int8_t tsTscEnableRecordSql = 0;
// the maximum number of results for projection query on super table that are returned from

2
src/inc/taosdef.h
View File

@ -164,6 +164,8 @@ do { \
#define TSDB_RELATION_OR 12
#define TSDB_RELATION_NOT 13
#define TSDB_RELATION_MATCH 14
#define TSDB_BINARY_OP_ADD 30
#define TSDB_BINARY_OP_SUBTRACT 31
#define TSDB_BINARY_OP_MULTIPLY 32

306
src/inc/ttokendef.h
View File

@ -37,159 +37,159 @@
#define TK_NOTNULL 19
#define TK_IS 20
#define TK_LIKE 21
#define TK_GLOB 22
#define TK_BETWEEN 23
#define TK_IN 24
#define TK_GT 25
#define TK_GE 26
#define TK_LT 27
#define TK_LE 28
#define TK_BITAND 29
#define TK_BITOR 30
#define TK_LSHIFT 31
#define TK_RSHIFT 32
#define TK_PLUS 33
#define TK_MINUS 34
#define TK_DIVIDE 35
#define TK_TIMES 36
#define TK_STAR 37
#define TK_SLASH 38
#define TK_REM 39
#define TK_CONCAT 40
#define TK_UMINUS 41
#define TK_UPLUS 42
#define TK_BITNOT 43
#define TK_SHOW 44
#define TK_DATABASES 45
#define TK_TOPICS 46
#define TK_FUNCTIONS 47
#define TK_MNODES 48
#define TK_DNODES 49
#define TK_ACCOUNTS 50
#define TK_USERS 51
#define TK_MODULES 52
#define TK_QUERIES 53
#define TK_CONNECTIONS 54
#define TK_STREAMS 55
#define TK_VARIABLES 56
#define TK_SCORES 57
#define TK_GRANTS 58
#define TK_VNODES 59
#define TK_DOT 60
#define TK_CREATE 61
#define TK_TABLE 62
#define TK_STABLE 63
#define TK_DATABASE 64
#define TK_TABLES 65
#define TK_STABLES 66
#define TK_VGROUPS 67
#define TK_DROP 68
#define TK_TOPIC 69
#define TK_FUNCTION 70
#define TK_DNODE 71
#define TK_USER 72
#define TK_ACCOUNT 73
#define TK_USE 74
#define TK_DESCRIBE 75
#define TK_ALTER 76
#define TK_PASS 77
#define TK_PRIVILEGE 78
#define TK_LOCAL 79
#define TK_COMPACT 80
#define TK_LP 81
#define TK_RP 82
#define TK_IF 83
#define TK_EXISTS 84
#define TK_AS 85
#define TK_OUTPUTTYPE 86
#define TK_AGGREGATE 87
#define TK_BUFSIZE 88
#define TK_PPS 89
#define TK_TSERIES 90
#define TK_DBS 91
#define TK_STORAGE 92
#define TK_QTIME 93
#define TK_CONNS 94
#define TK_STATE 95
#define TK_COMMA 96
#define TK_KEEP 97
#define TK_CACHE 98
#define TK_REPLICA 99
#define TK_QUORUM 100
#define TK_DAYS 101
#define TK_MINROWS 102
#define TK_MAXROWS 103
#define TK_BLOCKS 104
#define TK_CTIME 105
#define TK_WAL 106
#define TK_FSYNC 107
#define TK_COMP 108
#define TK_PRECISION 109
#define TK_UPDATE 110
#define TK_CACHELAST 111
#define TK_PARTITIONS 112
#define TK_UNSIGNED 113
#define TK_TAGS 114
#define TK_USING 115
#define TK_NULL 116
#define TK_NOW 117
#define TK_SELECT 118
#define TK_UNION 119
#define TK_ALL 120
#define TK_DISTINCT 121
#define TK_FROM 122
#define TK_VARIABLE 123
#define TK_INTERVAL 124
#define TK_SESSION 125
#define TK_STATE_WINDOW 126
#define TK_FILL 127
#define TK_SLIDING 128
#define TK_ORDER 129
#define TK_BY 130
#define TK_ASC 131
#define TK_DESC 132
#define TK_GROUP 133
#define TK_HAVING 134
#define TK_LIMIT 135
#define TK_OFFSET 136
#define TK_SLIMIT 137
#define TK_SOFFSET 138
#define TK_WHERE 139
#define TK_RESET 140
#define TK_QUERY 141
#define TK_SYNCDB 142
#define TK_ADD 143
#define TK_COLUMN 144
#define TK_MODIFY 145
#define TK_TAG 146
#define TK_CHANGE 147
#define TK_SET 148
#define TK_KILL 149
#define TK_CONNECTION 150
#define TK_STREAM 151
#define TK_COLON 152
#define TK_ABORT 153
#define TK_AFTER 154
#define TK_ATTACH 155
#define TK_BEFORE 156
#define TK_BEGIN 157
#define TK_CASCADE 158
#define TK_CLUSTER 159
#define TK_CONFLICT 160
#define TK_COPY 161
#define TK_DEFERRED 162
#define TK_DELIMITERS 163
#define TK_DETACH 164
#define TK_EACH 165
#define TK_END 166
#define TK_EXPLAIN 167
#define TK_FAIL 168
#define TK_FOR 169
#define TK_IGNORE 170
#define TK_IMMEDIATE 171
#define TK_INITIALLY 172
#define TK_INSTEAD 173
#define TK_MATCH 174
#define TK_MATCH 22
#define TK_GLOB 23
#define TK_BETWEEN 24
#define TK_IN 25
#define TK_GT 26
#define TK_GE 27
#define TK_LT 28
#define TK_LE 29
#define TK_BITAND 30
#define TK_BITOR 31
#define TK_LSHIFT 32
#define TK_RSHIFT 33
#define TK_PLUS 34
#define TK_MINUS 35
#define TK_DIVIDE 36
#define TK_TIMES 37
#define TK_STAR 38
#define TK_SLASH 39
#define TK_REM 40
#define TK_CONCAT 41
#define TK_UMINUS 42
#define TK_UPLUS 43
#define TK_BITNOT 44
#define TK_SHOW 45
#define TK_DATABASES 46
#define TK_TOPICS 47
#define TK_FUNCTIONS 48
#define TK_MNODES 49
#define TK_DNODES 50
#define TK_ACCOUNTS 51
#define TK_USERS 52
#define TK_MODULES 53
#define TK_QUERIES 54
#define TK_CONNECTIONS 55
#define TK_STREAMS 56
#define TK_VARIABLES 57
#define TK_SCORES 58
#define TK_GRANTS 59
#define TK_VNODES 60
#define TK_DOT 61
#define TK_CREATE 62
#define TK_TABLE 63
#define TK_STABLE 64
#define TK_DATABASE 65
#define TK_TABLES 66
#define TK_STABLES 67
#define TK_VGROUPS 68
#define TK_DROP 69
#define TK_TOPIC 70
#define TK_FUNCTION 71
#define TK_DNODE 72
#define TK_USER 73
#define TK_ACCOUNT 74
#define TK_USE 75
#define TK_DESCRIBE 76
#define TK_ALTER 77
#define TK_PASS 78
#define TK_PRIVILEGE 79
#define TK_LOCAL 80
#define TK_COMPACT 81
#define TK_LP 82
#define TK_RP 83
#define TK_IF 84
#define TK_EXISTS 85
#define TK_AS 86
#define TK_OUTPUTTYPE 87
#define TK_AGGREGATE 88
#define TK_BUFSIZE 89
#define TK_PPS 90
#define TK_TSERIES 91
#define TK_DBS 92
#define TK_STORAGE 93
#define TK_QTIME 94
#define TK_CONNS 95
#define TK_STATE 96
#define TK_COMMA 97
#define TK_KEEP 98
#define TK_CACHE 99
#define TK_REPLICA 100
#define TK_QUORUM 101
#define TK_DAYS 102
#define TK_MINROWS 103
#define TK_MAXROWS 104
#define TK_BLOCKS 105
#define TK_CTIME 106
#define TK_WAL 107
#define TK_FSYNC 108
#define TK_COMP 109
#define TK_PRECISION 110
#define TK_UPDATE 111
#define TK_CACHELAST 112
#define TK_PARTITIONS 113
#define TK_UNSIGNED 114
#define TK_TAGS 115
#define TK_USING 116
#define TK_NULL 117
#define TK_NOW 118
#define TK_SELECT 119
#define TK_UNION 120
#define TK_ALL 121
#define TK_DISTINCT 122
#define TK_FROM 123
#define TK_VARIABLE 124
#define TK_INTERVAL 125
#define TK_SESSION 126
#define TK_STATE_WINDOW 127
#define TK_FILL 128
#define TK_SLIDING 129
#define TK_ORDER 130
#define TK_BY 131
#define TK_ASC 132
#define TK_DESC 133
#define TK_GROUP 134
#define TK_HAVING 135
#define TK_LIMIT 136
#define TK_OFFSET 137
#define TK_SLIMIT 138
#define TK_SOFFSET 139
#define TK_WHERE 140
#define TK_RESET 141
#define TK_QUERY 142
#define TK_SYNCDB 143
#define TK_ADD 144
#define TK_COLUMN 145
#define TK_MODIFY 146
#define TK_TAG 147
#define TK_CHANGE 148
#define TK_SET 149
#define TK_KILL 150
#define TK_CONNECTION 151
#define TK_STREAM 152
#define TK_COLON 153
#define TK_ABORT 154
#define TK_AFTER 155
#define TK_ATTACH 156
#define TK_BEFORE 157
#define TK_BEGIN 158
#define TK_CASCADE 159
#define TK_CLUSTER 160
#define TK_CONFLICT 161
#define TK_COPY 162
#define TK_DEFERRED 163
#define TK_DELIMITERS 164
#define TK_DETACH 165
#define TK_EACH 166
#define TK_END 167
#define TK_EXPLAIN 168
#define TK_FAIL 169
#define TK_FOR 170
#define TK_IGNORE 171
#define TK_IMMEDIATE 172
#define TK_INITIALLY 173
#define TK_INSTEAD 174
#define TK_KEY 175
#define TK_OF 176
#define TK_RAISE 177

5
src/query/inc/sql.y
View File

@ -11,7 +11,7 @@
%left OR.
%left AND.
%right NOT.
%left EQ NE ISNULL NOTNULL IS LIKE GLOB BETWEEN IN.
%left EQ NE ISNULL NOTNULL IS LIKE MATCH GLOB BETWEEN IN.
%left GT GE LT LE.
%left BITAND BITOR LSHIFT RSHIFT.
%left PLUS MINUS.
@ -743,6 +743,9 @@ expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); }
// like expression
expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); }
// match expression
expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); }
//in expression
expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); }

16
src/query/src/qFilter.c
View File

@ -28,6 +28,7 @@ OptrStr gOptrStr[] = {
{TSDB_RELATION_GREATER_EQUAL, ">="},
{TSDB_RELATION_NOT_EQUAL, "!="},
{TSDB_RELATION_LIKE, "like"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_ISNULL, "is null"},
{TSDB_RELATION_NOTNULL, "not null"},
{TSDB_RELATION_IN, "in"},
@ -156,7 +157,7 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) {
__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal,
compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp,
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8
setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexComp,
};
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@ -195,7 +196,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break;
case TSDB_DATA_TYPE_BINARY: {
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparFn = 7;
} else if (optr == TSDB_RELATION_IN) {
comparFn = 8;
@ -207,7 +210,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
}
case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_LIKE) {
if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_LIKE) {
comparFn = 9;
} else if (optr == TSDB_RELATION_IN) {
comparFn = 8;
@ -1871,6 +1876,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
case TSDB_RELATION_LIKE: {
return ret == 0;
}
case TSDB_RELATION_MATCH: {
return ret == 0;
}
case TSDB_RELATION_IN: {
return ret == 1;
}
@ -2641,7 +2649,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t
}
if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
|| cunit->optr == TSDB_RELATION_NOT_EQUAL) {
continue;
}

1520
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

10
src/tsdb/src/tsdbRead.c
View File

@ -3708,6 +3708,9 @@ static bool tableFilterFp(const void* pNode, void* param) {
case TSDB_RELATION_LIKE: {
return ret == 0;
}
case TSDB_RELATION_MATCH: {
return ret == 0;
}
case TSDB_RELATION_IN: {
return ret == 1;
}
@ -4041,6 +4044,8 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
pCond->start->v = queryColInfo->q;
} else if (optr == TSDB_RELATION_LIKE) {
assert(0);
} else if (optr == TSDB_RELATION_MATCH) {
assert(0);
}
return TSDB_CODE_SUCCESS;
@ -4198,7 +4203,7 @@ static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, S
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) {
addToResult = pQueryInfo->compare(name, pQueryInfo->q);
} else if (pQueryInfo->optr == TSDB_RELATION_LIKE) {
} else if (pQueryInfo->optr == TSDB_RELATION_LIKE || pQueryInfo->optr == TSDB_RELATION_MATCH) {
addToResult = !pQueryInfo->compare(name, pQueryInfo->q);
}
} else {
@ -4230,7 +4235,8 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re
param->setupInfoFn(pExpr, param->pExtInfo);
tQueryInfo *pQueryInfo = pExpr->_node.info;
if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr != TSDB_RELATION_IN)) {
if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr == TSDB_RELATION_MATCH
&& pQueryInfo->optr != TSDB_RELATION_IN)) {
queryIndexedColumn(pSkipList, pQueryInfo, result);
} else {
queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn);

2
src/util/inc/tcompare.h
View File

@ -26,6 +26,7 @@ extern "C" {
#define TSDB_PATTERN_NOMATCH 1
#define TSDB_PATTERN_NOWILDCARDMATCH 2
#define TSDB_PATTERN_STRING_MAX_LEN 100
#define TSDB_REGEX_STRING_MAX_LEN 128
#define FLT_COMPAR_TOL_FACTOR 4
#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
@ -82,6 +83,7 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);

50
src/util/src/tcompare.c
View File

@ -13,10 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcompare.h"
#include <tulog.h>
#include "hash.h"
#include "os.h"
#include "ttype.h"
#include "tcompare.h"
#include "hash.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
@ -343,6 +344,43 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED | REG_ICASE;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
@ -403,7 +441,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break;
case TSDB_DATA_TYPE_BINARY: {
if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
if (optr == TSDB_RELATION_MATCH) {
comparFn = compareStrRegexComp;
} else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
comparFn = compareStrPatternComp;
} else if (optr == TSDB_RELATION_IN) {
comparFn = compareFindItemInSet;
@ -415,7 +455,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
}
case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_LIKE) {
if (optr == TSDB_RELATION_MATCH) {
comparFn = compareStrRegexComp;
} else if (optr == TSDB_RELATION_LIKE) {
comparFn = compareWStrPatternComp;
} else if (optr == TSDB_RELATION_IN) {
comparFn = compareFindItemInSet;

1
src/util/src/ttokenizer.c
View File

@ -53,6 +53,7 @@ static SKeyword keywordTable[] = {
{"NOTNULL", TK_NOTNULL},
{"IS", TK_IS},
{"LIKE", TK_LIKE},
{"MATCH", TK_MATCH},
{"GLOB", TK_GLOB},
{"BETWEEN", TK_BETWEEN},
{"IN", TK_IN},