Merge pull request #7312 from taosdata/feature/TD-5918

[TD-5918] add the configuration of max length wild cards

packaging/cfg/taos.cfg

@@ -144,6 +144,9 @@ keepColumnName            1
 # max length of an SQL
 # maxSQLLength          65480
 
+# max length of WildCards
+# maxWildCardsLength    100
+
 # the maximum number of records allowed for super table time sorting
 # maxNumOfOrderedRes    100000
 

src/client/src/tscSQLParser.c

@@ -3218,7 +3218,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   pCmd->command = TSDB_SQL_SHOW;
 
   const char* msg1 = "invalid name";
-  const char* msg2 = "pattern filter string too long";
+  const char* msg2 = "wildcard string should be less than %d characters";
   const char* msg3 = "database name too long";
   const char* msg4 = "invalid ip address";
   const char* msg5 = "database name is empty";
@@ -3262,8 +3262,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
 
-      if (!tscValidateTableNameLength(pCmd->payloadLen)) {
+      if (pPattern->n > tsMaxWildCardsLen){
-        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+        char tmp[64] = {0};
+        sprintf(tmp, msg2, tsMaxWildCardsLen);
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tmp);
       }
     }
   } else if (showType == TSDB_MGMT_TABLE_VNODES) {
@@ -4394,15 +4396,17 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) {
 
 // check for like expression
 static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
-  const char* msg1 = "wildcard string should be less than 20 characters";
+  const char* msg1 = "wildcard string should be less than %d characters";
   const char* msg2 = "illegal column name";
 
   tSqlExpr* pLeft  = pExpr->pLeft;
   tSqlExpr* pRight = pExpr->pRight;
 
   if (pExpr->tokenId == TK_LIKE) {
-    if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) {
+    if (pRight->value.nLen > tsMaxWildCardsLen) {
-      return invalidOperationMsg(msgBuf, msg1);
+      char tmp[64] = {0};
+      sprintf(tmp, msg1, tsMaxWildCardsLen);
+      return invalidOperationMsg(msgBuf, tmp);
     }
 
     SSchema* pSchema = tscGetTableSchema(pTableMeta);

src/common/inc/tglobal.h

@@ -70,6 +70,7 @@ extern int8_t   tsKeepOriginalColumnName;
 
 // client
 extern int32_t tsMaxSQLStringLen;
+extern int32_t tsMaxWildCardsLen;
 extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;

src/common/src/tglobal.c

@@ -25,6 +25,7 @@
 #include "tutil.h"
 #include "tlocale.h"
 #include "ttimezone.h"
+#include "tcompare.h"
 
 // TSDB
 bool tsdbForceKeepFile = false;
@@ -78,6 +79,7 @@ int32_t tsCompressMsgSize = -1;
 
 // client
 int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN;
+int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_MAX_LEN;
 int8_t  tsTscEnableRecordSql = 0;
 
 // the maximum number of results for projection query on super table that are returned from
@@ -987,6 +989,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_BYTE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "maxWildCardsLength";
+  cfg.ptr = &tsMaxWildCardsLen;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = TSDB_MAX_FIELD_LEN;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_BYTE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "maxNumOfOrderedRes";
   cfg.ptr = &tsMaxNumOfOrderedResults;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1534,6 +1546,7 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  assert(tsGlobalConfigNum <= TSDB_CFG_MAX_NUM);
 #ifdef TD_TSZ
   // lossy compress
   cfg.option = "lossyColumns";

src/util/inc/tcompare.h

@@ -25,7 +25,7 @@ extern "C" {
 #define TSDB_PATTERN_MATCH 0
 #define TSDB_PATTERN_NOMATCH 1
 #define TSDB_PATTERN_NOWILDCARDMATCH 2
-#define TSDB_PATTERN_STRING_MAX_LEN 20
+#define TSDB_PATTERN_STRING_MAX_LEN 100
 
 #define FLT_COMPAR_TOL_FACTOR    4
 #define FLT_EQUAL(_x, _y)        (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))

src/util/src/tcompare.c

@@ -264,18 +264,19 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
 
 static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
-  
+
-  char pattern[128] = {0};
+  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
+  char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char));
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  assert(varDataLen(pRight) < 128);
 
   size_t sz = varDataLen(pLeft);
-  char *buf = malloc(sz + 1); 
+  char *buf = malloc(sz + 1);
-  memcpy(buf, varDataVal(pLeft), sz); 
+  memcpy(buf, varDataVal(pLeft), sz);
   buf[sz] = 0;
 
   int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
   free(buf);
+  free(pattern);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
@@ -297,13 +298,13 @@ static int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
 static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
-  wchar_t pattern[128] = {0};
+  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
-  assert(TSDB_PATTERN_STRING_MAX_LEN < 128);
+  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
 
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  assert(varDataLen(pRight) < 128);
+
-  
   int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
+  free(pattern);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 

tests/pytest/fulltest.sh

@@ -381,8 +381,9 @@ python3 ./test.py -f query/querySession.py
 python3 test.py -f  alter/alter_create_exception.py
 python3 ./test.py -f insert/flushwhiledrop.py
 python3 ./test.py -f insert/schemalessInsert.py
-python3 ./test.py -f alter/alterColMultiTimes.py 
+python3 ./test.py -f alter/alterColMultiTimes.py
-python3 ./test.py -f query/queryTbnameUpperLower.py 
+python3 ./test.py -f query/queryWildcardLength.py
+python3 ./test.py -f query/queryTbnameUpperLower.py
 
 #======================p4-end===============
 

tests/pytest/functions/showOfflineThresholdIs864000.py

@@ -25,7 +25,7 @@ class TDTestCase:
 
     def run(self):
         tdSql.query("show variables")
-        tdSql.checkData(53, 1, 864000)
+        tdSql.checkData(54, 1, 864000)
 
     def stop(self):
         tdSql.close()

tests/pytest/query/queryWildcardLength.py

@@ -0,0 +1,207 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from copy import deepcopy
+import string
+import random
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def cleanTb(self):
+        query_sql = "show stables"
+        res_row_list = tdSql.query(query_sql, True)
+        stb_list = map(lambda x: x[0], res_row_list)
+        for stb in stb_list:
+            tdSql.execute(f'drop table if exists {stb}')
+
+        query_sql = "show tables"
+        res_row_list = tdSql.query(query_sql, True)
+        tb_list = map(lambda x: x[0], res_row_list)
+        for tb in tb_list:
+            tdSql.execute(f'drop table if exists {tb}')
+
+    def getLongWildcardStr(self, len=None):
+        """
+            generate long wildcard str
+        """    
+        maxWildCardsLength = int(tdSql.getVariable('maxWildCardsLength')[0])
+        if len:
+            chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len))
+        else:
+            chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(maxWildCardsLength+1))
+        return chars
+    
+    def genTableName(self):
+        '''
+            generate table name
+            hp_name--->'%str'
+            lp_name--->'str%'
+            ul_name--->'st_r'
+        '''
+        table_name = self.getLongWildcardStr()
+        table_name_list = list(table_name)
+        table_name_list.pop(-1)
+
+        if len(table_name_list) > 1:
+            lp_name = deepcopy(table_name_list)
+            lp_name[-1] = '%'
+            lp_name = ''.join(lp_name)
+
+            ul_name = list(lp_name)
+            ul_name[int(len(ul_name)/2)] = '_'
+            ul_name = ''.join(ul_name)
+
+            table_name_list = list(table_name)
+            hp_name = deepcopy(table_name_list)
+            hp_name.pop(1)
+            hp_name[0] = '%'
+            hp_name = ''.join(hp_name)
+        else:
+            hp_name = '%'
+            lp_name = '%'
+            ul_name = '_'
+        return table_name, hp_name, lp_name, ul_name
+
+    def checkRegularTableWildcardLength(self):
+        '''
+            check regular table wildcard length with % and _
+        '''
+        self.cleanTb()
+        table_name, hp_name, lp_name, ul_name = self.genTableName()
+        tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, a1 int)")
+        sql_list = [f'show tables like "{hp_name}"', f'show tables like "{lp_name}"', f'show tables like "{ul_name}"']
+        for sql in sql_list:
+            tdSql.query(sql)
+            if len(table_name) >= 1:
+                tdSql.checkRows(1)
+            else:
+                tdSql.error(sql)
+        
+        exceed_sql_list = [f'show tables like "%{hp_name}"', f'show tables like "{lp_name}%"', f'show tables like "{ul_name}%"']
+        for sql in exceed_sql_list:
+            tdSql.error(sql)
+
+    def checkSuperTableWildcardLength(self):
+        '''
+            check super table wildcard length with % and _
+        '''
+        self.cleanTb()
+        table_name, hp_name, lp_name, ul_name = self.genTableName()
+        tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, c1 int) tags (t1 int)")
+        sql_list = [f'show stables like "{hp_name}"', f'show stables like "{lp_name}"', f'show stables like "{ul_name}"']
+        for sql in sql_list:
+            tdSql.query(sql)
+            if len(table_name) >= 1:
+                tdSql.checkRows(1)
+            else:
+                tdSql.error(sql)
+
+        exceed_sql_list = [f'show stables like "%{hp_name}"', f'show stables like "{lp_name}%"', f'show stables like "{ul_name}%"']
+        for sql in exceed_sql_list:
+            tdSql.error(sql)
+    
+    def checkRegularWildcardSelectLength(self):
+        '''
+            check regular table wildcard select length with % and _
+        '''
+        self.cleanTb()
+        table_name, hp_name, lp_name, ul_name = self.genTableName()
+        tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200))")
+        tdSql.execute(f'insert into {table_name} values (now, "{table_name}", "{table_name}")')
+        sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"',
+                    f'select * from {table_name} where bi1 like "{lp_name}"',
+                    f'select * from {table_name} where bi1 like "{ul_name}"',
+                    f'select * from {table_name} where nc1 like "{hp_name}"',
+                    f'select * from {table_name} where nc1 like "{lp_name}"',
+                    f'select * from {table_name} where nc1 like "{ul_name}"']
+        for sql in sql_list:
+            tdSql.query(sql)
+            if len(table_name) >= 1:
+                tdSql.checkRows(1)
+            else:
+                tdSql.error(sql)
+        
+        exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"',
+                            f'select * from {table_name} where bi1 like "{lp_name}%"',
+                            f'select * from {table_name} where bi1 like "{ul_name}%"',
+                            f'select * from {table_name} where nc1 like "%{hp_name}"',
+                            f'select * from {table_name} where nc1 like "{lp_name}%"',
+                            f'select * from {table_name} where nc1 like "{ul_name}%"']
+        for sql in exceed_sql_list:
+            tdSql.error(sql)
+        
+    def checkStbWildcardSelectLength(self):
+        '''
+            check stb wildcard select length with % and _
+        '''
+        self.cleanTb()
+        table_name, hp_name, lp_name, ul_name = self.genTableName()
+
+        tdSql.execute(f'CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200)) tags (si1 binary(200), sc1 nchar(200))')
+        tdSql.execute(f'create table {table_name}_sub1 using {table_name} tags ("{table_name}", "{table_name}")')
+        tdSql.execute(f'insert into {table_name}_sub1 values (now, "{table_name}", "{table_name}");')
+
+        sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"',
+                    f'select * from {table_name} where bi1 like "{lp_name}"',
+                    f'select * from {table_name} where bi1 like "{ul_name}"',
+                    f'select * from {table_name} where nc1 like "{hp_name}"',
+                    f'select * from {table_name} where nc1 like "{lp_name}"',
+                    f'select * from {table_name} where nc1 like "{ul_name}"',
+                    f'select * from {table_name} where si1 like "{hp_name}"',
+                    f'select * from {table_name} where si1 like "{lp_name}"',
+                    f'select * from {table_name} where si1 like "{ul_name}"',
+                    f'select * from {table_name} where sc1 like "{hp_name}"',
+                    f'select * from {table_name} where sc1 like "{lp_name}"',
+                    f'select * from {table_name} where sc1 like "{ul_name}"']
+      
+        for sql in sql_list:
+            tdSql.query(sql)
+            if len(table_name) >= 1:
+                tdSql.checkRows(1)
+            else:
+                tdSql.error(sql)
+        exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"',
+                            f'select * from {table_name} where bi1 like "{lp_name}%"',
+                            f'select * from {table_name} where bi1 like "{ul_name}%"',
+                            f'select * from {table_name} where nc1 like "%{hp_name}"',
+                            f'select * from {table_name} where nc1 like "{lp_name}%"',
+                            f'select * from {table_name} where nc1 like "{ul_name}%"',
+                            f'select * from {table_name} where si1 like "%{hp_name}"',
+                            f'select * from {table_name} where si1 like "{lp_name}%"',
+                            f'select * from {table_name} where si1 like "{ul_name}%"',
+                            f'select * from {table_name} where sc1 like "%{hp_name}"',
+                            f'select * from {table_name} where sc1 like "{lp_name}%"',
+                            f'select * from {table_name} where sc1 like "{ul_name}%"']
+        for sql in exceed_sql_list:
+            tdSql.error(sql)
+
+    def run(self):
+        tdSql.prepare()
+        self.checkRegularTableWildcardLength()
+        self.checkSuperTableWildcardLength()
+        self.checkRegularWildcardSelectLength()
+        self.checkStbWildcardSelectLength()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
+

tests/pytest/util/sql.py

@@ -81,6 +81,22 @@ class TDSql:
             return self.queryResult
         return self.queryRows
 
+    def getVariable(self, search_attr):
+        '''
+            get variable of search_attr access "show variables"
+        '''
+        try:
+            sql = 'show variables'
+            param_list = self.query(sql, row_tag=True)
+            for param in param_list:
+                if param[0] == search_attr:
+                    return param[1], param_list
+        except Exception as e:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno, sql, repr(e))
+            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+            raise Exception(repr(e))
+
     def getColNameList(self, sql, col_tag=None):
         self.sql = sql
         try: