From 8411f25a2bde8737e69737bfd7a86558f113ce86 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 17 Apr 2024 18:04:33 +0800 Subject: [PATCH 1/5] fix: TD-26789 nchar match slow --- source/util/src/tcompare.c | 48 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 53fdc96c37..1a9487d4c2 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -24,6 +24,7 @@ #include "tutil.h" #include "types.h" #include "osString.h" +#include int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -1203,28 +1204,52 @@ int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight) { return comparestrRegexMatch(pLeft, pRight) ? 0 : 1; } +static regex_t *threadGetRegComp(const char *pPattern) { + static __thread regex_t pRegex; + static __thread char *pOldPattern; + if (NULL != pOldPattern) { + if( strcmp(pOldPattern, pPattern) == 0) { + return &pRegex; + } else { + regfree(&pRegex); + taosMemoryFree(pOldPattern); + pOldPattern == NULL; + } + } + + pOldPattern = taosMemoryMalloc(strlen(pPattern) + 1); + if (NULL == pOldPattern) { + uError("Failed to Malloc when compile regex pattern %s.", pPattern); + return NULL; + } + strcpy(pOldPattern, pPattern); + int32_t cflags = REG_EXTENDED; + int32_t ret = regcomp(&pRegex, pPattern, cflags); + if (ret != 0) { + char msgbuf[256] = {0}; + regerror(ret, &pRegex, msgbuf, tListLen(msgbuf)); + uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); + regfree(&pRegex); + return NULL; + } + return &pRegex; +} + static int32_t doExecRegexMatch(const char *pString, const char *pPattern) { int32_t ret = 0; - regex_t regex; char msgbuf[256] = {0}; - - int32_t cflags = REG_EXTENDED; - if ((ret = regcomp(®ex, pPattern, cflags)) != 0) { - regerror(ret, ®ex, msgbuf, tListLen(msgbuf)); - - uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); - regfree(®ex); + regex_t *regex = threadGetRegComp(pPattern); + if (regex == NULL) { return 1; } regmatch_t pmatch[1]; - ret = regexec(®ex, pString, 1, pmatch, 0); + ret = regexec(regex, pString, 1, pmatch, 0); if (ret != 0 && ret != REG_NOMATCH) { - regerror(ret, ®ex, msgbuf, sizeof(msgbuf)); + regerror(ret, regex, msgbuf, sizeof(msgbuf)); uDebug("Failed to match %s with pattern %s, reason %s", pString, pPattern, msgbuf) } - regfree(®ex); return (ret == 0) ? 0 : 1; } @@ -1245,7 +1270,6 @@ int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { taosMemoryFree(pattern); return (ret == 0) ? 0 : 1; - ; } int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { From bac72b2dd24c435b9873844a6b4d4ea6ad9a908e Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 17 Apr 2024 18:12:43 +0800 Subject: [PATCH 2/5] fix: mem free --- source/util/src/tcompare.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 1a9487d4c2..ea4ee40a29 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1230,6 +1230,8 @@ static regex_t *threadGetRegComp(const char *pPattern) { regerror(ret, &pRegex, msgbuf, tListLen(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); regfree(&pRegex); + taosMemoryFree(pOldPattern); + pOldPattern == NULL; return NULL; } return &pRegex; From 9c5f7d6656d865499e699907a0967e2b96a0da3f Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 17 Apr 2024 20:06:29 +0800 Subject: [PATCH 3/5] thread mem leak --- include/util/tcompare.h | 1 + source/util/src/tcompare.c | 22 ++++++++++++---------- source/util/src/tworker.c | 2 ++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 65457b287a..9694bee92d 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -83,6 +83,7 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight); int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); +void DestoryThreadLocalRegComp(); int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ea4ee40a29..d845e5aed7 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -24,7 +24,6 @@ #include "tutil.h" #include "types.h" #include "osString.h" -#include int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -1204,19 +1203,16 @@ int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight) { return comparestrRegexMatch(pLeft, pRight) ? 0 : 1; } +static threadlocal regex_t pRegex; +static threadlocal char *pOldPattern; static regex_t *threadGetRegComp(const char *pPattern) { - static __thread regex_t pRegex; - static __thread char *pOldPattern; if (NULL != pOldPattern) { if( strcmp(pOldPattern, pPattern) == 0) { return &pRegex; } else { - regfree(&pRegex); - taosMemoryFree(pOldPattern); - pOldPattern == NULL; + DestoryThreadLocalRegComp(); } } - pOldPattern = taosMemoryMalloc(strlen(pPattern) + 1); if (NULL == pOldPattern) { uError("Failed to Malloc when compile regex pattern %s.", pPattern); @@ -1229,14 +1225,20 @@ static regex_t *threadGetRegComp(const char *pPattern) { char msgbuf[256] = {0}; regerror(ret, &pRegex, msgbuf, tListLen(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); - regfree(&pRegex); - taosMemoryFree(pOldPattern); - pOldPattern == NULL; + DestoryThreadLocalRegComp(); return NULL; } return &pRegex; } +void DestoryThreadLocalRegComp() { + if (NULL != pOldPattern) { + regfree(&pRegex); + taosMemoryFree(pOldPattern); + pOldPattern == NULL; + } +} + static int32_t doExecRegexMatch(const char *pString, const char *pPattern) { int32_t ret = 0; char msgbuf[256] = {0}; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 0712010458..d2651c1d2c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "tgeosctx.h" #include "tlog.h" +#include "tcompare.h" #define QUEUE_THRESHOLD (1000 * 1000) @@ -103,6 +104,7 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { } destroyThreadLocalGeosCtx(); + DestoryThreadLocalRegComp(); return NULL; } From 2f9313fbadbe5b07f93efe6c4519c3abb7c3e59d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 18 Apr 2024 08:45:51 +0800 Subject: [PATCH 4/5] fix: compile error --- source/util/src/tcompare.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index d845e5aed7..679490abac 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1235,7 +1235,7 @@ void DestoryThreadLocalRegComp() { if (NULL != pOldPattern) { regfree(&pRegex); taosMemoryFree(pOldPattern); - pOldPattern == NULL; + pOldPattern = NULL; } } From 417f7f397286c38f3810ffe308508886e93f94ce Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:35:53 +0800 Subject: [PATCH 5/5] Update tcompare.c --- source/util/src/tcompare.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 679490abac..26122a4a29 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1204,7 +1204,7 @@ int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight) { } static threadlocal regex_t pRegex; -static threadlocal char *pOldPattern; +static threadlocal char *pOldPattern = NULL; static regex_t *threadGetRegComp(const char *pPattern) { if (NULL != pOldPattern) { if( strcmp(pOldPattern, pPattern) == 0) {