From 3916161f851def5e3e7895d30b1cf42e27092627 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Aug 2021 08:14:48 +0800 Subject: [PATCH 01/10] [TD-5930] fix mem leak --- src/query/src/qAggMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index dad05df22a..1b1ebec4a3 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4047,9 +4047,9 @@ void block_func_merge(SQLFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); mergeTableBlockDist(pResInfo, &info); + taosArrayDestroy(info.dataBlockInfos); pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; From c585958a7723c90d9b397bee60610b6d216a3b3e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Aug 2021 12:37:35 +0800 Subject: [PATCH 02/10] [TD-5918] fix assert core by expand wild cards --- src/common/src/tglobal.c | 2 +- src/util/src/tcompare.c | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 3c904dc034..f9135605bb 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -991,7 +991,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; + cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 7577451f88..906995053b 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -280,25 +280,26 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - - char pattern[128] = {0}; + + assert(varDataLen(pRight) < TSDB_MAX_FIELD_LEN); + char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - assert(varDataLen(pRight) < 128); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); - memcpy(buf, varDataVal(pLeft), sz); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; int32_t ret = patternMatch(pattern, buf, sz, &pInfo); free(buf); + free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } int32_t taosArrayCompareString(const void* a, const void* b) { const char* x = *(const char**)a; const char* y = *(const char**)b; - + return compareLenPrefixedStr(x, y); } @@ -307,19 +308,19 @@ int32_t taosArrayCompareString(const void* a, const void* b) { // return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1; //} int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { - return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; + return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - wchar_t pattern[128] = {0}; - assert(TSDB_PATTERN_STRING_MAX_LEN < 128); + assert(varDataLen(pRight) < TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); + wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - assert(varDataLen(pRight) < 128); - + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } From e8af7af7d615b44449d3e4506b67cf4895988f17 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Aug 2021 17:20:57 +0800 Subject: [PATCH 03/10] [TD-5918] fix assert core by expand wild cards --- src/util/src/tcompare.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 906995053b..a3c01d2be7 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -281,7 +281,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - assert(varDataLen(pRight) < TSDB_MAX_FIELD_LEN); + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); @@ -314,7 +314,7 @@ int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - assert(varDataLen(pRight) < TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); From 28725856d74f974c16dad8e68e0ed0142c765691 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 10 Aug 2021 07:01:41 -0300 Subject: [PATCH 04/10] [TD-5918]bug fixed in branch fix/TD-5918 modify testcases test maxWildCardsLength=16384 by manual --- tests/pytest/query/queryWildcardLength.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/tests/pytest/query/queryWildcardLength.py b/tests/pytest/query/queryWildcardLength.py index d15085f751..1fc46fe7d6 100644 --- a/tests/pytest/query/queryWildcardLength.py +++ b/tests/pytest/query/queryWildcardLength.py @@ -157,19 +157,6 @@ class TDTestCase: tdSql.execute(f'create table {table_name}_sub1 using {table_name} tags ("{table_name}", "{table_name}")') tdSql.execute(f'insert into {table_name}_sub1 values (now, "{table_name}", "{table_name}");') - # TODO sc1 leave a bug ---> TD-5918 - # sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"', - # f'select * from {table_name} where bi1 like "{lp_name}"', - # f'select * from {table_name} where bi1 like "{ul_name}"', - # f'select * from {table_name} where nc1 like "{hp_name}"', - # f'select * from {table_name} where nc1 like "{lp_name}"', - # f'select * from {table_name} where nc1 like "{ul_name}"', - # f'select * from {table_name} where si1 like "{hp_name}"', - # f'select * from {table_name} where si1 like "{lp_name}"', - # f'select * from {table_name} where si1 like "{ul_name}"', - # f'select * from {table_name} where sc1 like "{hp_name}"', - # f'select * from {table_name} where sc1 like "{lp_name}"', - # f'select * from {table_name} where sc1 like "{ul_name}"'] sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"', f'select * from {table_name} where bi1 like "{lp_name}"', f'select * from {table_name} where bi1 like "{ul_name}"', @@ -178,7 +165,11 @@ class TDTestCase: f'select * from {table_name} where nc1 like "{ul_name}"', f'select * from {table_name} where si1 like "{hp_name}"', f'select * from {table_name} where si1 like "{lp_name}"', - f'select * from {table_name} where si1 like "{ul_name}"'] + f'select * from {table_name} where si1 like "{ul_name}"', + f'select * from {table_name} where sc1 like "{hp_name}"', + f'select * from {table_name} where sc1 like "{lp_name}"', + f'select * from {table_name} where sc1 like "{ul_name}"'] + for sql in sql_list: tdSql.query(sql) if len(table_name) >= 1: @@ -211,7 +202,6 @@ class TDTestCase: tdSql.close() tdLog.success("%s successfully executed" % __file__) - tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) From 78844c5ea893d44daa82d8156c3e7b0a74a7b8b8 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 10 Aug 2021 18:06:54 +0800 Subject: [PATCH 05/10] [TD-5907]add some nodes[ci skip] --- Jenkinsfile | 42 ++- tests/pytest/crash_gen/valgrind_taos.supp | 368 +++++++++++++++++++++- 2 files changed, 386 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f1de92cded..0ccf302ba6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,8 +4,6 @@ properties([pipelineTriggers([githubPush()])]) node { git url: 'https://github.com/taosdata/TDengine.git' } - -def skipstage=0 def skipbuild=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -34,7 +32,7 @@ def abort_previous(){ } def pre_test(){ - + sh'hostname' sh ''' sudo rmtaos || echo "taosd has not installed" ''' @@ -144,11 +142,11 @@ pipeline { git checkout -qf FETCH_HEAD ''' - script{ - env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD ${env.CHANGE_TARGET}|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) - env.skipbuild=sh(script: "git log -1 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]'", returnStatus: true) + script{ + skipbuild='2' + skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) + println skipbuild } - println env.skipstage sh''' rm -rf ${WORKSPACE}.tes ''' @@ -158,17 +156,17 @@ pipeline { stage('Parallel test stage') { //only build pr when { + allOf { changeRequest() - expression { - env.skipstage != 0 - env.skipbuild ==1 + expression{ + return skipbuild.trim() == '2' } + } } parallel { stage('python_1_s1') { - agent{label 'p1'} + agent any steps { - pre_test() timeout(time: 45, unit: 'MINUTES'){ sh ''' @@ -181,7 +179,7 @@ pipeline { } } stage('python_2_s5') { - agent{label 'p2'} + agent any steps { pre_test() @@ -195,7 +193,7 @@ pipeline { } } stage('python_3_s6') { - agent{label 'p3'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() @@ -208,7 +206,7 @@ pipeline { } } stage('test_b1_s2') { - agent{label 'b1'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() @@ -221,8 +219,7 @@ pipeline { } stage('test_crash_gen_s3') { - agent{label "b2"} - + agent any steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -260,8 +257,7 @@ pipeline { } stage('test_valgrind_s4') { - agent{label "b3"} - + agent any steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -286,7 +282,7 @@ pipeline { } } stage('test_b4_s7') { - agent{label 'b4'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() @@ -305,7 +301,7 @@ pipeline { } } stage('test_b5_s8') { - agent{label 'b5'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() @@ -318,7 +314,7 @@ pipeline { } } stage('test_b6_s9') { - agent{label 'b6'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() @@ -331,7 +327,7 @@ pipeline { } } stage('test_b7_s10') { - agent{label 'b7'} + agent any steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index 376567b7e8..b9296f008e 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17742,4 +17742,370 @@ fun:taosGetFqdn fun:taosCheckGlobalCfg fun:taos_init_imp -} \ No newline at end of file +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/bin/python3.8 + fun:PyObject_GetItem + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:PyCode_NewWithPosOnlyArgs + fun:PyCode_New + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/tslibs/parsing.cpython-38-x86_64-linux-gnu.so + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/tslibs/parsing.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/local/lib/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_New + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_New + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun: malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/interval.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun: malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + obj:/usr/bin/python3.8 + fun:PyObject_CallFunctionObjArgs + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyObject_GetAttr + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8) + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8) + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8) + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8) + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/interval.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8) + obj:/usr/bin/python3.8) + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/tslibs/np_datetime.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/tslibs/ccalendar.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/interval.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/hashtable.cpython-38-x86_64-linux-gnu.so + obj:/usr/local/lib/python3.8/dist-packages/pandas/_libs/hashtable.cpython-38-x86_64-linux-gnu.so + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall +} +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + obj:/usr/local/lib/python3.8/dist-packages/pandas/* + ... +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_New + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_New + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:_PyObject_MakeTpCall + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:PyObject_GetAttr + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_New + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall +} \ No newline at end of file From 4a210d79340e5bb5b0ac5d0ce9b31eeceabf2d19 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 10 Aug 2021 18:09:41 +0800 Subject: [PATCH 06/10] test[ci skip] --- tests/pytest/crash_gen/valgrind_taos.supp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index b9296f008e..6c92f33018 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -18108,4 +18108,4 @@ fun:_PyEval_EvalFrameDefault fun:_PyEval_EvalCodeWithName fun:_PyFunction_Vectorcall -} \ No newline at end of file +} \ No newline at end of file From 5f8e37504acdb17a9f1a949122bd81dd897fdd31 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 11 Aug 2021 14:01:59 +0800 Subject: [PATCH 07/10] test --- Jenkinsfile | 80 ++++++++++++----------- tests/pytest/crash_gen/valgrind_taos.supp | 2 +- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 64ee9b2134..8fa7e78a5f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,9 +4,6 @@ properties([pipelineTriggers([githubPush()])]) node { git url: 'https://github.com/taosdata/TDengine.git' } - -def skipbuild=0 - def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -33,7 +30,6 @@ def abort_previous(){ milestone(buildNumber) } def pre_test(){ - sh'hostname' sh ''' sudo rmtaos || echo "taosd has not installed" @@ -52,12 +48,18 @@ def pre_test(){ git checkout master ''' } - else { + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else{ sh ''' cd ${WKC} git checkout develop ''' - } + } } sh''' cd ${WKC} @@ -75,7 +77,13 @@ def pre_test(){ git checkout master ''' } - else { + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WK} + git checkout 2.0 + ''' + } + else{ sh ''' cd ${WK} git checkout develop @@ -95,19 +103,17 @@ def pre_test(){ make > /dev/null make install > /dev/null cd ${WKC}/tests - pip3 install ${WKC}/src/connector/python + pip3 install ${WKC}/src/connector/python/ ''' return 1 } pipeline { agent none - environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' } - stages { stage('pre_build'){ agent{label 'master'} @@ -123,19 +129,22 @@ pipeline { rm -rf ${WORKSPACE}.tes cp -r ${WORKSPACE} ${WORKSPACE}.tes cd ${WORKSPACE}.tes - + git fetch ''' script { if (env.CHANGE_TARGET == 'master') { sh ''' git checkout master - git pull origin master ''' } - else { + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + git checkout 2.0 + ''' + } + else{ sh ''' git checkout develop - git pull origin develop ''' } } @@ -143,34 +152,31 @@ pipeline { git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD ''' - script{ skipbuild='2' skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) println skipbuild - } sh''' rm -rf ${WORKSPACE}.tes ''' } } - stage('Parallel test stage') { //only build pr when { - allOf { + allOf{ changeRequest() - expression{ + expression{ return skipbuild.trim() == '2' - } - } + } } parallel { stage('python_1_s1') { - agent any + agent{label " slave1 || slave11 "} steps { + pre_test() timeout(time: 55, unit: 'MINUTES'){ sh ''' @@ -183,7 +189,7 @@ pipeline { } } stage('python_2_s5') { - agent any + agent{label " slave5 || slave15 "} steps { pre_test() @@ -197,7 +203,7 @@ pipeline { } } stage('python_3_s6') { - agent any + agent{label " slave6 || slave16 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -210,7 +216,7 @@ pipeline { } } stage('test_b1_s2') { - agent any + agent{label " slave2 || slave12 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -221,9 +227,9 @@ pipeline { } } } - stage('test_crash_gen_s3') { - agent any + agent{label " slave3 || slave13 "} + steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -255,13 +261,12 @@ pipeline { ./test-all.sh b2fq date ''' - } - + } } } - stage('test_valgrind_s4') { - agent any + agent{label " slave4 || slave14 "} + steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -286,7 +291,7 @@ pipeline { } } stage('test_b4_s7') { - agent any + agent{label " slave7 || slave17 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -305,7 +310,7 @@ pipeline { } } stage('test_b5_s8') { - agent any + agent{label " slave8 || slave18 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -318,7 +323,7 @@ pipeline { } } stage('test_b6_s9') { - agent any + agent{label " slave9 || slave19 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -331,7 +336,7 @@ pipeline { } } stage('test_b7_s10') { - agent any + agent{label " slave10 || slave20 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() @@ -423,6 +428,5 @@ pipeline { from: "support@taosdata.com" ) } - } - -} + } +} \ No newline at end of file diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index 6c92f33018..b9296f008e 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -18108,4 +18108,4 @@ fun:_PyEval_EvalFrameDefault fun:_PyEval_EvalCodeWithName fun:_PyFunction_Vectorcall -} \ No newline at end of file +} \ No newline at end of file From c88d6d3cfaf138e77da69de0aef78189bcd11f5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 11 Aug 2021 17:51:09 +0800 Subject: [PATCH 08/10] [TD-5999]:modify range merge test case --- src/query/tests/rangeMergeTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/tests/rangeMergeTest.cpp b/src/query/tests/rangeMergeTest.cpp index e65508a300..f7fc558ccf 100644 --- a/src/query/tests/rangeMergeTest.cpp +++ b/src/query/tests/rangeMergeTest.cpp @@ -330,7 +330,7 @@ void intDataTest() { filterAddRange(h, ra + i, TSDB_RELATION_AND); } filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); + ASSERT_EQ(num, 1); filterFreeRangeCtx(h); From bc2181f4e4dbb9920680d6de206b34d3f1c558ea Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 11 Aug 2021 18:43:42 +0800 Subject: [PATCH 09/10] [TD-5835]: update performance test script --- tests/pytest/tools/taosdemoPerformance.py | 62 +++++++++++++++++++---- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 6b5681dfbc..c28f94b3db 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -19,11 +19,16 @@ import json import sys class taosdemoPerformace: - def __init__(self, commitID, dbName, branch, type): + def __init__(self, commitID, dbName, branch, type, numOfTables, numOfRows, numOfInt, numOfDouble, numOfBinary): self.commitID = commitID self.dbName = dbName self.branch = branch self.type = type + self.numOfTables = numOfTables + self.numOfRows = numOfRows + self.numOfInt = numOfInt + self.numOfDouble = numOfDouble + self.numOfBinary = numOfBinary self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -51,13 +56,13 @@ class taosdemoPerformace: stb = { "name": "meters", "child_table_exists": "no", - "childtable_count": 10000, + "childtable_count": self.numOfTables, "childtable_prefix": "stb_", "auto_create_table": "no", "data_source": "rand", "batch_create_tbl_num": 10, "insert_mode": "taosc", - "insert_rows": 100000, + "insert_rows": self.numOfRows, "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, @@ -68,7 +73,9 @@ class taosdemoPerformace: "sample_file": "./sample.csv", "tags_file": "", "columns": [ - {"type": "INT", "count": 4} + {"type": "INT", "count": self.numOfInt}, + {"type": "DOUBLE", "count": self.numOfDouble}, + {"type": "BINARY", "len": 128, "count": self.numOfBinary} ], "tags": [ {"type": "INT", "count": 1}, @@ -76,6 +83,7 @@ class taosdemoPerformace: ] } + stables = [] stables.append(stb) @@ -163,7 +171,7 @@ class taosdemoPerformace: cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) - cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float, branch binary(50), type binary(20))") + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float, branch binary(50), type binary(20), numoftables int, numofrows int, numofint int, numofdouble int, numofbinary int)") print("==================== taosdemo performance ====================") print("create tables time: %f" % float(self.createTableTime)) print("insert records time: %f" % float(self.insertRecordsTime)) @@ -171,13 +179,14 @@ class taosdemoPerformace: print("avg delay: %f" % float(self.avgDelay)) print("max delay: %f" % float(self.maxDelay)) print("min delay: %f" % float(self.minDelay)) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f, '%s', '%s')" % + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f, '%s', '%s', %d, %d, %d, %d, %d)" % (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), - self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.branch, self.type)) + self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.branch, + self.type, self.numOfTables, self.numOfRows, self.numOfInt, self.numOfDouble, self.numOfBinary)) cursor.close() cursor1 = self.conn.cursor() - cursor1.execute("drop database if exists %s" % self.insertDB) + # cursor1.execute("drop database if exists %s" % self.insertDB) cursor1.close() if __name__ == '__main__': @@ -209,8 +218,43 @@ if __name__ == '__main__': default='glibc', type=str, help='build type (default: glibc)') + parser.add_argument( + '-i', + '--num-of-int', + action='store', + default=4, + type=int, + help='num of int columns (default: 4)') + parser.add_argument( + '-D', + '--num-of-double', + action='store', + default=0, + type=int, + help='num of double columns (default: 4)') + parser.add_argument( + '-B', + '--num-of-binary', + action='store', + default=0, + type=int, + help='num of binary columns (default: 4)') + parser.add_argument( + '-t', + '--num-of-tables', + action='store', + default=10000, + type=int, + help='num of tables (default: 10000)') + parser.add_argument( + '-r', + '--num-of-rows', + action='store', + default=100000, + type=int, + help='num of rows (default: 100000)') args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.git_branch, args.build_type) + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.git_branch, args.build_type, args.num_of_tables, args.num_of_rows, args.num_of_int, args.num_of_double, args.num_of_binary) perftest.insertData() perftest.createTablesAndStoreData() From a767acea517cef07e5ac5caae0b9e0b683df3eef Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 12 Aug 2021 07:00:52 +0800 Subject: [PATCH 10/10] [TD-5875]: taosdemo show progress (#7288) * [TD-5875]: taosdemo show progress * empty commit for CI --- src/kit/taosdemo/taosdemo.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3838344a8b..7870dd6b03 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -245,7 +245,6 @@ typedef struct SArguments_S { uint32_t disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms, us or ns. accordig to database precision uint32_t method_of_delete; - char ** arg_list; uint64_t totalInsertRows; uint64_t totalAffectedRows; bool demo_mode; // use default column name and semi-random data @@ -637,7 +636,6 @@ SArguments g_args = { 0, // disorderRatio 1000, // disorderRange 1, // method_of_delete - NULL, // arg_list 0, // totalInsertRows; 0, // totalAffectedRows; true, // demo_mode; @@ -6407,6 +6405,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { bool flagSleep = true; uint64_t sleepTimeTotal = 0; + int percentComplete = 0; + int64_t totalRows = insertRows * pThreadInfo->ntables; + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -6583,6 +6584,11 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalAffectedRows += affectedRows; + int currentPercent = pThreadInfo->totalAffectedRows * 100 / totalRows; + if (currentPercent > percentComplete ) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, currentPercent); + percentComplete = currentPercent; + } int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", @@ -6604,6 +6610,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } } } + if (percentComplete < 100) + printf("[%d]:%d%%\n", pThreadInfo->threadID, percentComplete); free_of_interlace: tmfree(pThreadInfo->buffer); @@ -6641,6 +6649,9 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->samplePos = 0; + int percentComplete = 0; + int64_t totalRows = insertRows * pThreadInfo->ntables; + for (uint64_t tableSeq = pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; tableSeq ++) { @@ -6746,6 +6757,11 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->totalAffectedRows += affectedRows; + int currentPercent = pThreadInfo->totalAffectedRows * 100 / totalRows; + if (currentPercent > percentComplete ) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, currentPercent); + percentComplete = currentPercent; + } int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", @@ -6768,6 +6784,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { __func__, __LINE__, pThreadInfo->samplePos); } } // tableSeq + if (percentComplete < 100) + printf("[%d]:%d%%\n", pThreadInfo->threadID, percentComplete); free_of_progressive: tmfree(pThreadInfo->buffer);