From a0e13e04fe0de2a2232fd2ab6ebb448fd992b253 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 3 Nov 2022 18:27:43 +0800 Subject: [PATCH 001/101] fix(query): add mode with selectivity rows --- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 30 +++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index afe639b856..d3f03e8e9c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2734,7 +2734,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mode", .type = FUNCTION_TYPE_MODE, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, .translateFunc = translateMode, .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c8f0b3d826..43a8ef78d8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -254,8 +254,9 @@ typedef struct SUniqueInfo { } SUniqueInfo; typedef struct SModeItem { - int64_t count; - char data[]; + int64_t count; + STuplePos tuplePos; + char data[]; } SModeItem; typedef struct SModeInfo { @@ -263,6 +264,10 @@ typedef struct SModeInfo { uint8_t colType; int16_t colBytes; SHashObj* pHash; + + STuplePos nullTuplePos; + bool nullTupleSaved; + char pItems[]; } SModeInfo; @@ -5377,10 +5382,13 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } else { pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } + pInfo->nullTupleSaved = false; + pInfo->nullTuplePos.pageId = -1; + return true; } -static void doModeAdd(SModeInfo* pInfo, char* data) { +static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -5389,10 +5397,17 @@ static void doModeAdd(SModeInfo* pInfo, char* data) { memcpy(pItem->data, data, hashKeyBytes); pItem->count += 1; + if (pCtx->subsidiaries.num > 0) { + pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, NULL); + } + taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); pInfo->numOfPoints++; } else { (*pHashItem)->count += 1; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + } } } @@ -5414,7 +5429,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - doModeAdd(pInfo, data); + doModeAdd(pInfo, i, pCtx, data); if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { taosHashCleanup(pInfo->pHash); @@ -5422,6 +5437,11 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { + pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + pInfo->nullTupleSaved = true; + } + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -5447,8 +5467,10 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); colDataAppend(pCol, currentRow, pResItem->data, false); + setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); } else { colDataAppendNULL(pCol, currentRow); + setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } return pResInfo->numOfRes; From 846b2ca85c3f308684beff10067972d0419813aa Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 4 Nov 2022 11:39:19 +0800 Subject: [PATCH 002/101] fix test cases --- tests/system-test/0-others/udf_cfg2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index 67848a9430..a7a63bd8cf 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -281,7 +281,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -290,7 +289,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions From aa117b0531c67af6d9457c529963638d56a790b6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 4 Nov 2022 11:39:19 +0800 Subject: [PATCH 003/101] fix test cases --- tests/system-test/0-others/udfTest.py | 2 -- tests/system-test/0-others/udf_create.py | 2 -- tests/system-test/0-others/udf_restart_taosd.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index a58e22ba1c..48102c01e8 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -279,7 +279,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -288,7 +287,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 736943cc56..2fbd43eb86 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -281,7 +281,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -290,7 +289,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 02c59c21e1..5fc5158f1f 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -278,7 +278,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -287,7 +286,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions From d7a883bedcd7be63ba36cdec6f91b9a60ba0adea Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 7 Nov 2022 10:16:18 +0800 Subject: [PATCH 004/101] add test cases --- tests/system-test/2-query/mode.py | 208 ++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 tests/system-test/2-query/mode.py diff --git a/tests/system-test/2-query/mode.py b/tests/system-test/2-query/mode.py new file mode 100644 index 0000000000..72bae55261 --- /dev/null +++ b/tests/system-test/2-query/mode.py @@ -0,0 +1,208 @@ +from random import randint, random +from numpy import equal +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self, dbname="db"): + tdSql.execute( + f'''create table {dbname}.stb + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + f''' + create table {dbname}.tb + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(2): + tdSql.execute(f'create table {dbname}.ctb{i+1} using {dbname}.stb tags ( {i+1} )') + + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:00.000', 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:01.000', 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:02.000', 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:03.000', 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:00.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into {dbname}.tb values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_mode_function(self, dbname="db"): + + # normal table + tdSql.query(f"select ts,mode(c1) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 9) + + tdSql.query(f"select ts,mode(c2) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -99999999999999999) + + tdSql.query(f"select ts,mode(c3) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -999) + + tdSql.query(f"select ts,mode(c4) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -99) + + tdSql.query(f"select ts,mode(c5) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -9.99) + + tdSql.query(f"select ts,mode(c6) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -1e+21) + + tdSql.query(f"select ts,mode(c7) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, True) + + tdSql.query(f"select ts,mode(c8) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 'binary9') + + tdSql.query(f"select ts,mode(c9) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 'nchar9') + + tdSql.query(f"select ts,c3,c5,c8,mode(c1) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -999) + tdSql.checkData(0, 2, -9.99) + tdSql.checkData(0, 3, 'binary9') + tdSql.checkData(0, 4, 9) + + # super table + tdSql.query(f"select ts,mode(c1) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 9) + + tdSql.query(f"select ts,mode(c2) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -99999) + + tdSql.query(f"select ts,mode(c3) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -999) + + tdSql.query(f"select ts,mode(c4) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 2)) + tdSql.checkData(0, 1, -99) + + tdSql.query(f"select ts,mode(c5) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -9.99) + + tdSql.query(f"select ts,mode(c6) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -99.99) + + tdSql.query(f"select ts,mode(c7) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, True) + + tdSql.query(f"select ts,mode(c8) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 'binary9') + + tdSql.query(f"select ts,mode(c9) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 'nchar9') + + tdSql.query(f"select ts,mode(c1),c3,c5,c8 from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 9) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, -9.99) + tdSql.checkData(0, 4, 'binary9') + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test results ==============") + + self.check_mode_function() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From a635e658988473d521fcf70c069d65408dfdf050 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 7 Nov 2022 10:16:18 +0800 Subject: [PATCH 005/101] add test cases --- tests/system-test/fulltest.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index c65c574e7c..50e0de6cbd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -140,6 +140,8 @@ python3 ./test.py -f 2-query/max.py python3 ./test.py -f 2-query/max.py -R python3 ./test.py -f 2-query/min.py python3 ./test.py -f 2-query/min.py -R +python3 ./test.py -f 2-query/mode.py +python3 ./test.py -f 2-query/mode.py -R python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Now.py -R python3 ./test.py -f 2-query/percentile.py @@ -382,6 +384,7 @@ python3 ./test.py -f 2-query/Now.py -Q 2 python3 ./test.py -f 2-query/Today.py -Q 2 python3 ./test.py -f 2-query/max.py -Q 2 python3 ./test.py -f 2-query/min.py -Q 2 +python3 ./test.py -f 2-query/mode.py -Q 2 python3 ./test.py -f 2-query/count.py -Q 2 python3 ./test.py -f 2-query/last.py -Q 2 python3 ./test.py -f 2-query/first.py -Q 2 @@ -477,6 +480,7 @@ python3 ./test.py -f 2-query/Now.py -Q 3 python3 ./test.py -f 2-query/Today.py -Q 3 python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 +python3 ./test.py -f 2-query/mode.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 @@ -574,6 +578,7 @@ python3 ./test.py -f 2-query/Now.py -Q 4 python3 ./test.py -f 2-query/Today.py -Q 4 python3 ./test.py -f 2-query/max.py -Q 4 python3 ./test.py -f 2-query/min.py -Q 4 +python3 ./test.py -f 2-query/mode.py -Q 4 python3 ./test.py -f 2-query/count.py -Q 4 python3 ./test.py -f 2-query/last.py -Q 4 python3 ./test.py -f 2-query/first.py -Q 4 From 90fc572394bb37776cee92e08d188b96319560cc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 7 Nov 2022 15:32:48 +0800 Subject: [PATCH 006/101] avoid epset leak --- source/libs/scheduler/src/schRemote.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 47de2528fa..17dc3d588f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -425,6 +425,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -438,6 +439,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { code); if (pMsg) { taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); } return TSDB_CODE_SUCCESS; } @@ -492,6 +494,7 @@ _return: tFreeSSchedulerHbRsp(&rsp); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); SCH_RET(code); } From 70f7e3e379ffcf2a6282e34a442b884c9990ed92 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 7 Nov 2022 15:42:53 +0800 Subject: [PATCH 007/101] perf: optimize sma calculation --- include/common/tdataformat.h | 1 + source/common/src/tdataformat.c | 384 +++++++++++++++++- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 105 ----- 6 files changed, 396 insertions(+), 119 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 60e8f63c22..5c470982bb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -134,6 +134,7 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal); int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // STRUCT ================================ struct STColumn { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 513c54c7e9..1a7e9b2347 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -923,7 +923,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { - if(!pTag || !pTagVal){ + if (!pTag || !pTagVal) { return false; } @@ -1723,3 +1723,385 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { _exit: return code; } + +#define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ + do { \ + (SUM) += (VAL); \ + if ((MAX) < (VAL)) (MAX) = (VAL); \ + if ((MIN) > (VAL)) (MIN) = (VAL); \ + } while (0) + +static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = 0; + *min = 1; + *numOfNull = 0; + + int8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT8_MIN; + *min = INT8_MAX; + *numOfNull = 0; + + int8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((int8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT16_MIN; + *min = INT16_MAX; + *numOfNull = 0; + + int16_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((int16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT32_MIN; + *min = INT32_MAX; + *numOfNull = 0; + + int32_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((int32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT64_MIN; + *min = INT64_MAX; + *numOfNull = 0; + + int64_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((int64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(double *)sum = 0; + *(double *)max = -FLT_MAX; + *(double *)min = FLT_MAX; + *numOfNull = 0; + + float val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((float *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((float *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(double *)sum = 0; + *(double *)max = -DBL_MAX; + *(double *)min = DBL_MAX; + *numOfNull = 0; + + double val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((double *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((double *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT8_MAX; + *numOfNull = 0; + + uint8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((uint8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT16_MAX; + *numOfNull = 0; + + uint16_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((uint16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT32_MAX; + *numOfNull = 0; + + uint32_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((uint32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT64_MAX; + *numOfNull = 0; + + uint64_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + *numOfNull++; + break; + case 2: + val = ((uint64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { + NULL, + tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL + tColDataCalcSMATinyInt, // TSDB_DATA_TYPE_TINYINT + tColDataCalcSMATinySmallInt, // TSDB_DATA_TYPE_SMALLINT + tColDataCalcSMAInt, // TSDB_DATA_TYPE_INT + tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT + tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT + tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT + tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT + tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT + tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL // TSDB_DATA_TYPE_MEDIUMBLOB +}; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 6ae52f1fd7..b942f0df57 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -193,7 +193,6 @@ int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision); void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey); int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); -void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, @@ -716,14 +715,14 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); // tsdbCache ============================================================================================== typedef struct SCacheRowsReader { - SVnode *pVnode; - STSchema *pSchema; - uint64_t uid; - uint64_t suid; - char **transferBuf; // todo remove it soon - int32_t numOfCols; - int32_t type; - int32_t tableIndex; // currently returned result tables + SVnode *pVnode; + STSchema *pSchema; + uint64_t uid; + uint64_t suid; + char **transferBuf; // todo remove it soon + int32_t numOfCols; + int32_t type; + int32_t tableIndex; // currently returned result tables STableKeyInfo *pTableList; // table id list int32_t numOfTables; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 43be51a694..9f59707ddc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -427,13 +427,13 @@ static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *p {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; -extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); +// extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (pBuilder->calcSma) { if (COL_VAL_IS_VALUE(pColVal)) { - tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); + // tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); } else { pBuilder->sma.numOfNull++; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 27beb22165..c5a6352485 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -519,8 +519,8 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - SColumnDataAgg sma; - tsdbCalcColDataSMA(pColData, &sma); + SColumnDataAgg sma = {.colId = pColData->cid}; + tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e02a28cdf..737d421aa0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1512,111 +1512,6 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { return n; } -#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \ - do { \ - (SUM_V) += (VAL); \ - if (!(MINSET)) { \ - (MIN_V) = (VAL); \ - (MINSET) = 1; \ - } else if ((MIN_V) > (VAL)) { \ - (MIN_V) = (VAL); \ - } \ - if (!(MAXSET)) { \ - (MAX_V) = (VAL); \ - (MAXSET) = 1; \ - } else if ((MAX_V) < (VAL)) { \ - (MAX_V) = (VAL); \ - } \ - } while (0) - -static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - int8_t val = *(int8_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - int16_t val = *(int16_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int32_t val = *(int32_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int64_t val = *(int64_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - float val = *(float *)&pColVal->value.val; - SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - double val = *(double *)&pColVal->value.val; - SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint8_t val = *(uint8_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint16_t val = *(uint16_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - uint32_t val = *(uint32_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint64_t val = *(uint64_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = { - NULL, - tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL - tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT - tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT - tSmaUpdateInt, // TSDB_DATA_TYPE_INT - tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT - tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT - tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR - tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR - tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT - tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT - tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT - tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON - NULL, // TSDB_DATA_TYPE_VARBINARY - NULL, // TSDB_DATA_TYPE_DECIMAL - NULL, // TSDB_DATA_TYPE_BLOB - NULL, // TSDB_DATA_TYPE_MEDIUMBLOB -}; -void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { - *pColAgg = (SColumnDataAgg){.colId = pColData->cid}; - uint8_t minSet = 0; - uint8_t maxSet = 0; - - SColVal cv; - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - tColDataGetValue(pColData, iVal, &cv); - - if (COL_VAL_IS_VALUE(&cv)) { - tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); - } else { - pColAgg->numOfNull++; - } - } -} - int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf) { int32_t code = 0; From c17f8244be3372dcb0066d78003f04f5892eb805 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 7 Nov 2022 16:45:29 +0800 Subject: [PATCH 008/101] enh(tdb): new interface of meta prep async commit --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaCommit.c | 1 + source/libs/tdb/inc/tdb.h | 1 + source/libs/tdb/src/db/tdbDb.c | 19 ++++++++- source/libs/tdb/src/db/tdbPager.c | 51 ++++++++++++++++++++++-- source/libs/tdb/src/inc/tdbInt.h | 1 + 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 44ecf64419..ac9fabf052 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -102,6 +102,7 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); int metaCommit(SMeta* pMeta); int metaFinishCommit(SMeta* pMeta); +int metaPrepareAsyncCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 01ad833d20..0be0c3e407 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -35,6 +35,7 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { // commit the meta txn int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); } int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, &pMeta->txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, &pMeta->txn); } // abort the meta txn int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, &pMeta->txn); } diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 3d92d164b4..7ab2bc3995 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -36,6 +36,7 @@ int32_t tdbClose(TDB *pDb); int32_t tdbBegin(TDB *pDb, TXN *pTxn); int32_t tdbCommit(TDB *pDb, TXN *pTxn); int32_t tdbPostCommit(TDB *pDb, TXN *pTxn); +int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn); int32_t tdbAbort(TDB *pDb, TXN *pTxn); int32_t tdbAlter(TDB *pDb, int pages); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 855e3510f2..5aff5b7bb2 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -138,7 +138,24 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerPostCommit(pPager, pTxn); if (ret < 0) { - tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, + pTxn->txnId); + return -1; + } + } + + return 0; +} + +int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; + + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerPrepareAsyncCommit(pPager, pTxn); + if (ret < 0) { + tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, + pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index cee54c1a73..c3ae1dc739 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -287,6 +287,10 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return -1; } + return 0; +} + +int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { // remove the journal file if (tdbOsClose(pPager->jfd) < 0) { tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); @@ -305,15 +309,54 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return 0; } -int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); +int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { + SPage *pPage; + int ret; + + // sync the journal file + ret = tdbOsFSync(pPager->jfd); + if (ret < 0) { + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pPager->inTran = 0; + // loop to write the dirty pages to file + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + if (pPage->isLocal) continue; + ret = tdbPagerWritePageToDB(pPager, pPage); + if (ret < 0) { + tdbError("failed to write page to db since %s", tstrerror(terrno)); + return -1; + } + } + tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); + pPager->dbOrigSize = pPager->dbFileSize; + + // release the page + iter = tRBTreeIterCreate(&pPager->rbt, 1); + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + if (pPage->isLocal) continue; + pPage->isDirty = 0; + + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + /* + tRBTreeCreate(&pPager->rbt, pageCmpFn); + + // sync the db file + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + */ return 0; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b45747c972..e5ece98b28 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -191,6 +191,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage); int tdbPagerBegin(SPager *pPager, TXN *pTxn); int tdbPagerCommit(SPager *pPager, TXN *pTxn); int tdbPagerPostCommit(SPager *pPager, TXN *pTxn); +int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn); int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); From d590a38d62b5fa938f2ce0268b87b55b03ed0812 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 7 Nov 2022 17:36:58 +0800 Subject: [PATCH 009/101] fix: fix tsdbRead crash caused by reuse after free TD-20246 --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bfde5b3076..233a64bf77 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -364,9 +364,9 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyAllBlockScanInfo(SHashObj* pTableMap) { +static void destroyAllBlockScanInfo(SHashObj* pTableMap, bool clearEntry) { void* p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { + while (clearEntry && ((p = taosHashIterate(pTableMap, p)) != NULL)) { clearBlockScanInfo(*(STableBlockScanInfo**)p); } @@ -3763,7 +3763,7 @@ void tsdbReaderClose(STsdbReader* pReader) { cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); - destroyAllBlockScanInfo(pReader->status.pTableMap); + destroyAllBlockScanInfo(pReader->status.pTableMap, (pReader->innerReader[0] == NULL) ? true : false); blockDataDestroy(pReader->pResBlock); clearBlockScanInfoBuf(&pReader->blockInfoBuf); From 005b6780fad65df1ff3bd8832f37d5b46722551d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 7 Nov 2022 18:14:02 +0800 Subject: [PATCH 010/101] fix: sma null bug --- source/common/src/tdataformat.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1a7e9b2347..d151945979 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1749,7 +1749,7 @@ static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, i switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0; @@ -1781,7 +1781,7 @@ static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((int8_t *)pColData->pData)[iVal]; @@ -1813,7 +1813,7 @@ static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((int16_t *)pColData->pData)[iVal]; @@ -1845,7 +1845,7 @@ static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, in switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((int32_t *)pColData->pData)[iVal]; @@ -1877,7 +1877,7 @@ static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((int64_t *)pColData->pData)[iVal]; @@ -1909,7 +1909,7 @@ static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((float *)pColData->pData)[iVal]; @@ -1941,7 +1941,7 @@ static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((double *)pColData->pData)[iVal]; @@ -1973,7 +1973,7 @@ static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *su switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((uint8_t *)pColData->pData)[iVal]; @@ -2005,7 +2005,7 @@ static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_ switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((uint16_t *)pColData->pData)[iVal]; @@ -2037,7 +2037,7 @@ static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, i switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((uint32_t *)pColData->pData)[iVal]; @@ -2069,7 +2069,7 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum switch (tColDataGetBitValue(pColData, iVal)) { case 0: case 1: - *numOfNull++; + (*numOfNull)++; break; case 2: val = ((uint64_t *)pColData->pData)[iVal]; From f7fa81ee2c384fdfa9c2d491ca3140b655e3e035 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 7 Nov 2022 18:51:13 +0800 Subject: [PATCH 011/101] fix: fix error handling for query msg preprocessing --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d401f9c17b..17bc1526b5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -155,9 +155,13 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + code = vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + if (code) { + dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, terrstr(code)); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); From 1a4cf6964b1ed2c1ca0f874cd1e536af73e07cd8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 7 Nov 2022 18:28:45 +0800 Subject: [PATCH 012/101] fix: add post commit to tq & stream --- source/dnode/vnode/src/tq/tqMeta.c | 16 ++++++++++++++++ source/dnode/vnode/src/tq/tqSnapshot.c | 2 ++ source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 ++ source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 2 ++ source/libs/stream/src/streamMeta.c | 1 + source/libs/stream/src/streamState.c | 4 ++++ source/libs/tdb/src/db/tdbBtree.c | 1 + 7 files changed, 28 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index b852acb936..27c491c86b 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -125,6 +125,10 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ return -1; } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + return -1; + } + return 0; } @@ -147,6 +151,10 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + return 0; } @@ -226,6 +234,10 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + tEncoderClear(&encoder); taosMemoryFree(buf); return 0; @@ -250,6 +262,10 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + return 0; } diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index c52e0e2c09..b68763867e 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 92e5f8df7a..08d5931bc3 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index b4a7ce7737..31e44a5b6d 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaStore, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 98e4e77cb0..682bfbaf8d 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -70,6 +70,7 @@ _err: void streamMetaClose(SStreamMeta* pMeta) { tdbCommit(pMeta->db, &pMeta->txn); + tdbPostCommit(pMeta->db, &pMeta->txn); tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pCheckpointDb); tdbClose(pMeta->db); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 9829955baf..cf2ead364b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -142,6 +142,7 @@ _err: void streamStateClose(SStreamState* pState) { tdbCommit(pState->db, &pState->txn); + tdbPostCommit(pState->db, &pState->txn); tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); @@ -168,6 +169,9 @@ int32_t streamStateCommit(SStreamState* pState) { if (tdbCommit(pState->db, &pState->txn) < 0) { return -1; } + if (tdbPostCommit(pState->db, &pState->txn) < 0) { + return -1; + } memset(&pState->txn, 0, sizeof(TXN)); if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e4d3cc3488..e3860f85c6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -140,6 +140,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg // tdbUnrefPage(pPage); tdbPCacheRelease(pPager->pCache, pPage, &txn); tdbCommit(pPager->pEnv, &txn); + tdbPostCommit(pPager->pEnv, &txn); tdbTxnClose(&txn); } From 492dd68a3e79df9af2e9344e96de68679764dde5 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 8 Nov 2022 08:28:52 +0800 Subject: [PATCH 013/101] update --- tests/system-test/1-insert/delete_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 718a4497dc..aaad723b89 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor()) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' From 02d627722b46d4b2efe556f8f981697b837582fe Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 8 Nov 2022 09:04:05 +0800 Subject: [PATCH 014/101] test:modify case delete_data.py to three cases --- .../system-test/1-insert/delete_childtable.py | 232 +++++++++++++++++ .../1-insert/delete_normaltable.py | 224 +++++++++++++++++ tests/system-test/1-insert/delete_stable.py | 233 ++++++++++++++++++ tests/system-test/fulltest.sh | 3 + 4 files changed, 692 insertions(+) create mode 100644 tests/system-test/1-insert/delete_childtable.py create mode 100644 tests/system-test/1-insert/delete_normaltable.py create mode 100644 tests/system-test/1-insert/delete_stable.py diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py new file mode 100644 index 0000000000..584e88330c --- /dev/null +++ b/tests/system-test/1-insert/delete_childtable.py @@ -0,0 +1,232 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.ntbname = 'ntb' + self.rowNum = 5 + self.tbnum = 2 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1,stbname=''): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + tdSql.query(f'select count(*) from {stbname}') + if tb_num <= 1: + if len(tdSql.queryResult) != 0: + tdLog.exit('delete case failure!') + else: + tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(row_num-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ctb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,) + self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,i+1,self.stbname) + self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data) + self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}_{i}') + tdSql.execute(f'drop table {self.stbname}') + def run(self): + self.delete_data_ctb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ctb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/delete_normaltable.py b/tests/system-test/1-insert/delete_normaltable.py new file mode 100644 index 0000000000..66ef91d955 --- /dev/null +++ b/tests/system-test/1-insert/delete_normaltable.py @@ -0,0 +1,224 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),True) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 3 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1,stbname=''): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(row_num-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname) + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname) + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.ntbname}') + tdSql.execute(f'drop table {self.ntbname}') + tdSql.execute(f'drop database {self.dbname}') + def run(self): + self.delete_data_ntb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ntb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/delete_stable.py b/tests/system-test/1-insert/delete_stable.py new file mode 100644 index 0000000000..313b6ce731 --- /dev/null +++ b/tests/system-test/1-insert/delete_stable.py @@ -0,0 +1,233 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 3 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + for i in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num*tb_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows((row_num-1)*tb_num) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + for i in range(tb_num): + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((i+1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((row_num-i-1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((row_num-i)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(tb_num*(row_num - i-1)) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(tb_num*row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + def delete_data_stb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_error(self.stbname,col_name,col_type,self.base_data) + self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,self.tbnum) + self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,self.tbnum) + self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,self.tbnum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}') + tdSql.execute(f'drop table {self.stbname}') + tdSql.execute(f'drop database {self.dbname}') + def run(self): + self.delete_data_stb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_stb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 03ac0e0fb6..340e5abed3 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -210,6 +210,9 @@ python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/tb_100w_data_order.py # TD-20200 python3 ./test.py -f 1-insert/delete_data.py +python3 ./test.py -f 1-insert/delete_stable.py +python3 ./test.py -f 1-insert/delete_childtable.py +python3 ./test.py -f 1-insert/delete_normaltable.py python3 ./test.py -f 1-insert/keep_expired.py python3 ./test.py -f 2-query/join2.py From 9863879951d28bab71553f07bb9b425e01c52095 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 09:21:43 +0800 Subject: [PATCH 015/101] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 77 ++++++++----------------- 2 files changed, 26 insertions(+), 52 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index cd9f29978d..94361554d0 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -265,6 +265,7 @@ typedef struct SExchangeInfo { SLoadRemoteDataInfo loadInfo; uint64_t self; SLimitInfo limitInfo; + int64_t openedTs; // start exec time stamp } SExchangeInfo; typedef struct SScanInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b7c3eed069..00733fa21f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1847,40 +1847,41 @@ static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { return NULL; } -static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { - int32_t code = 0; - int64_t startTs = taosGetTimestampUs(); - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + +static int32_t getCompletedSources(const SArray* pArray) { + size_t total = taosArrayGetSize(pArray); int32_t completed = 0; - for (int32_t k = 0; k < totalSources; ++k) { - SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); + for (int32_t k = 0; k < total; ++k) { + SSourceDataInfo* p = taosArrayGet(pArray, k); if (p->status == EX_SOURCE_DATA_EXHAUSTED) { completed += 1; } } + return completed; +} + +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { + int32_t code = 0; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); + int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); if (completed == totalSources) { - setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); return; } while (1) { -// printf("1\n"); tsem_wait(&pExchangeInfo->ready); -// printf("2\n"); for (int32_t i = 0; i < totalSources; ++i) { SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { -// printf("========:%d is completed\n", i); continue; } -// printf("index:%d - status:%d\n", i, pDataInfo->status); if (pDataInfo->status != EX_SOURCE_DATA_READY) { -// printf("-----------%d, status:%d, continue\n", i, pDataInfo->status); continue; } @@ -1896,27 +1897,18 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; -// printf("%d completed, try next\n", i); - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu, + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, completed, i + 1, totalSources); + pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); taosMemoryFreeClear(pDataInfo->pRsp); - -// if (completed == totalSources) { -// return; -// } else { -// break; -// } - break; + break; } SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; int32_t index = 0; char* pStart = pRetrieveRsp->data; while (index++ < pRetrieveRsp->numOfBlocks) { - printf("results, numOfBLock: %d\n", pRetrieveRsp->numOfBlocks); SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); if (code != 0) { @@ -1927,25 +1919,16 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); -// int32_t completed = 0; if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - -// for (int32_t k = 0; k < totalSources; ++k) { -// SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); -// if (p->status == EX_SOURCE_DATA_EXHAUSTED) { -// completed += 1; -// } -// } - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, completed:%d try next %d/%" PRIzu, + ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - completed, i + 1, totalSources); + i + 1, totalSources); } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", @@ -1963,23 +1946,12 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn goto _error; } } - -// if (completed == totalSources) { -// setAllSourcesCompleted(pOperator, startTs); -// } - return; - } + } // end loop - int32_t completed = 0; - for (int32_t k = 0; k < totalSources; ++k) { - SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - if (completed == totalSources) { + int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (complete1 == totalSources) { + qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); return; } } @@ -2098,6 +2070,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { return code; } + pExchangeInfo->openedTs = taosGetTimestampUs(); } OPTR_SET_OPENED(pOperator); From fe685e723a8f6d78bb52699447456417721f0fa2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 09:32:12 +0800 Subject: [PATCH 016/101] docs: set monitor default value --- docs/en/14-reference/12-config/index.md | 2 +- docs/zh/14-reference/12-config/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index aadf563a63..bb5516ae70 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy | Applicable | Server only | | Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. | Value Range | 0: monitoring disabled, 1: monitoring enabled | -| Default | 0 | +| Default | 1 | ### monitorFqdn diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 54c362de95..145c5eed93 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -106,7 +106,7 @@ taos --dump-config | 适用范围 | 仅服务端适用 | | 含义 | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况，包括 CPU、内存、硬盘、网络带宽的监控记录，监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 | | 取值范围 | 0：关闭监控服务， 1：激活监控服务。 | -| 缺省值 | 0 | +| 缺省值 | 1 | ### monitorFqdn From 4b49dde403211dd7fa21f482873935d3bb789490 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 8 Nov 2022 09:54:19 +0800 Subject: [PATCH 017/101] fix(shell): adjust show result format --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d4f6e15b16..9bb02159f0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -226,7 +226,7 @@ void shellRunSingleCommandImp(char *command) { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected in set (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d row(s) affected(%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From 2d6c4ebda8a65dad9387c8d31a54758983c032bf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 8 Nov 2022 10:18:27 +0800 Subject: [PATCH 018/101] fix: state_window error --- source/libs/parser/src/parTranslater.c | 31 ++++++++++++-------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d3ff787f50..d3f343764b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3021,20 +3021,17 @@ static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSe return code; } -static EDealRes checkStateExpr(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - STranslateContext* pCxt = pContext; - SColumnNode* pCol = (SColumnNode*)pNode; - - int32_t type = pCol->node.resType.type; - if (!IS_INTEGER_TYPE(type) && type != TSDB_DATA_TYPE_BOOL && !IS_VAR_DATA_TYPE(type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); - } - if (COLUMN_TYPE_TAG == pCol->colType) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); - } +static int32_t checkStateExpr(STranslateContext* pCxt, SNode* pNode) { + int32_t type = ((SExprNode*)pNode)->resType.type; + if (!IS_INTEGER_TYPE(type) && type != TSDB_DATA_TYPE_BOOL && !IS_VAR_DATA_TYPE(type)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); } - return DEAL_RES_CONTINUE; + + if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); + } + + return TSDB_CODE_SUCCESS; } static bool hasPartitionByTbname(SNodeList* pPartitionByList) { @@ -3066,11 +3063,11 @@ static int32_t translateStateWindow(STranslateContext* pCxt, SSelectStmt* pSelec } SStateWindowNode* pState = (SStateWindowNode*)pSelect->pWindow; - nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = checkStateWindowForStream(pCxt, pSelect); + int32_t code = checkStateExpr(pCxt, pState->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = checkStateWindowForStream(pCxt, pSelect); } - return pCxt->errCode; + return code; } static int32_t translateSessionWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { From 30058f761940089b9e6a10bda7d25c697eee8d2d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 10:46:32 +0800 Subject: [PATCH 019/101] enh: optimize the error code when create and delete a database in a dropping state --- include/common/tmsg.h | 1 + include/util/taoserror.h | 22 ++++++++--- source/client/src/clientMsgHandler.c | 6 ++- source/common/src/tmsg.c | 2 + source/dnode/mnode/impl/src/mndDb.c | 50 +++++++++++++------------ source/dnode/mnode/impl/src/mndStream.c | 7 +--- source/util/src/tarray.c | 2 + source/util/src/terror.c | 2 + 8 files changed, 57 insertions(+), 35 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8aa85fa6b4..aabc84659c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -852,6 +852,7 @@ typedef struct { int16_t hashSuffix; int8_t hashMethod; SArray* pVgroupInfos; // Array of SVgroupInfo + int32_t errCode; } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, const SUseDbRsp* pRsp); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 447a90447a..a03dc7d9f9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -154,7 +154,7 @@ int32_t* taosGetErrno(); // mnode-sdb #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) +#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // internal #define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) #define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) #define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) @@ -218,14 +218,24 @@ int32_t* taosGetErrno(); // mnode-db #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) // +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) // #define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) +// #define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) // 2.x #define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) -#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) -#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) +#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) // +// #define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) // 2.x +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) // +#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) // internal +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) +// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x +// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x0393) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC_PARTITONSTAOS_DEF_ERROR_CODE(0, 0x0394) // 2.x +// #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x +#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) // mnode-node diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 938cc4e41d..1b42005a3c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -212,7 +212,11 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); if (strlen(usedbRsp.db) == 0) { - return TSDB_CODE_MND_DB_NOT_EXIST; + if (usedbRsp.errCode != 0) { + return usedbRsp.errCode; + } else { + return TSDB_CODE_APP_ERROR; + } } SName name = {0}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4eaa934676..c39d4b0e11 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2489,6 +2489,7 @@ int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1; } + if (tEncodeI32(pEncoder, pRsp->errCode) < 0) return -1; return 0; } @@ -2553,6 +2554,7 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) { taosArrayPush(pRsp->pVgroupInfos, &vgInfo); } + if (tDecodeI32(pDecoder, &pRsp->errCode) < 0) return -1; return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index aad2832aa6..992acd9dee 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -285,8 +285,17 @@ static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); - if (pDb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + if (pDb == NULL) { + if (terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { + terrno = TSDB_CODE_MND_DB_IN_CREATING; + } else if (terrno = TSDB_CODE_SDB_OBJ_DROPPING) { + terrno = TSDB_CODE_MND_DB_IN_DROPPING; + } else { + terrno = TSDB_CODE_APP_ERROR; + mFatal("db:%s, failed to acquire db since %s", db, terrstr()); + } } return pDb; } @@ -594,16 +603,22 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; goto _OVER; } - } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and response after trans finished", createReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; + } else { + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { + mInfo("db:%s, is creating and response after trans finished", createReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } else { + goto _OVER; + } + } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) { goto _OVER; - } else { + } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) { + // continue + } else { // TSDB_CODE_APP_ERROR goto _OVER; } - } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); @@ -786,7 +801,6 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } @@ -836,7 +850,6 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } @@ -1066,11 +1079,8 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { if (pDb == NULL) { if (dropReq.ignoreNotExists) { code = mndBuildDropDbRsp(pDb, &pReq->info.rspLen, &pReq->info.rsp, true); - goto _OVER; - } else { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto _OVER; } + goto _OVER; } if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { @@ -1197,10 +1207,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); if (usedbReq.vgVersion < vgVersion) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } + if (usedbRsp.pVgroupInfos == NULL) goto _OVER; mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; @@ -1209,16 +1216,13 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { } usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); code = 0; - - // no jump, need to construct rsp } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); usedbRsp.uid = usedbReq.dbId; usedbRsp.vgVersion = usedbReq.vgVersion; + usedbRsp.errCode = terrno; mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 36ba0aaf87..594c13f957 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -287,9 +287,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, memcpy(pObj->sourceDb, pCreate->sourceDB, TSDB_DB_FNAME_LEN); SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB); if (pSourceDb == NULL) { - /*ASSERT(0);*/ - mInfo("stream:%s failed to create, source db %s not exist", pCreate->name, pObj->sourceDb); - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mInfo("stream:%s failed to create, source db %s not exist since %s", pCreate->name, pObj->sourceDb, terrstr()); return -1; } pObj->sourceDbUid = pSourceDb->uid; @@ -298,8 +296,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pTargetDb = mndAcquireDbByStb(pMnode, pObj->targetSTbName); if (pTargetDb == NULL) { - mInfo("stream:%s failed to create, target db %s not exist", pCreate->name, pObj->targetDb); - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mInfo("stream:%s failed to create, target db %s not exist since %s", pCreate->name, pObj->targetDb, terrstr()); return -1; } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 309e6b30ae..95065972a3 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -26,12 +26,14 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pArray->size = 0; pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); return NULL; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f7e56f372f..0e6568d692 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -225,11 +225,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exis TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database in dropping status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") From 36450e6dd7d05a1c475f2b9e3d99b9e3dfb2ea20 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 11:08:14 +0800 Subject: [PATCH 020/101] enh: optimize the error code when create and delete a database in a dropping state --- source/client/src/clientMsgHandler.c | 3 ++- source/libs/catalog/inc/catalogInt.h | 3 ++- source/libs/parser/src/parTranslater.c | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 1b42005a3c..d46ff7fe5b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -175,7 +175,8 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; - if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); struct SCatalog* pCatalog = NULL; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8c699bb59b..4d3a3a1ab4 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -538,7 +538,8 @@ typedef struct SCtgOperation { (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) -#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST) +#define CTG_DB_NOT_EXIST(code) \ + (code == TSDB_CODE_MND_DB_NOT_EXIST || code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) #define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d3ff787f50..90bc5c0e3b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2181,7 +2181,8 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge if (TSDB_DB_NAME_T == pTargetName->type) { int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); - if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { code = TSDB_CODE_SUCCESS; } return code; @@ -2196,7 +2197,8 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge } else { taosArrayPush(*pVgroupList, &vgInfo); } - } else if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + } else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { code = TSDB_CODE_SUCCESS; } return code; From cac2764cd6debd559353d0a54a1150382134a1d8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 8 Nov 2022 11:15:13 +0800 Subject: [PATCH 021/101] docs: add examples of case expressions --- docs/en/12-taos-sql/06-select.md | 14 ++++++++++++++ docs/zh/12-taos-sql/06-select.md | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 5e96311494..0f4e260a84 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -334,6 +334,20 @@ If no comparison or condition is true, the result after ELSE is returned, or NUL The return type of the CASE expression is the result type of the first WHEN WHEN part, and the result type of the other WHEN WHEN parts and ELSE parts can be converted to them, otherwise TDengine will report an error. +### Examples + +A device has three status codes to display its status. The statements are as follows: + +```sql +SELECT CASE dev_status WHEN 1 THEN 'Running' WHEN 2 THEN 'Warning' WHEN 3 THEN 'Downtime' ELSE 'Unknown' END FROM dev_table; +``` + +The average voltage value of the smart meter is counted. When the voltage is less than 200 or more than 250, it is considered that the statistics is wrong, and the value is corrected to 220. The statement is as follows: + +```sql +SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) FROM meters; +``` + ## JOIN TDengine supports natural joins between supertables, between standard tables, and between subqueries. The difference between natural joins and inner joins is that natural joins require that the fields being joined in the supertables or standard tables must have the same name. Data or tag columns must be joined with the equivalent column in another table. diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 83af412fe1..9d4faae23a 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -325,14 +325,28 @@ CASE WHEN condition THEN result [WHEN condition THEN result ...] [ELSE result] E ``` ### 说明 + TDengine 通过 CASE 表达式让用户可以在 SQL 语句中使用 IF ... THEN ... ELSE 逻辑。 + 第一种 CASE 语法返回第一个 value 等于 compare_value 的 result，如果没有 compare_value 符合，则返回 ELSE 之后的 result，如果没有 ELSE 部分，则返回 NULL。 第二种语法返回第一个 condition 为真的 result。 如果没有 condition 符合，则返回 ELSE 之后的 result，如果没有 ELSE 部分，则返回 NULL。 CASE 表达式的返回类型为第一个 WHEN THEN 部分的 result 类型，其余 WHEN THEN 部分和 ELSE 部分，result 类型都需要可以向其转换，否则 TDengine 会报错。 +### 示例 +某设备有三个状态码，显示其状态，语句如下： + +```sql +SELECT CASE dev_status WHEN 1 THEN 'Running' WHEN 2 THEN 'Warning' WHEN 3 THEN 'Downtime' ELSE 'Unknown' END FROM dev_table; +``` + +统计智能电表的电压平均值，当电压小于 200 或大于 250 时认为是统计有误，修正其值为 220，语句如下： + +```sql +SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) FROM meters; +``` ## JOIN 子句 From 8f7c0b8c60c468ea67672d906642b4ce10b11775 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 8 Nov 2022 11:35:12 +0800 Subject: [PATCH 022/101] fix: state_window error --- source/libs/planner/test/planStateTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 6985bc8807..3dafaa8bf5 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -29,7 +29,7 @@ TEST_F(PlanStateTest, basic) { TEST_F(PlanStateTest, stateExpr) { useDb("root", "test"); - run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(CASE WHEN c1 > 10 THEN 1 ELSE 0 END)"); } TEST_F(PlanStateTest, selectFunc) { From 080f531b3241d37739b90e1ebee0011a9b1b4f9d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 8 Nov 2022 11:44:58 +0800 Subject: [PATCH 023/101] perf: optimize col data api --- source/common/src/tdataformat.c | 129 ++++++-------------------------- 1 file changed, 24 insertions(+), 105 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 513c54c7e9..12cb6a0026 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -923,7 +923,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { - if(!pTag || !pTagVal){ + if (!pTag || !pTagVal) { return false; } @@ -1164,31 +1164,27 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type); + memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + pColData->nData += tDataTypes[pColData->type].bytes; } + pColData->nVal++; _exit: return code; } static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_VALUE; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_NONE; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_NULL; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1216,16 +1212,11 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1268,11 +1259,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1290,9 +1277,8 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p return code; } static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1325,11 +1311,7 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1353,15 +1335,7 @@ static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *p return code; } -static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; - - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; -} +#define tColDataAppendValue40 tColDataPutValue static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1374,12 +1348,7 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1393,12 +1362,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1408,12 +1372,7 @@ static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *p SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1423,12 +1382,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *p SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1447,12 +1401,7 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1461,12 +1410,7 @@ static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *p if (code) return code; SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1485,12 +1429,7 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1499,12 +1438,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *p if (code) return code; SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1513,12 +1447,7 @@ static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1527,12 +1456,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1541,12 +1465,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 From 2621dec4aeefe14ee647cfc714a4e82c7b174b6b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 12:02:44 +0800 Subject: [PATCH 024/101] enh(query): improve the perf. --- source/libs/function/src/builtinsimpl.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1501bb6d67..d24393a766 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1555,6 +1555,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value +#if 0 if ((*val) == pData[i]) { continue; } @@ -1565,6 +1566,23 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } +#endif + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } } numOfElems += 1; From fd53afd70ca44948017eebe11d4a3464f7692fcb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 8 Nov 2022 12:23:22 +0800 Subject: [PATCH 025/101] fix: add installation folder to system path (#17943) * fix: add installation folder to system path * fix: add c:\tdengine in env path to issc setup --- packaging/release.bat | 2 +- packaging/tools/make_install.bat | 97 ++++++++++++++++++++++---------- packaging/tools/tdengine.iss | 23 ++++++++ 3 files changed, 91 insertions(+), 31 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index 4c82c5ead5..4ab7297f03 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -39,7 +39,7 @@ if not exist %work_dir%\debug\ver-%2-x86 ( md %work_dir%\debug\ver-%2-x86 ) cd %work_dir%\debug\ver-%2-x64 -call vcvarsall.bat x64 +rem #call vcvarsall.bat x64 cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 cmake --build . rd /s /Q C:\TDengine diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index f5d1e45690..c19519f9a1 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -1,4 +1,7 @@ @echo off + +for /F %%a in ('echo prompt $E ^| cmd') do set "ESC=%%a" + goto %1 :needAdmin @@ -11,60 +14,94 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 -set tagert_dir=C:\\TDengine +set target_dir=C:\\TDengine -if not exist %tagert_dir% ( - mkdir %tagert_dir% +if not exist %target_dir% ( + mkdir %target_dir% ) -if not exist %tagert_dir%\\cfg ( - mkdir %tagert_dir%\\cfg +if not exist %target_dir%\\cfg ( + mkdir %target_dir%\\cfg ) -if not exist %tagert_dir%\\include ( - mkdir %tagert_dir%\\include +if not exist %target_dir%\\include ( + mkdir %target_dir%\\include ) -if not exist %tagert_dir%\\driver ( - mkdir %tagert_dir%\\driver +if not exist %target_dir%\\driver ( + mkdir %target_dir%\\driver ) if not exist C:\\TDengine\\cfg\\taos.cfg ( - copy %source_dir%\\packaging\\cfg\\taos.cfg %tagert_dir%\\cfg\\taos.cfg > nul + copy %source_dir%\\packaging\\cfg\\taos.cfg %target_dir%\\cfg\\taos.cfg > nul ) if exist %binary_dir%\\test\\cfg\\taosadapter.toml ( - if not exist %tagert_dir%\\cfg\\taosadapter.toml ( - copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul + if not exist %target_dir%\\cfg\\taosadapter.toml ( + copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul ) ) -copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul -copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul -copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul -copy %binary_dir%\\build\\lib\\taos.lib %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\lib\\taos_static.lib %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\lib\\taos.dll %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\bin\\taos.exe %tagert_dir% > nul -copy %binary_dir%\\build\\bin\\taosd.exe %tagert_dir% > nul -copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul +copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul +copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul +copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul +copy %binary_dir%\\build\\lib\\taos.lib %target_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos_static.lib %target_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos.dll %target_dir%\\driver > nul +copy %binary_dir%\\build\\bin\\taos.exe %target_dir% > nul if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( - copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul + copy %binary_dir%\\build\\bin\\taosBenchmark.exe %target_dir% > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll.lib ( - copy %binary_dir%\\build\\lib\\taosws.dll.lib %tagert_dir%\\driver > nul + copy %binary_dir%\\build\\lib\\taosws.dll.lib %target_dir%\\driver > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll ( - copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver > nul - copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %tagert_dir%\\include > nul + copy %binary_dir%\\build\\lib\\taosws.dll %target_dir%\\driver > nul + copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %target_dir%\\include > nul ) if exist %binary_dir%\\build\\bin\\taosdump.exe ( - copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul -) -if exist %binary_dir%\\build\\bin\\taosadapter.exe ( - copy %binary_dir%\\build\\bin\\taosadapter.exe %tagert_dir% > nul + copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) -mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof +copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul +copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul + +if exist %binary_dir%\\build\\bin\\taosadapter.exe ( + copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul +) + +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close) + +echo. +echo Please manually remove C:\TDengine from your system PATH environment after you remove TDengine software +echo. +echo To start/stop TDengine with administrator privileges: %ESC%[92msc start/stop taosd %ESC%[0m + +if exist %binary_dir%\\build\\bin\\taosadapter.exe ( + echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m +) + +goto :eof + :hasAdmin + +sc query "taosd" && sc stop taosd && sc delete taosd +sc query "taosadapter" && sc stop taosadapter && sc delete taosd + copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul if exist C:\\TDengine\\driver\\taosws.dll ( copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul ) + sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND + +set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment" +for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\"') do ( + + rem // make addition persistent through reboots + reg add "%env%" /f /v Path /t REG_EXPAND_SZ /d "%%J;C:\TDengine" + + rem // apply change to the current process + for %%a in ("%%J;C:\TDengine") do path %%~a +) + +rem // use setx to set a temporary throwaway value to trigger a WM_SETTINGCHANGE +rem // applies change to new console windows without requiring a reboot +(setx /m foo bar & reg delete "%env%" /f /v foo) >NUL 2>NUL + diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index ec9c432092..981bee91b8 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -60,6 +60,29 @@ Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: i Source: {#MyAppSourceDir}{#MyAppExeName}; DestDir: "{app}"; Excludes: {#MyAppExcludeSource} ; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs + +[Registry] +Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ + ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\TDengine"; \ + Check: NeedsAddPath('C:\TDengine') + +[Code] +function NeedsAddPath(Param: string): boolean; +var + OrigPath: string; +begin + if not RegQueryStringValue(HKEY_LOCAL_MACHINE, + 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', + 'Path', OrigPath) + then begin + Result := True; + exit; + end; + { look for the path with leading and trailing semicolon } + { Pos() returns 0 if not found } + Result := Pos(';' + Param + ';', ';' + OrigPath + ';') = 0; +end; + [UninstallDelete] Name: {app}\driver; Type: filesandordirs Name: {app}\connector; Type: filesandordirs From abd95562a0a24ba7d94168e111fe0bde9190e079 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 12:36:28 +0800 Subject: [PATCH 026/101] enh: optimize the error code when create and delete a database in a dropping state --- source/dnode/mnode/impl/src/mndDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 992acd9dee..3d9b91dce8 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -290,7 +290,7 @@ SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { terrno = TSDB_CODE_MND_DB_IN_CREATING; - } else if (terrno = TSDB_CODE_SDB_OBJ_DROPPING) { + } else if (terrno == TSDB_CODE_SDB_OBJ_DROPPING) { terrno = TSDB_CODE_MND_DB_IN_DROPPING; } else { terrno = TSDB_CODE_APP_ERROR; From fc73d39add1072be7be929cf833d6d4b70aa258c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 8 Nov 2022 13:27:17 +0800 Subject: [PATCH 027/101] fix: fix compare invalid read issues --- source/libs/scalar/src/filter.c | 4 ++-- source/util/src/tcompare.c | 36 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 939be1809a..44df1ddda2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1169,7 +1169,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; - out.columnData->info.bytes = tDataTypes[type].bytes; + out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; //reserved space for simple_copy for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; @@ -1191,7 +1191,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); out.columnData->pData = NULL; } else { - void *data = taosMemoryCalloc(1, tDataTypes[type].bytes); + void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); //reserved space for simple_copy if (NULL == data) { FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 5d73a1464b..d84a3d25c6 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -249,7 +249,7 @@ int32_t compareJsonVal(const void *pLeft, const void *pRight) { } int32_t compareInt8Int16(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int16_t right = GET_INT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -257,7 +257,7 @@ int32_t compareInt8Int16(const void *pLeft, const void *pRight) { } int32_t compareInt8Int32(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int32_t right = GET_INT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -265,7 +265,7 @@ int32_t compareInt8Int32(const void *pLeft, const void *pRight) { } int32_t compareInt8Int64(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int64_t right = GET_INT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -273,7 +273,7 @@ int32_t compareInt8Int64(const void *pLeft, const void *pRight) { } int32_t compareInt8Float(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); float right = GET_FLOAT_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -281,7 +281,7 @@ int32_t compareInt8Float(const void *pLeft, const void *pRight) { } int32_t compareInt8Double(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); double right = GET_DOUBLE_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -289,7 +289,7 @@ int32_t compareInt8Double(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint8(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint8_t right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -297,7 +297,7 @@ int32_t compareInt8Uint8(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint16(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint16_t right = GET_UINT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -305,7 +305,7 @@ int32_t compareInt8Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint32_t right = GET_UINT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -313,7 +313,7 @@ int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint64_t right = GET_UINT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -321,7 +321,7 @@ int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { } int32_t compareInt16Int8(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int8_t right = GET_INT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -329,7 +329,7 @@ int32_t compareInt16Int8(const void *pLeft, const void *pRight) { } int32_t compareInt16Int32(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int32_t right = GET_INT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -337,7 +337,7 @@ int32_t compareInt16Int32(const void *pLeft, const void *pRight) { } int32_t compareInt16Int64(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int64_t right = GET_INT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -345,7 +345,7 @@ int32_t compareInt16Int64(const void *pLeft, const void *pRight) { } int32_t compareInt16Float(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); float right = GET_FLOAT_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -353,7 +353,7 @@ int32_t compareInt16Float(const void *pLeft, const void *pRight) { } int32_t compareInt16Double(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); double right = GET_DOUBLE_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -361,7 +361,7 @@ int32_t compareInt16Double(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint8(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint8_t right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -369,7 +369,7 @@ int32_t compareInt16Uint8(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint16(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint16_t right = GET_UINT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -377,7 +377,7 @@ int32_t compareInt16Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint32_t right = GET_UINT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -385,7 +385,7 @@ int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint64(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint64_t right = GET_UINT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; From fb7d45303bd473301c8820abc01c3632a6c31257 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 13:46:18 +0800 Subject: [PATCH 028/101] fix: add log for use db rsp --- source/client/src/clientMsgHandler.c | 9 +++++++++ source/dnode/mnode/impl/src/mndDnode.c | 3 +++ source/libs/qcom/src/querymsg.c | 9 +++++++++ 3 files changed, 21 insertions(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index d46ff7fe5b..e17e7e79c6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -220,6 +220,15 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } } + tscTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo* pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); + tscTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); + for (int32_t j = 0; j < pInfo->epSet.numOfEps; ++j) { + tscTrace("vgId:%d, index:%d epset:%s:%u", pInfo->vgId, j, pInfo->epSet.eps[j].fqdn, pInfo->epSet.eps[j].port); + } + } + SName name = {0}; tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 04f340b0ff..75ae6f467b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -376,6 +376,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { if (pVgroup->vnodeGid[vg].syncState != pVload->syncState || pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) { + mTrace("vgId:%d, role changed, old state:%s restored:%d new state:%s restored:%d", pVgroup->vgId, + syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore, + syncStr(pVload->syncState), pVload->syncRestore); pVgroup->vnodeGid[vg].syncState = pVload->syncState; pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore; roleChanged = true; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 953abd5956..181abc2247 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -306,6 +306,15 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_USEDB_OVER; } + qTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo *pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); + qTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); + for (int32_t j = 0; j < pInfo->epSet.numOfEps; ++j) { + qTrace("vgId:%d, index:%d epset:%s:%u", pInfo->vgId, j, pInfo->epSet.eps[j].fqdn, pInfo->epSet.eps[j].port); + } + } + code = queryBuildUseDbOutput(pOut, &usedbRsp); PROCESS_USEDB_OVER: From 032208e36ab272c425a6a74eed0ca42d52ba7f78 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 14:01:29 +0800 Subject: [PATCH 029/101] enh(query): optimize the query perf. --- source/libs/function/src/builtinsimpl.c | 199 +++++++++++++++--------- 1 file changed, 126 insertions(+), 73 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d24393a766..ab972bdab0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1273,14 +1273,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1304,14 +1310,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1335,14 +1347,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1366,14 +1384,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1399,14 +1423,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1430,14 +1460,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1461,14 +1497,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1492,14 +1534,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1524,14 +1572,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1554,7 +1608,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } pBuf->assign = true; } else { - // ignore the equivalent data value #if 0 if ((*val) == pData[i]) { continue; From d09351da68d894c174adfbaa037f3c7acefa8ced Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 8 Nov 2022 14:22:00 +0800 Subject: [PATCH 030/101] fix: filter merge issue --- source/libs/executor/src/executorimpl.c | 9 +++++++++ source/libs/scalar/src/filter.c | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b7c3eed069..6a93ab0efb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3384,6 +3384,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + return NULL; + } + STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { @@ -3404,6 +3409,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + return NULL; + } STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 939be1809a..2a6b85af9d 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -912,8 +912,8 @@ int32_t filterDetachCnfGroups(SArray *group, SArray *left, SArray *right) { if (taosArrayGetSize(left) <= 0) { if (taosArrayGetSize(right) <= 0) { - fltError("both groups are empty"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + fltDebug("both groups are empty"); + return TSDB_CODE_SUCCESS; } SFilterGroup *gp = NULL; From 7bba9996cf50f0899045824b77490c73761b118d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 8 Nov 2022 14:36:18 +0800 Subject: [PATCH 031/101] perf: optimize insert --- include/common/tdataformat.h | 13 ++++++++++--- source/common/src/tdataformat.c | 16 +--------------- source/common/src/trow.c | 20 +++++++++++--------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5c470982bb..71e260c001 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -74,9 +74,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SValue ================================ -int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type); -int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); -int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -283,6 +281,15 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + } else { + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} + #endif #ifdef __cplusplus diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8684426e5b..8c003066dc 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -56,7 +56,7 @@ typedef struct { #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) // SValue -int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { if (IS_VAR_DATA_TYPE(type)) { return tPutBinary(p, pValue->pData, pValue->nData); } else { @@ -65,20 +65,6 @@ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { } } -int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); - } else { - memcpy(&pValue->val, p, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - -int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { - // TODO - return 0; -} - // STSRow2 ======================================================================== static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ac4b5f86de..d39d3c501a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -73,8 +73,8 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } else if (tdValTypeIsNone(pVal->valType)) { printf("NONE "); return; - } - if(!pVal->val) { + } + if (!pVal->val) { ASSERT(0); printf("BadVal "); return; @@ -1083,13 +1083,15 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV } else if (tdValTypeIsNull(cv.valType)) { *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); } else { - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - value.nData = varDataLen(cv.val); - value.pData = varDataVal(cv.val); - } else { - tGetValue(cv.val, &value, pTColumn->type); - } + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pColVal->value.nData = varDataLen(cv.val); + pColVal->value.pData = varDataVal(cv.val); + } else { + memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); + } } } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index a03c9ab525..52b74aea3f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -607,7 +607,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { SColVal *tRowIterNext(SRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { - tsdbRowGetColVal(pIter->pRow, pIter->pTSchema, pIter->i, &pIter->colVal); + tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); pIter->i++; return &pIter->colVal; From f66eded1e181596f85a1ee3aa7b3e928d6e572e1 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 8 Nov 2022 14:40:44 +0800 Subject: [PATCH 032/101] fix(stream): state window, error result after deleting data --- include/libs/stream/streamState.h | 12 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 33 ++- source/libs/stream/src/streamState.c | 243 ++++++++++++------ tests/script/tsim/stream/session0.sim | 2 +- 5 files changed, 195 insertions(+), 97 deletions(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 225466a015..9443df5e14 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -60,19 +60,19 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number); -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); int32_t streamStateSessionClear(SStreamState* pState); -int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); -int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey); +int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); -SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); @@ -99,7 +99,9 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); -// char* streamStateSessionDump(SStreamState* pState); +#if 0 +char* streamStateSessionDump(SStreamState* pState); +#endif #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6d13048123..7faf75d5cd 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1438,7 +1438,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. SSessionKey startWin = {0}; - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin); + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin); if (IS_INVALID_SESSION_WIN_KEY(startWin)) { // window has been closed. continue; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7b6ed5b67d..e5b49acf0b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3550,7 +3550,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - int32_t code = streamStateSessionGetKey(pAggSup->pState, pKey, pKey); + int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -3561,10 +3561,11 @@ bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->session void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* pCurWin) { pCurWin->sessionWin.groupId = groupId; - pCurWin->sessionWin.win.skey = startTs - pAggSup->gap; - pCurWin->sessionWin.win.ekey = endTs + pAggSup->gap; + pCurWin->sessionWin.win.skey = startTs; + pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, &pCurWin->pOutputBuf, &size); + int32_t code = + streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; } else { @@ -3575,7 +3576,7 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, (const void**)&pWinInfo->pOutputBuf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3680,7 +3681,7 @@ SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* setSessionWinOutputInfo(pStUpdated, pNextWin); int32_t size = 0; pNextWin->sessionWin = pCurWin->sessionWin; - int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, (const void**)&pNextWin->pOutputBuf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(*pNextWin); } @@ -3894,9 +3895,11 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; - SStreamStateCur* pCur = streamStateSessionGetCur(pChAggSup->pState, pWinKey); - SResultRow* pResult = NULL; - SResultRow* pChResult = NULL; + SSessionKey chWinKey = *pWinKey; + chWinKey.win.ekey = chWinKey.win.skey; + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); + SResultRow* pResult = NULL; + SResultRow* pChResult = NULL; while (1) { SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; @@ -4112,6 +4115,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); +#if 0 + char* pBuf = streamStateSessionDump(pAggSup->pState); + qDebug("===stream===final session%s", pBuf); + taosMemoryFree(pBuf); +#endif + doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); @@ -4306,6 +4315,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); +#if 0 + char* pBuf = streamStateSessionDump(pAggSup->pState); + qDebug("===stream===semi session%s", pBuf); + taosMemoryFree(pBuf); +#endif + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index cf2ead364b..ccb0dd4a92 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -29,7 +29,7 @@ typedef struct SStateSessionKey { int64_t opNum; } SStateSessionKey; -static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { +static inline int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { if (pWin1->groupId > pWin2->groupId) { return 1; } else if (pWin1->groupId < pWin2->groupId) { @@ -45,6 +45,28 @@ static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pW return 0; } +static inline int sessionWinKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; + } + + if (pWin1->win.skey > pWin2->win.skey) { + return 1; + } else if (pWin1->win.skey < pWin2->win.skey) { + return -1; + } + + if (pWin1->win.ekey > pWin2->win.ekey) { + return 1; + } else if (pWin1->win.ekey < pWin2->win.ekey) { + return -1; + } + + return 0; +} + static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStateSessionKey* pWin1 = (SStateSessionKey*)pKey1; SStateSessionKey* pWin2 = (SStateSessionKey*)pKey2; @@ -55,7 +77,7 @@ static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* return -1; } - return sessionKeyCmpr(&pWin1->key, &pWin2->key); + return sessionWinKeyCmpr(&pWin1->key, &pWin2->key); } static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { @@ -400,7 +422,6 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key SStateKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -426,7 +447,6 @@ SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -452,7 +472,6 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -496,33 +515,18 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, cons return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); } -SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSessionKey* key) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL); - - int32_t c = -2; - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c); - if (c != 0) { - streamStateFreeCur(pCur); - return NULL; - } - pCur->number = pState->number; - return pCur; -} - int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); + SSessionKey resKey = *key; void* tmp = NULL; - if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); + if (code == 0) { + *key = resKey; *pVal = tdbRealloc(NULL, *pVLen); memcpy(*pVal, tmp, *pVLen); - streamStateFreeCur(pCur); - return 0; } streamStateFreeCur(pCur); - return -1; + return code; } int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { @@ -544,7 +548,6 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -558,6 +561,34 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons return pCur; } +SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + if (c <= 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { @@ -572,7 +603,6 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -586,13 +616,13 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess return pCur; } -int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) { if (!pCur) { return -1; } - const SStateSessionKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + SStateSessionKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { return -1; } if (pKTmp->opNum != pCur->number) { @@ -606,15 +636,15 @@ int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, c } int32_t streamStateSessionClear(SStreamState* pState) { - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; - streamStateSessionPut(pState, &key, NULL, 0); - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pState, &key); + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); while (1) { SSessionKey delKey = {0}; void* buf = NULL; int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, buf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); if (code == 0) { + ASSERT(size > 0); memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); } else { @@ -623,63 +653,106 @@ int32_t streamStateSessionClear(SStreamState* pState) { streamStateCurNext(pState, pCur); } streamStateFreeCur(pCur); - streamStateSessionDel(pState, &key); return 0; } -SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - SSessionKey resKey = *key; - while (1) { +int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return -1; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return -1; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + streamStateFreeCur(pCur); + return -1; + } + + SSessionKey resKey = *key; + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; + } + + if (c > 0) { + streamStateCurNext(pState, pCur); + code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; + } + } else if (c < 0) { streamStateCurPrev(pState, pCur); - SSessionKey tmpKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); - if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { - resKey = tmpKey; - } else { - break; + code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; } } + streamStateFreeCur(pCur); - return streamStateSessionGetRanomCur(pState, &resKey); + return -1; } -int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - SSessionKey resKey = *key; - int32_t res = -1; - while (1) { - SSessionKey tmpKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); - if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { - res = 0; - resKey = tmpKey; - streamStateCurPrev(pState, pCur); - } else { - break; +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, + int32_t* pVLen) { + // todo refactor + int32_t res = 0; + SSessionKey originKey = *key; + SSessionKey searchKey = *key; + searchKey.win.skey = key->win.skey - gap; + searchKey.win.ekey = key->win.ekey + gap; + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); + if (!tmp) { + return -1; + } + + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + if (code == 0) { + if (sessionRangeKeyCmpr(&searchKey, key) == 0) { + memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); + goto _end; + } + streamStateCurNext(pState, pCur); + } else { + *key = originKey; + streamStateFreeCur(pCur); + pCur = streamStateSessionSeekKeyNext(pState, key); + } + + code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + if (code == 0) { + if (sessionRangeKeyCmpr(&searchKey, key) == 0) { + memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); + goto _end; } } - *curKey = resKey; + + *key = originKey; + res = 1; + memset(tmp, 0, valSize); + +_end: + + *pVal = tmp; streamStateFreeCur(pCur); return res; } -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { - // todo refactor - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - int32_t size = *pVLen; - void* tmp = NULL; - *pVal = tdbRealloc(NULL, size); - memset(*pVal, 0, size); - if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { - memcpy(*pVal, tmp, *pVLen); - streamStateFreeCur(pCur); - return 0; - } - streamStateFreeCur(pCur); - return 1; -} - int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { // todo refactor @@ -692,16 +765,18 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } @@ -712,11 +787,12 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch pCur = streamStateSessionSeekKeyNext(pState, key); } - code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } } @@ -746,8 +822,11 @@ char* streamStateSessionDump(SStreamState* pState) { tdbTbcMoveToFirst(pCur->pCur); SSessionKey key = {0}; - int32_t code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0); + void* buf = NULL; + int32_t bufSize = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &key, &buf, &bufSize); if (code != 0) { + streamStateFreeCur(pCur); return NULL; } @@ -762,12 +841,14 @@ char* streamStateSessionDump(SStreamState* pState) { key = (SSessionKey){0}; code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0); if (code != 0) { + streamStateFreeCur(pCur); return dumpBuf; } len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey); len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey); len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId); } + streamStateFreeCur(pCur); return dumpBuf; } #endif diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index afae8ef5da..5e95428e0a 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -201,7 +201,7 @@ if $loop_count == 10 then endi if $rows != 1 then - print ======$rows + print =====rows=$rows goto loop2 endi From b0a54d3fab495963f8c253b5332b9e17b5ad77eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 14:54:42 +0800 Subject: [PATCH 033/101] enh(query): improve the perf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ source/libs/function/src/builtinsimpl.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bfde5b3076..41d50f119d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -985,6 +985,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); + ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // make sure it is aligned to 8bit + // null value exists, check one-by-one if (pData->flag != HAS_VALUE) { for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ab972bdab0..00a148ea25 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -906,6 +906,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_FLOAT: { float* plist = (float*)pCol->pData; + float val = 0; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -913,8 +914,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { numOfElem += 1; pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; + val += plist[i]; } + pAvgRes->sum.dsum = val; break; } From f7fe64aa6ddb92c4de13fb2376593c0411610655 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Tue, 8 Nov 2022 15:15:05 +0800 Subject: [PATCH 034/101] fix(taosAdapter): fix cors config (#17958) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c507ae2536..79d54f522e 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 8c3d57d + GIT_TAG 0d5663d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2bf72657b850fde02bf5ed82d1e03cd8cf46f10f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 8 Nov 2022 16:50:27 +0800 Subject: [PATCH 035/101] test:modify two options that downloads the installPackages --- packaging/MPtestJenkinsfile | 47 ++++++++++++++++++++----------------- packaging/testpackage.sh | 35 +++++++++++++++++---------- 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 5b79345916..6e0350d03b 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -13,7 +13,7 @@ def sync_source(branch_name) { git branch git pull || git pull git log | head -n 20 - git submodule update --init --recursive + git clean -fxd ''' return 1 } @@ -37,19 +37,24 @@ def build_run() { pipeline { agent none parameters { + choice( + name: 'sourcePath', + choices: ['web', 'nas'], + description: 'choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + ) string ( name:'version', - defaultValue:'3.0.0.1', + defaultValue:'3.0.1.6', description: 'release version number,eg: 3.0.0.1 or 3.0.0.' ) string ( name:'baseVersion', - defaultValue:'3.0.0.1', + defaultValue:'3.0.1.6', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) string ( name:'toolsVersion', - defaultValue:'2.1.2', + defaultValue:'2.2.7', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) string ( @@ -62,7 +67,7 @@ pipeline { WORK_DIR = '/var/lib/jenkins/workspace' TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' - BRANCH_NAME = 'test/chr/TD-14699' + BRANCH_NAME = 'test/TD-20194' TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" @@ -104,17 +109,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -127,17 +132,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py dpkg -r tdengine ''' @@ -152,17 +157,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -175,17 +180,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py sudo rpm -e tdengine ''' @@ -199,7 +204,7 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -215,7 +220,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.21 ''' } @@ -227,7 +232,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.24 ''' } @@ -241,7 +246,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.21 ''' } diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 5b9a12179d..1dcd73f03e 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -6,6 +6,8 @@ version=$2 originPackageName=$3 originversion=$4 testFile=$5 +# sourcePath:web/nas +sourcePath=$6 subFile="taos.tar.gz" # Color setting @@ -71,16 +73,22 @@ fi function wgetFile { file=$1 +versionPath=$2 +sourceP=$3 -if [ ! -f ${file} ];then - echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" - wget https://www.taosdata.com/assets-download/3.0/${file} -else - echoColor YD "${file} already exists and use new file " +if [ -f ${file} ];then + echoColor YD "${file} already exists ,it will delete it and download it again " rm -rf ${file} - echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" - wget https://www.taosdata.com/assets-download/3.0/${file} fi + +if [ ${sourcePath} = 'web' ];then + echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} +elif [ ${sourcePath} = 'nas' ];then + echoColor BD "====download====:scp root@${nasServerIP}:${sourcePath}/${file} ." + scp root@${nasServerIP}:${sourcePath}/${file} . +fi + } function newPath { @@ -142,8 +150,9 @@ if [ -d ${installPath}/${tdPath} ] ;then fi echoColor G "===== download installPackage =====" -cd ${installPath} && wgetFile ${packgeName} -cd ${oriInstallPath} && wgetFile ${originPackageName} + cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} + cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} +fi cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . @@ -193,7 +202,7 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath} if [ ! -f {originPackageName} ];then echoColor YD "download base installPackage" - wgetFile ${originPackageName} + wgetFile ${originPackageName} ${originversion} ${sourcePath} fi echoColor YD "unzip the base installation package" echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} @@ -238,13 +247,13 @@ cd ${installPath} if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-x64.tar.gz . - tar xf taosTools-2.1.3-Linux-x64.tar.gz + wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web + tar xf taosTools-2.1.3-Linux-x64.tar.gz cd taosTools-2.1.3 && bash install-taostools.sh elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-arm64.tar.gz . + wgetFile taosTools-2.1.3-Linux-arm64.tar.gz v2.1.3 web tar xf taosTools-2.1.3-Linux-arm64.tar.gz cd taosTools-2.1.3 && bash install-taostools.sh fi From f85d66f5eee9846157a5d100c96e96fb7319f6ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 16:59:00 +0800 Subject: [PATCH 036/101] enh(query): opt query perf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 ++- source/libs/function/src/builtinsimpl.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 41d50f119d..a20742c55f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -985,7 +985,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); - ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // make sure it is aligned to 8bit + // make sure it is aligned to 8bit + ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // null value exists, check one-by-one if (pData->flag != HAS_VALUE) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 00a148ea25..541e2e1954 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3096,7 +3096,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); + TSKEY cts = *(TSKEY*) colDataGetData(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveCurrentVal(pCtx, i, cts, type, data); pResInfo->numOfRes = 1; From df2c9819a663bee0319646af38155ba970ded52d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 8 Nov 2022 17:01:09 +0800 Subject: [PATCH 037/101] test:modify two options that downloads the installPackages --- packaging/testpackage.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 1dcd73f03e..ba8b95fc0d 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -75,18 +75,19 @@ function wgetFile { file=$1 versionPath=$2 sourceP=$3 - +nasServerIP="192.168.1.131" +packagePath="/nas/TDengine/v${versionPath}/" if [ -f ${file} ];then echoColor YD "${file} already exists ,it will delete it and download it again " rm -rf ${file} fi -if [ ${sourcePath} = 'web' ];then +if [ ${sourceP} = 'web' ];then echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" wget https://www.taosdata.com/assets-download/3.0/${file} -elif [ ${sourcePath} = 'nas' ];then - echoColor BD "====download====:scp root@${nasServerIP}:${sourcePath}/${file} ." - scp root@${nasServerIP}:${sourcePath}/${file} . +elif [ ${sourceP} = 'nas' ];then + echoColor BD "====download====:scp root@${nasServerIP}:${packagePath}/${file} ." + scp root@${nasServerIP}:${packagePath}/${file} . fi } From e238e6e946559fce355a6bc79c1b505b72ac3322 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 8 Nov 2022 17:09:31 +0800 Subject: [PATCH 038/101] test:modify two options that downloads the installPackages --- packaging/testpackage.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index ba8b95fc0d..aca9ceae75 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -76,7 +76,7 @@ file=$1 versionPath=$2 sourceP=$3 nasServerIP="192.168.1.131" -packagePath="/nas/TDengine/v${versionPath}/" +packagePath="/nas/TDengine/v${versionPath}/community" if [ -f ${file} ];then echoColor YD "${file} already exists ,it will delete it and download it again " rm -rf ${file} @@ -151,9 +151,9 @@ if [ -d ${installPath}/${tdPath} ] ;then fi echoColor G "===== download installPackage =====" - cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} - cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} -fi +cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} +cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} + cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . From 1dbb086045d2bedde98fd5eef4acb70ed7f92998 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 17:17:24 +0800 Subject: [PATCH 039/101] enh: check vgroup status changes at regular intervals --- include/common/tmsg.h | 2 + include/libs/catalog/catalog.h | 1 + source/common/src/tmsg.c | 4 ++ source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 18 ++++++-- source/dnode/mnode/impl/src/mndDnode.c | 16 ++++--- source/dnode/mnode/impl/src/mndMain.c | 61 ++++++++++++++++++++++++++ 7 files changed, 94 insertions(+), 9 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index aabc84659c..961eefb8a5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -838,6 +838,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; // ms } SUseDbReq; int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); @@ -853,6 +854,7 @@ typedef struct { int8_t hashMethod; SArray* pVgroupInfos; // Array of SVgroupInfo int32_t errCode; + int64_t stateTs; // ms } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, const SUseDbRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3902224f0b..7717d3bccc 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -124,6 +124,7 @@ typedef struct SDbVgVersion { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SDbVgVersion; typedef struct STbSVersion { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c39d4b0e11..524da05739 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2238,6 +2238,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTable) < 0) return -1; + if (tEncodeI64(&encoder, pReq->stateTs) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2254,6 +2255,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTable) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->stateTs) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2490,6 +2492,7 @@ int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { } if (tEncodeI32(pEncoder, pRsp->errCode) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->stateTs) < 0) return -1; return 0; } @@ -2555,6 +2558,7 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) { } if (tDecodeI32(pDecoder, &pRsp->errCode) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->stateTs) < 0) return -1; return 0; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 74a92c9fcd..04ac5aba49 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -321,6 +321,7 @@ typedef struct { int32_t vgVersion; SDbCfg cfg; SRWLatch lock; + int64_t stateTs; } SDbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 3d9b91dce8..f2a4462bf5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1182,6 +1182,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); pRsp->uid = pDb->uid; pRsp->vgVersion = pDb->vgVersion; + pRsp->stateTs = pDb->stateTs; pRsp->vgNum = taosArrayGetSize(pRsp->pVgroupInfos); pRsp->hashMethod = pDb->cfg.hashMethod; pRsp->hashPrefix = pDb->cfg.hashPrefix; @@ -1234,6 +1235,8 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { goto _OVER; } + mDebug("db:%s, process usedb req vgVersion:%d stateTs:%" PRId64 ", rsp vgVersion:%d stateTs:%" PRId64, + usedbReq.db, usedbReq.vgVersion, usedbReq.stateTs, usedbRsp.vgVersion, usedbRsp.stateTs); code = 0; } } @@ -1290,13 +1293,19 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { - mInfo("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable /* && + pDbVgVersion->stateTs == pDb->stateTs */) { + mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, + pDb->vgVersion, pDb->stateTs, numOfTable); mndReleaseDb(pMnode, pDb); continue; } else { - mInfo("db:%s, vgroup version changed from %d to %d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, - pDb->vgVersion); + mInfo("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, changed to vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, + pDb->vgVersion, pDb->stateTs, numOfTable); } usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); @@ -1310,6 +1319,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); usedbRsp.uid = pDb->uid; usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.stateTs = pDb->stateTs; usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); usedbRsp.hashMethod = pDb->cfg.hashMethod; usedbRsp.hashPrefix = pDb->cfg.hashPrefix; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 75ae6f467b..7eb9d4fb1f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" +#include "mndDb.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -356,7 +357,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode->lastAccessTime = curMs; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, - pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); + pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) { SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v); @@ -376,9 +377,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { if (pVgroup->vnodeGid[vg].syncState != pVload->syncState || pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) { - mTrace("vgId:%d, role changed, old state:%s restored:%d new state:%s restored:%d", pVgroup->vgId, - syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore, - syncStr(pVload->syncState), pVload->syncRestore); + mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d", + pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore, + syncStr(pVload->syncState), pVload->syncRestore); pVgroup->vnodeGid[vg].syncState = pVload->syncState; pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore; roleChanged = true; @@ -387,7 +388,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } } if (roleChanged) { - // notify scheduler role has changed + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb->stateTs != curMs) { + mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, pDb->stateTs, curMs); + pDb->stateTs = curMs; + } + mndReleaseDb(pMnode, pDb); } } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1b2d85bd29..476ce267d4 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -139,6 +139,63 @@ static void mndIncreaseUpTime(SMnode *pMnode) { } } +static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + bool roleChanged = false; + for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { + if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) { + if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_ERROR) { + mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0", + pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore); + pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_ERROR; + pVgroup->vnodeGid[vg].syncRestore = 0; + roleChanged = true; + } + break; + } + } + + if (roleChanged) { + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb->stateTs != curMs) { + mInfo("db:%s, stateTs changed by offline check, old newTs:%" PRId64 " newTs:%" PRId64, pDb->name, pDb->stateTs, + curMs); + pDb->stateTs = curMs; + } + mndReleaseDb(pMnode, pDb); + } + + sdbRelease(pSdb, pVgroup); + } +} + +static void mndCheckDnodeOffline(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + int64_t curMs = taosGetTimestampMs(); + + void *pIter = NULL; + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + bool online = mndIsDnodeOnline(pDnode, curMs); + if (!online) { + mInfo("dnode:%d, in offline state", pDnode->id); + mndSetVgroupOffline(pMnode, pDnode->id, curMs); + } + + sdbRelease(pSdb, pDnode); + } +} + static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -174,6 +231,10 @@ static void *mndThreadFp(void *param) { if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } + + if (sec % (tsStatusInterval * 5) == 0) { + mndCheckDnodeOffline(pMnode); + } } return NULL; From c4f0c340f9d5690ab17d78f5130a816f7abb0775 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 8 Nov 2022 17:25:28 +0800 Subject: [PATCH 040/101] test:modify two options that downloads the installPackages --- packaging/MPtestJenkinsfile | 4 ++-- packaging/testpackage.sh | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 6e0350d03b..5dc8024f29 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -39,7 +39,7 @@ pipeline { parameters { choice( name: 'sourcePath', - choices: ['web', 'nas'], + choices: ['nas','web'], description: 'choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' ) string ( @@ -67,7 +67,7 @@ pipeline { WORK_DIR = '/var/lib/jenkins/workspace' TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' - BRANCH_NAME = 'test/TD-20194' + BRANCH_NAME = '3.0' TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index aca9ceae75..b0e33835dd 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -248,14 +248,19 @@ cd ${installPath} if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web - tar xf taosTools-2.1.3-Linux-x64.tar.gz + if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then + wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web + tar xf taosTools-2.1.3-Linux-x64.tar.gz + fi cd taosTools-2.1.3 && bash install-taostools.sh elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-arm64.tar.gz v2.1.3 web - tar xf taosTools-2.1.3-Linux-arm64.tar.gz + if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then + wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web + tar xf taosTools-2.1.3-Linux-arm64.tar.gz + fi + cd taosTools-2.1.3 && bash install-taostools.sh fi From 5a5d2c448b9113a66686746c1b19325b8d248a22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 17:26:31 +0800 Subject: [PATCH 041/101] enh: process some msg in mnode read queue --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 976f2cfd8b..ec4f42c847 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -91,7 +91,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -102,7 +102,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -116,7 +116,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -127,7 +127,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -151,7 +151,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; From 1c9b0b38d82a4b799df1a7ec438f170b41497724 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 8 Nov 2022 17:35:38 +0800 Subject: [PATCH 042/101] fix: set operator subplan type error --- source/libs/planner/src/planSpliter.c | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0af8f5075e..249ba1815d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -66,16 +66,36 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SLogicNode* pNod return pSubplan; } -static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, ESubplanType subplanType) { +static bool splHasScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return true; + } + + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) { + return true; + } + return splHasScan((SLogicNode*)pChild); + } + + return false; +} + +static void splSetSubplanType(SLogicSubplan* pSubplan) { + pSubplan->subplanType = splHasScan(pSubplan->pNode) ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE; +} + +static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; } pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; - pSubplan->subplanType = subplanType; pSubplan->pNode = pNode; pNode->pParent = NULL; + splSetSubplanType(pSubplan); return pSubplan; } @@ -1204,7 +1224,7 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl SNode* pChild = NULL; FOREACH(pChild, pSplitNode->pChildren) { - SLogicSubplan* pNewSubplan = splCreateSubplan(pCxt, (SLogicNode*)pChild, pUnionSubplan->subplanType); + SLogicSubplan* pNewSubplan = splCreateSubplan(pCxt, (SLogicNode*)pChild); code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, (SNode*)pNewSubplan); if (TSDB_CODE_SUCCESS == code) { REPLACE_NODE(NULL); @@ -1390,10 +1410,9 @@ static int32_t insertSelectSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SLogicSubplan* pNewSubplan = NULL; SNodeList* pSubplanChildren = info.pSubplan->pChildren; - ESubplanType subplanType = info.pSubplan->subplanType; int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, SUBPLAN_TYPE_MODIFY); if (TSDB_CODE_SUCCESS == code) { - pNewSubplan = splCreateSubplan(pCxt, info.pQueryRoot, subplanType); + pNewSubplan = splCreateSubplan(pCxt, info.pQueryRoot); if (NULL == pNewSubplan) { code = TSDB_CODE_OUT_OF_MEMORY; } From 6e1a2b1e54ddca0372e1995a896e6d9cb4d9ff1d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 17:38:09 +0800 Subject: [PATCH 043/101] enh(query): improve the perf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++++----- source/libs/function/src/builtinsimpl.c | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a20742c55f..77e6dc5649 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -957,11 +957,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - if (asc) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); - } else { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { - colDataAppendInt64(pColData, rowIndex++, &pBlockData->aTSKEY[j]); + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); + + if (!asc) { // reverse the array list + int32_t mid = remain / 2; + TSKEY* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[i]; + pts[j] = pts[remain - j - 1]; + pts[remain - j - 1] = t; } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 541e2e1954..1f2fe5c658 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3103,8 +3103,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } #endif + + // save selectivity value for column consisted of all null values if (numOfElems == 0) { - // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); } SET_VAL(pResInfo, numOfElems, 1); From cc3e97da36bbe3bea44428f24c4a0ff4320bff21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 17:41:29 +0800 Subject: [PATCH 044/101] fix(query): fix an typo. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 77e6dc5649..f185e31de6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -959,11 +959,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); + + // todo: opt perf by extract the loop if (!asc) { // reverse the array list - int32_t mid = remain / 2; + int32_t mid = remain >> 1u; TSKEY* pts = (int64_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[i]; + int64_t t = pts[j]; pts[j] = pts[remain - j - 1]; pts[remain - j - 1] = t; } From 6c0f46ff03adcb3cf13bf6abeb340918020ad3f1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 18:10:26 +0800 Subject: [PATCH 045/101] fix: the vnode was created to the wrong dnode when network was misconfigured --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 51213a6ab3..2265f58060 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -191,6 +191,16 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, replica:%d id:%d fqdn:%s port:%u", req.vgId, i, req.replicas[i].id, req.replicas[i].fqdn, req.replicas[i].port); } + + SReplica *pReplica = &req.replicas[req.selfIndex]; + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + return -1; + } + vmGenerateVnodeCfg(&req, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { @@ -285,6 +295,15 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + SReplica *pReplica = &alterReq.replicas[alterReq.selfIndex]; + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", alterReq.vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + return -1; + } + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); @@ -341,6 +360,12 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vgId = dropReq.vgId; dDebug("vgId:%d, start to drop vnode", vgId); + if (dropReq.dnodeId != pMgmt->pData->dnodeId) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d not matched with local dnode", dropReq.vgId, dropReq.dnodeId); + return -1; + } + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); From 3c6246f85f91bd2164c72973d18fcf503b185f73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 18:18:33 +0800 Subject: [PATCH 046/101] TS-2015 --- source/libs/transport/src/transCli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 98677c64d3..c3abfa971e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -474,7 +474,7 @@ void* destroyConnPool(void* pool) { } static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { - char key[32] = {0}; + char key[TSDB_FQDN_LEN + 64] = {0}; CONN_CONSTRUCT_HASH_KEY(key, ip, port); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); @@ -525,7 +525,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->status = ConnInPool; if (conn->list == NULL) { - char key[32] = {0}; + char key[TSDB_FQDN_LEN + 64] = {0}; CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port); tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); conn->list = taosHashGet((SHashObj*)pool, key, strlen(key)); From 663138ba1ef0959b9a77bb199070445d3c764db2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 18:33:57 +0800 Subject: [PATCH 047/101] enh(query): optimize the query perf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 129 +++++++++++++++++++------ 1 file changed, 99 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f185e31de6..2ffbe78803 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -891,7 +891,7 @@ static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int or } } -int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { +static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; bool asc = ASCENDING_TRAVERSE(pReader->order); @@ -908,6 +908,26 @@ int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SData return endPos; } +static void copyPrimaryTsCol(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + + // todo: opt perf by extract the loop + // reverse the array list + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + } +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -947,30 +967,17 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } endIndex += step; - int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); - if (remain > pReader->capacity) { // output buffer check - remain = pReader->capacity; + int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (dumpedRows > pReader->capacity) { // output buffer check + dumpedRows = pReader->capacity; } + int32_t i = 0; int32_t rowIndex = 0; - int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); - - - // todo: opt perf by extract the loop - if (!asc) { // reverse the array list - int32_t mid = remain >> 1u; - TSKEY* pts = (int64_t*)pColData->pData; - for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; - pts[j] = pts[remain - j - 1]; - pts[remain - j - 1] = t; - } - } - + copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -985,18 +992,80 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; } else if (pData->cid == pColData->info.colId) { if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); } else { - if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { - uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; - memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); + if (IS_MATHABLE_TYPE(pColData->info.type)) { + + uint8_t* p = NULL; + if (asc) { + p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + p = pData->pData + tDataTypes[pData->type].bytes * startIndex; + } // make sure it is aligned to 8bit ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + + if (!asc) { + switch(pColData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + { + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + int32_t mid = dumpedRows >> 1u; + int8_t* pts = (int8_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + int32_t mid = dumpedRows >> 1u; + int16_t* pts = (int16_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t mid = dumpedRows >> 1u; + int32_t* pts = (int32_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + } + } // null value exists, check one-by-one if (pData->flag != HAS_VALUE) { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { uint8_t v = tColDataGetBitValue(pData, j); if (v == 0 || v == 1) { colDataSetNull_f(pColData->nullbitmap, rowIndex); @@ -1004,7 +1073,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } } } else { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); } @@ -1014,7 +1083,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } } @@ -1022,12 +1091,12 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, i); - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } - pResBlock->info.rows = remain; - pDumpInfo->rowIndex += step * remain; + pResBlock->info.rows = dumpedRows; + pDumpInfo->rowIndex += step * dumpedRows; // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { @@ -1046,7 +1115,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, dumpedRows, unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; From 0183fe6e7376f498945f3a1d5c46575064cb9ad0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 18:36:24 +0800 Subject: [PATCH 048/101] fix: crash if db not exist while process status msg --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndMain.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7eb9d4fb1f..521f924fad 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -389,7 +389,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } if (roleChanged) { SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); - if (pDb->stateTs != curMs) { + if (pDb != NULL && pDb->stateTs != curMs) { mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, pDb->stateTs, curMs); pDb->stateTs = curMs; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 476ce267d4..dcfc046a1e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -164,7 +164,7 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) if (roleChanged) { SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); - if (pDb->stateTs != curMs) { + if (pDb != NULL && pDb->stateTs != curMs) { mInfo("db:%s, stateTs changed by offline check, old newTs:%" PRId64 " newTs:%" PRId64, pDb->name, pDb->stateTs, curMs); pDb->stateTs = curMs; From ecf3e2c00064e1ca7cc985d5989253d0655293b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 18:44:28 +0800 Subject: [PATCH 049/101] enh(query): optimize the perf. --- source/libs/function/src/builtinsimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1f2fe5c658..198892a99e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3096,9 +3096,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - TSKEY cts = *(TSKEY*) colDataGetData(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); + TSKEY* cts = (TSKEY*) colDataGetData(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || pInfo->ts < (*cts)) { + doSaveCurrentVal(pCtx, i, *cts, type, data); pResInfo->numOfRes = 1; } } From 495bfe932a7412c4587b32e890aebcb41b84c797 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 18:56:44 +0800 Subject: [PATCH 050/101] fix invalid free --- source/libs/scheduler/src/schTask.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index dbdccff302..7e5b3faedb 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -439,6 +439,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(code); @@ -446,6 +448,8 @@ _return: taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -942,7 +946,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1115,7 +1119,7 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); From dc2c419f2212c36b26a183fc7187fc6d10d46ef1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 8 Nov 2022 19:01:35 +0800 Subject: [PATCH 051/101] enh(query): optimize the query perf. --- source/libs/function/src/builtinsimpl.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 198892a99e..5f3d9c138e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3088,6 +3088,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } #else + int64_t* pts = (int64_t*)pInput->pPTS->pData; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; @@ -3096,9 +3097,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - TSKEY* cts = (TSKEY*) colDataGetData(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || pInfo->ts < (*cts)) { - doSaveCurrentVal(pCtx, i, *cts, type, data); + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); pResInfo->numOfRes = 1; } } @@ -3285,11 +3286,13 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { } } #else + + int64_t* pts = (int64_t*)pInput->pPTS->pData; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); - numOfElems++; + TSKEY cts = pts[i]; + numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); pResInfo->numOfRes = 1; From 7b30a6ee23f9572a12f018cb6c1b54a6e220a99b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 20:45:11 +0800 Subject: [PATCH 052/101] free unfinished ahandle while quit --- include/libs/qcom/query.h | 2 ++ include/libs/transport/trpc.h | 4 +++ source/client/src/clientEnv.c | 3 ++- source/libs/qcom/src/queryUtil.c | 6 +++++ source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 31 +++++++++++++++--------- 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 651b379851..c6f7b5cf2e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -199,6 +199,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +void destroyAhandle(void* ahandle); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 53edcd4fee..8a0cccc71f 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -72,6 +72,7 @@ typedef struct SRpcMsg { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -97,6 +98,9 @@ typedef struct SRpcInit { // set up timeout for particular msg RpcTfp tfp; + // destroy client ahandle; + RpcDfp dfp; + void *parent; } SRpcInit; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb25a595af..bb93e4d934 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -140,12 +140,13 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; - // rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cf064881c2..6eadf80e3d 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -146,6 +146,12 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { } taosMemoryFreeClear(pMsgBody); } +void destroyAhandle(void *ahandle) { + SMsgSendInfo *pSendInfo = ahandle; + if (pSendInfo == NULL) return; + + destroySendMsgInfo(pSendInfo); +} int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index b9167501e2..3b7182f983 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d3db4879d1..feb55985de 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -53,6 +53,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 21444018cd..42dc65abea 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -143,6 +143,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); +static void cliReleaseUnfinishedMsg(SCliConn* conn); // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); @@ -163,17 +164,6 @@ static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); -static void cliReleaseUnfinishedMsg(SCliConn* conn) { - for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { - SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); - if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { - if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { - conn->ctx.freeFunc(msg->ctx->ahandle); - } - } - destroyCmsg(msg); - } -} #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -266,6 +256,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { static void* cliWorkThread(void* arg); +static void cliReleaseUnfinishedMsg(SCliConn* conn) { + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); + if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { + if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { + conn->ctx.freeFunc(msg->ctx->ahandle); + continue; + } + if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); + pTransInst->destroyFp(msg->ctx->ahandle); + } + } + destroyCmsg(msg); + } +} bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; From 1ed333140b8ffd638d0a228b46c31f4e95c2acef Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 8 Nov 2022 22:09:58 +0800 Subject: [PATCH 053/101] fix invalid free --- source/libs/transport/src/transCli.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 42dc65abea..bae9295749 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -265,9 +265,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { conn->ctx.freeFunc(msg->ctx->ahandle); - continue; - } - if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + } else if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); pTransInst->destroyFp(msg->ctx->ahandle); } From a4e96ca8332152faa36d922d68459b203f9b04f1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 22:43:12 +0800 Subject: [PATCH 054/101] refact: adjust sync resp mgr --- source/libs/sync/inc/syncRespMgr.h | 9 +-- source/libs/sync/inc/syncTools.h | 3 - source/libs/sync/src/syncMain.c | 109 +++------------------------ source/libs/sync/src/syncRespMgr.c | 114 +++++++++++++++-------------- 4 files changed, 74 insertions(+), 161 deletions(-) diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index 22e1005e5c..9026ecb66e 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -41,13 +41,12 @@ typedef struct SSyncRespMgr { SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl); void syncRespMgrDestroy(SSyncRespMgr *pObj); -int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub); -int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index); -int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); -int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); +uint64_t syncRespMgrAdd(SSyncRespMgr *pObj, const SRespStub *pStub); +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t seq); +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub); +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *pInfo); void syncRespClean(SSyncRespMgr *pObj); void syncRespCleanRsp(SSyncRespMgr *pObj); -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 932432d1f8..193579030f 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -26,9 +26,6 @@ typedef struct SRaftId { SyncGroupId vgId; } SRaftId; -// for compatibility, the same as syncPropose -int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak); - // ------------------ for debug ------------------- void syncRpcMsgPrint(SRpcMsg* pMsg); void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0d7306e4fd..c42e75a9c0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -452,11 +452,6 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { return ret; } -int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - int32_t ret = syncPropose(rid, pMsg, isWeak); - return ret; -} - SSyncState syncGetState(int64_t rid) { SSyncState state = {.state = TAOS_SYNC_STATE_ERROR}; @@ -558,109 +553,27 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho return lastIndex; } -#if 0 -SyncTerm syncGetMyTerm(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return TAOS_SYNC_STATE_ERROR; - } - ASSERT(rid == pSyncNode->rid); - SyncTerm term = pSyncNode->pRaftStore->currentTerm; - - syncNodeRelease(pSyncNode); - return term; -} - -SyncIndex syncGetLastIndex(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return SYNC_INDEX_INVALID; - } - ASSERT(rid == pSyncNode->rid); - SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); - - syncNodeRelease(pSyncNode); - return lastIndex; -} - -SyncIndex syncGetCommitIndex(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return SYNC_INDEX_INVALID; - } - ASSERT(rid == pSyncNode->rid); - SyncIndex cmtIndex = pSyncNode->commitIndex; - - syncNodeRelease(pSyncNode); - return cmtIndex; -} - -SyncGroupId syncGetVgId(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return TAOS_SYNC_STATE_ERROR; - } - ASSERT(rid == pSyncNode->rid); - SyncGroupId vgId = pSyncNode->vgId; - - syncNodeRelease(pSyncNode); - return vgId; -} - -void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - memset(pEpSet, 0, sizeof(*pEpSet)); - return; - } - ASSERT(rid == pSyncNode->rid); - pEpSet->numOfEps = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); - pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; - (pEpSet->numOfEps)++; - sInfo("vgId:%d, sync get epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; - sInfo("vgId:%d, sync get epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); - - syncNodeRelease(pSyncNode); -} -#endif - void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - memset(pEpSet, 0, sizeof(*pEpSet)); - return; - } - pEpSet->numOfEps = 0; + + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) return; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); - pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; - (pEpSet->numOfEps)++; - sInfo("vgId:%d, sync get retry epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, - pEpSet->eps[i].port); + SEp* pEp = &pEpSet->eps[i]; + tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); + pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + pEpSet->numOfEps++; + sInfo("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port); } if (pEpSet->numOfEps > 0) { pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps; } - sInfo("vgId:%d, sync get retry epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); + sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse); syncNodeRelease(pSyncNode); } -static void syncGetAndDelRespRpc(SSyncNode* pSyncNode, uint64_t index, SRpcHandleInfo* pInfo) { - SRespStub stub; - int32_t ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub); - if (ret == 1) { - *pInfo = stub.rpcMsg.info; - } - - sTrace("vgId:%d, get seq:%" PRIu64 " rpc handle:%p", pSyncNode->vgId, index, pInfo->handle); -} - int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { @@ -2759,7 +2672,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde .flag = flag, }; - syncGetAndDelRespRpc(ths, cbMeta.seqNum, &rpcMsg.info); + syncRespMgrGetAndDel(ths->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, &cbMeta); } } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 56d5e944b8..de8f1927ae 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -13,21 +13,22 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncRespMgr.h" #include "syncRaftEntry.h" #include "syncRaftStore.h" SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { - SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); + SSyncRespMgr *pObj = taosMemoryCalloc(1, sizeof(SSyncRespMgr)); if (pObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pObj, 0, sizeof(SSyncRespMgr)); pObj->pRespHash = taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - ASSERT(pObj->pRespHash != NULL); + if (pObj->pRespHash == NULL) return NULL; + pObj->ttl = ttl; pObj->data = data; pObj->seqNum = 0; @@ -38,93 +39,84 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { void syncRespMgrDestroy(SSyncRespMgr *pObj) { if (pObj != NULL) { - taosThreadMutexLock(&(pObj->mutex)); + taosThreadMutexLock(&pObj->mutex); taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexDestroy(&(pObj->mutex)); taosMemoryFree(pObj); } } -int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +uint64_t syncRespMgrAdd(SSyncRespMgr *pObj, const SRespStub *pStub) { + taosThreadMutexLock(&pObj->mutex); - uint64_t keyCode = ++(pObj->seqNum); - taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); + uint64_t seq = ++(pObj->seqNum); + int32_t code = taosHashPut(pObj->pRespHash, &seq, sizeof(uint64_t), pStub, sizeof(SRespStub)); + sNTrace(pObj->data, "save message handle:%p, type:%s seq:%" PRIu64 " code:0x%x", pStub->rpcMsg.info.handle, + TMSG_INFO(pStub->rpcMsg.msgType), seq, code); - sNTrace(pObj->data, "save message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), - keyCode, pStub->rpcMsg.info.handle); - taosThreadMutexUnlock(&(pObj->mutex)); - return keyCode; + taosThreadMutexUnlock(&pObj->mutex); + return seq; } -int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t seq) { + taosThreadMutexLock(&pObj->mutex); - taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + int32_t code = taosHashRemove(pObj->pRespHash, &seq, sizeof(seq)); + sNTrace(pObj->data, "remove message handle, seq:%" PRIu64 " code:%d", seq, code); - taosThreadMutexUnlock(&(pObj->mutex)); - return 0; + taosThreadMutexUnlock(&pObj->mutex); + return code; } -int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub) { + taosThreadMutexLock(&pObj->mutex); - void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); + SRespStub *pTmp = taosHashGet(pObj->pRespHash, &seq, sizeof(uint64_t)); if (pTmp != NULL) { memcpy(pStub, pTmp, sizeof(SRespStub)); + sNTrace(pObj->data, "get message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), seq, + pStub->rpcMsg.info.handle); - sNTrace(pObj->data, "get message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), - index, pStub->rpcMsg.info.handle); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object } - taosThreadMutexUnlock(&(pObj->mutex)); + + taosThreadMutexUnlock(&pObj->mutex); return 0; // get none object } -int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *pInfo) { + taosThreadMutexLock(&pObj->mutex); - void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); - if (pTmp != NULL) { - memcpy(pStub, pTmp, sizeof(SRespStub)); + SRespStub *pStub = taosHashGet(pObj->pRespHash, &seq, sizeof(uint64_t)); + if (pStub != NULL) { + *pInfo = pStub->rpcMsg.info; + sNTrace(pObj->data, "get-and-del message handle:%p, type:%s seq:%" PRIu64, pStub->rpcMsg.info.handle, + TMSG_INFO(pStub->rpcMsg.msgType), seq); + taosHashRemove(pObj->pRespHash, &seq, sizeof(uint64_t)); - sNTrace(pObj->data, "get-and-del message handle, type:%s seq:%" PRIu64 " handle:%p", - TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle); - taosHashRemove(pObj->pRespHash, &index, sizeof(index)); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object } - taosThreadMutexUnlock(&(pObj->mutex)); + + taosThreadMutexUnlock(&pObj->mutex); return 0; // get none object } -void syncRespCleanRsp(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, -1, true); - taosThreadMutexUnlock(&(pObj->mutex)); -} - -void syncRespClean(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, pObj->ttl, false); - taosThreadMutexUnlock(&(pObj->mutex)); -} - -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { +static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; int sum = 0; SSyncNode *pSyncNode = pObj->data; - SArray *delIndexArray = taosArrayInit(0, sizeof(uint64_t)); - ASSERT(delIndexArray != NULL); - sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); + SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t)); + if (delIndexArray == NULL) return; + sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); while (pStub) { size_t len; - void * key = taosHashGetKey(pStub, &len); + void *key = taosHashGetKey(pStub, &len); uint64_t *pSeqNum = (uint64_t *)key; sum++; @@ -149,15 +141,15 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { pStub->rpcMsg.contLen = 0; // TODO: and make rpcMsg body, call commit cb - // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta); pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; if (pStub->rpcMsg.info.handle != NULL) { - tmsgSendRsp(&(pStub->rpcMsg)); + tmsgSendRsp(&pStub->rpcMsg); } } - pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub); + pStub = taosHashIterate(pObj->pRespHash, pStub); } int32_t arraySize = taosArrayGetSize(delIndexArray); @@ -170,3 +162,15 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { } taosArrayDestroy(delIndexArray); } + +void syncRespCleanRsp(SSyncRespMgr *pObj) { + taosThreadMutexLock(&pObj->mutex); + syncRespCleanByTTL(pObj, -1, true); + taosThreadMutexUnlock(&pObj->mutex); +} + +void syncRespClean(SSyncRespMgr *pObj) { + taosThreadMutexLock(&pObj->mutex); + syncRespCleanByTTL(pObj, pObj->ttl, false); + taosThreadMutexUnlock(&pObj->mutex); +} From 1c8a2d696dd038ba213926b2252170b2d0d89f79 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 22:59:05 +0800 Subject: [PATCH 055/101] fix: send response on enqueue msg failed --- source/libs/sync/src/syncMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0d7306e4fd..c6dc62415e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -719,8 +719,11 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { sNTrace(pSyncNode, "propose message, type:%s", TMSG_INFO(pMsg->msgType)); ret = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); if (ret != 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + if (terrno != 0) ret = terrno; sError("vgId:%d, failed to enqueue msg since %s", pSyncNode->vgId, terrstr()); + syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + SRpcMsg rsp = {.code = ret, .info = pMsg->info}; + tmsgSendRsp(&rsp); } } From bf1aca1b9ea8adff3d8c59c1effe913a95c28204 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 8 Nov 2022 23:00:14 +0800 Subject: [PATCH 056/101] fix: send response on enqueue msg failed --- source/libs/sync/src/syncMain.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c6dc62415e..19015c00b9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -719,11 +719,8 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { sNTrace(pSyncNode, "propose message, type:%s", TMSG_INFO(pMsg->msgType)); ret = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); if (ret != 0) { - if (terrno != 0) ret = terrno; sError("vgId:%d, failed to enqueue msg since %s", pSyncNode->vgId, terrstr()); syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); - SRpcMsg rsp = {.code = ret, .info = pMsg->info}; - tmsgSendRsp(&rsp); } } From 68f2f9211691be18871802d48233115a730edec1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 09:07:21 +0800 Subject: [PATCH 057/101] fix(query): fix the syntax errors. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4914deed30..0f970b6830 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1117,7 +1117,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, dumpedRows, unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 52c038c500..7d662f8784 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1897,7 +1897,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->numOfRows == 0) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu, + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); taosMemoryFreeClear(pDataInfo->pRsp); From e3aabacf9a8a6f7e3da1cb65e7454cc3622b3ad4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 09:47:21 +0800 Subject: [PATCH 058/101] fix(query): fix error in min/max functions. --- source/libs/function/src/builtinsimpl.c | 46 ++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 58e41d5abb..e3a76ad33a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -911,7 +911,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_FLOAT: { float* plist = (float*)pCol->pData; - float val = 0; +// float val = 0; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -919,9 +919,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { numOfElem += 1; pAvgRes->count += 1; - val += plist[i]; + pAvgRes->sum.dsum += plist[i]; } - pAvgRes->sum.dsum = val; +// pAvgRes->sum.dsum = val; break; } @@ -1282,14 +1282,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1319,14 +1319,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1356,14 +1356,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1393,14 +1393,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1432,14 +1432,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1469,14 +1469,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1506,14 +1506,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1543,14 +1543,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1581,14 +1581,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // ignore the equivalent data value // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); @@ -1629,14 +1629,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { #endif // NOTE: An faster version to avoid one additional comparison with FPU. if (isMinFunc) { // min - if (*val < pData[i]) { + if (*val > pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else { // max - if (*val > pData[i]) { + if (*val < pData[i]) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); From 0c427b5f5b63408f4d764b096b46ebe9b8882e76 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 10:39:32 +0800 Subject: [PATCH 059/101] refactor: do some internal refactor. --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 173 +++++++++++++----------- source/libs/function/src/builtinsimpl.c | 1 + 4 files changed, 97 insertions(+), 81 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 71e260c001..2eda2f66cc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -130,7 +130,7 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) void tColDataClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); -uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal); +uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8c003066dc..73b887342c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1557,7 +1557,7 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); } -uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) { +uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { uint8_t v; switch (pColData->flag) { case HAS_NONE: diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0f970b6830..9dab1e1d33 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -910,7 +910,7 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData return endPos; } -static void copyPrimaryTsCol(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, +static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, int32_t dumpedRows, bool asc) { if (asc) { memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); @@ -930,6 +930,97 @@ static void copyPrimaryTsCol(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpIn } } +// a faster version of copy procedure. +static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + uint8_t* p = NULL; + if (asc) { + p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + p = pData->pData + tDataTypes[pData->type].bytes * startIndex; + } + + int32_t step = asc? 1:-1; + + // make sure it is aligned to 8bit + ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + + // 1. copy data in a batch model + memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + + // 2. reverse the array list in case of descending order scan data block + if (!asc) { + switch(pColData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + { + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + int32_t mid = dumpedRows >> 1u; + int8_t* pts = (int8_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + int32_t mid = dumpedRows >> 1u; + int16_t* pts = (int16_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t mid = dumpedRows >> 1u; + int32_t* pts = (int32_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + } + } + + // 3. if the null value exists, check items one-by-one + if (pData->flag != HAS_VALUE) { + int32_t rowIndex = 0; + + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { + uint8_t v = tColDataGetBitValue(pData, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + pColData->hasNull = true; + } + } + } +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -997,84 +1088,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colDataAppendNNULL(pColData, 0, dumpedRows); } else { if (IS_MATHABLE_TYPE(pColData->info.type)) { - - uint8_t* p = NULL; - if (asc) { - p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; - } else { - int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; - p = pData->pData + tDataTypes[pData->type].bytes * startIndex; - } - - // make sure it is aligned to 8bit - ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); - memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); - - if (!asc) { - switch(pColData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - { - int32_t mid = dumpedRows >> 1u; - int64_t* pts = (int64_t*)pColData->pData; - for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; - pts[j] = pts[dumpedRows - j - 1]; - pts[dumpedRows - j - 1] = t; - } - break; - } - - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: { - int32_t mid = dumpedRows >> 1u; - int8_t* pts = (int8_t*)pColData->pData; - for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; - pts[j] = pts[dumpedRows - j - 1]; - pts[dumpedRows - j - 1] = t; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: { - int32_t mid = dumpedRows >> 1u; - int16_t* pts = (int16_t*)pColData->pData; - for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; - pts[j] = pts[dumpedRows - j - 1]; - pts[dumpedRows - j - 1] = t; - } - break; - } - - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: { - int32_t mid = dumpedRows >> 1u; - int32_t* pts = (int32_t*)pColData->pData; - for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; - pts[j] = pts[dumpedRows - j - 1]; - pts[dumpedRows - j - 1] = t; - } - break; - } - } - } - - // null value exists, check one-by-one - if (pData->flag != HAS_VALUE) { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { - uint8_t v = tColDataGetBitValue(pData, j); - if (v == 0 || v == 1) { - colDataSetNull_f(pColData->nullbitmap, rowIndex); - } - } - } - } else { + copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + } else { // varchar/nchar type for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e3a76ad33a..eb30945109 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3022,6 +3022,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } #endif + if (numOfElems == 0) { // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); From 459bed5f86fd60bc1d615d40fe04a3d08e7c6c1c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 10:41:53 +0800 Subject: [PATCH 060/101] enh(query): improve the query perf. --- source/libs/function/src/builtinsimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eb30945109..6eb57c1a18 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3007,6 +3007,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } #else + int64_t* pts = (int64_t*) pInput->pPTS->pData; for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; @@ -3015,7 +3016,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); + TSKEY cts = pts[i]; if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); pResInfo->numOfRes = 1; From 372d26f74ba19bfe8088bcacd52faf7d7eadb366 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 11:24:11 +0800 Subject: [PATCH 061/101] refactor(sync): delete assert, call FpCommitCb when multi replica --- source/libs/sync/src/syncMain.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 14823be098..b4a3d30f7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -225,7 +225,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { sError("sync begin snapshot error"); return -1; } - + int32_t code = 0; if (syncNodeIsMnode(pSyncNode)) { @@ -390,7 +390,7 @@ bool syncIsReadyForRead(int64_t rid) { if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry( - pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); + pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); if (code == 0 && pEntry != NULL) { if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { ready = true; @@ -2462,11 +2462,36 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd } else { syncEntryDestory(pEntry); } + return -1; } else { // del resp mgr, call FpCommitCb - ASSERT(0); + + SRpcMsg rpcMsg = {0}; + syncClientRequest2RpcMsg(pMsg, &rpcMsg); + + SFsmCbMeta cbMeta = { + .index = pEntry->index, + .lastConfigIndex = SYNC_INDEX_INVALID, + .isWeak = pEntry->isWeak, + .code = -1, + .state = ths->state, + .seqNum = pEntry->seqNum, + .term = pEntry->term, + .currentTerm = ths->pRaftStore->currentTerm, + .flag = 0, + }; + + syncRespMgrGetAndDel(ths->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, &cbMeta); + + if (h) { + taosLRUCacheRelease(ths->pLogStore->pCache, h, false); + } else { + syncEntryDestory(pEntry); + } + return -1; } } From 9533ad41987a9d16ade49c934dae3f17502dbd91 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 9 Nov 2022 11:24:45 +0800 Subject: [PATCH 062/101] feat: taosbenchmark support query fails then continue (#17980) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e0d5250d84..d18d85171d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 0fb640b + GIT_TAG a921bd4 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From da13891ca491b581efa2fee4dc8da6c60b178450 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 9 Nov 2022 09:29:59 +0800 Subject: [PATCH 063/101] fix(stream):partition operator memory leak --- source/libs/executor/src/groupoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 891eb6e7a4..2a7d1b089b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -918,6 +918,8 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { blockDataDestroy(pResBlock); } } + taosArrayDestroy(pParInfo->rowIds); + pParInfo->rowIds = NULL; blockDataUpdateTsWindow(pDest, pInfo->tsColIndex); pDest->info.groupId = pParInfo->groupId; pOperator->resultInfo.totalRows += pDest->info.rows; @@ -1016,6 +1018,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->tbnameCalSup); cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); + taosHashCleanup(pInfo->pPartitions); taosMemoryFreeClear(param); } From 97fb25372ef66344bac952c253ee40281dd2cc7b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 9 Nov 2022 12:58:44 +0800 Subject: [PATCH 064/101] fix(ci):modify ci test --- tests/script/tsim/stream/basic1.sim | 428 ++++++++++++++++------------ 1 file changed, 253 insertions(+), 175 deletions(-) diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index b20e2e3592..bb6604687c 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -23,443 +23,516 @@ sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); -sleep 1000 + +$loop_count = 0 + +loop0: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; -if $rows != 4 then - print ======$rows +$loop_count = $loop_count + 1 +if $loop_count == 20 then return -1 endi +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop0 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop0 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop0 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop0 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop0 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop0 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop0 endi if $data13 != 2 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop0 endi if $data14 != 2 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop0 endi if $data15 != 3 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop0 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 500 + +$loop_count = 0 +loop1: +sleep 200 + sql select * from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + print count(*) , count(d) , sum(a) , max(b) , min(c) print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 if $rows != 4 then print ======$rows - return -1 + goto loop1 endi # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop1 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop1 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop1 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop1 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop1 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop1 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop1 endi if $data13 != 12 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop1 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop1 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop1 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223002,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop2: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 2 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop2 endi if $data12 != 2 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop2 endi if $data13 != 24 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop2 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop2 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop2 endi sql insert into t1 values(1648791223003,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop3: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop3 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop3 endi if $data13 != 36 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop3 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop3 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop3 endi sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); -sleep 100 + +$loop_count = 0 +loop4: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop4 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop4 endi if $data13 != 6 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop4 endi if $data14 != 3 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop4 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop4 endi sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); -sleep 100 + +$loop_count = 0 +loop5: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 2 if $data21 != 2 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop5 endi if $data22 != 2 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop5 endi if $data23 != 6 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop5 endi if $data24 != 2 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop5 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop5 endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop6: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 0 if $data01 != 4 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop6 endi if $data02 != 4 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop6 endi if $data03 != 50 then - print ======$data03 != 50 - return -1 + print =====data03=$data03 != 50 + goto loop6 endi if $data04 != 20 then - print ======$data04 != 20 - return -1 + print =====data04=$data04 != 20 + goto loop6 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop6 endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop7: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 4 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop7 endi if $data12 != 4 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop7 endi if $data13 != 46 then - print ======$data13 != 46 - return -1 + print =====data13=$data13 != 46 + goto loop7 endi if $data14 != 20 then - print ======$data14 != 20 - return -1 + print =====data14=$data14 != 20 + goto loop7 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop7 endi # row 2 if $data21 != 4 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop7 endi if $data22 != 4 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop7 endi if $data23 != 15 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop7 endi if $data24 != 4 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop7 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop7 endi sql create database test2 vgroups 1; @@ -479,11 +552,11 @@ sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2 $loop_count = 0 -loop0: -sleep 300 +loop8: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -491,7 +564,7 @@ sql select * from streamt; if $rows != 4 then print =====rows=$rows - goto loop0 + goto loop8 endi sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6,6.0) t5 values(1648791213000,7,7,7,7.0); @@ -499,11 +572,11 @@ sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6 $loop_count = 0 -loop1: -sleep 300 +loop9: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -511,51 +584,51 @@ sql select * from streamt order by c4 desc; if $rows != 5 then print =====rows=$rows - goto loop1 + goto loop9 endi # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop1 + goto loop9 endi if $data02 != 7 then print =====data02=$data02 - goto loop1 + goto loop9 endi # row 1 if $data11 != 1 then print =====data11=$data11 - goto loop1 + goto loop9 endi if $data12 != 6 then print =====data12=$data12 - goto loop1 + goto loop9 endi # row 2 if $data21 != 1 then print =====data21=$data21 - goto loop1 + goto loop9 endi if $data22 != 5 then print =====data22=$data22 - goto loop1 + goto loop9 endi sql insert into t1 values(1648791213000,8,8,8,8.0); $loop_count = 0 -loop2: -sleep 300 +loop10: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -564,28 +637,29 @@ sql select * from streamt order by c4 desc; # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop2 + goto loop10 endi if $data02 != 8 then print =====data02=$data02 - goto loop2 + goto loop10 endi $loop_count = 0 -loop3: -sleep 300 +loop11: +sleep 200 + +sql select count(*) from streamt3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi -sql select count(*) from streamt3; # row 0 if $data00 != 5 then print =====data00=$data00 - goto loop3 + goto loop11 endi #max,min selectivity @@ -601,25 +675,26 @@ sql insert into ts1 values(1648791222001,2,2,3); sleep 50 $loop_count = 0 -loop3: +loop12: +sleep 200 + sql select * from streamtST3; -sleep 300 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi # row 0 if $data02 != 1 then print =====data02=$data02 - goto loop3 + goto loop12 endi # row 1 if $data12 != 2 then print =====data12=$data12 - goto loop3 + goto loop12 endi @@ -629,19 +704,22 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); +$loop_count = 0 +loop13: sleep 200 + sql select * from streamt4; # row 0 if $rows != 0 then print =====rows=$rows - return -1 + goto loop13 endi sql insert into t1 values(1648791213000,6,2,3,1.0); $loop_count = 0 -loop4: +loop14: sleep 200 sql select * from streamt4; @@ -652,13 +730,13 @@ endi if $data01 != 1 then print =====data01=$data01 - goto loop4 + goto loop14 endi sql insert into t1 values(1648791213000,2,2,3,1.0); $loop_count = 0 -loop5: +loop15: sleep 200 sql select * from streamt4; @@ -669,7 +747,7 @@ endi if $rows != 0 then print =====rows=$rows - goto loop5 + goto loop15 endi From b48752686e489a86d2597b91e4d8dcde16616efb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 13:45:46 +0800 Subject: [PATCH 065/101] refactor: do some internal refactor. --- include/common/tdatablock.h | 2 +- source/client/src/clientMsgHandler.c | 3 +- source/common/src/tdatablock.c | 31 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 3 +- source/dnode/mnode/impl/src/mndShow.c | 3 +- source/dnode/vnode/src/tq/tqExec.c | 4 +- source/libs/command/src/command.c | 3 +- source/libs/command/src/explain.c | 3 +- source/libs/executor/inc/executorimpl.h | 8 +- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/executor/src/executor.c | 21 + source/libs/executor/src/executorimpl.c | 628 +----------------- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/joinoperator.c | 7 +- source/libs/executor/src/projectoperator.c | 5 +- source/libs/executor/src/scanoperator.c | 17 +- source/libs/executor/src/sortoperator.c | 6 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 22 +- source/libs/stream/src/streamDispatch.c | 6 +- 21 files changed, 87 insertions(+), 697 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fdbb2c5123..502ba10d33 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -244,7 +244,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e17e7e79c6..591d469d7c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -442,8 +442,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d0121aa15..536cbed33e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2197,7 +2197,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return rname.ctbShortName; } -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { + int32_t dataLen = 0; + // todo extract method int32_t* version = (int32_t*)data; *version = 1; @@ -2238,7 +2240,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = blockDataGetSerialMetaSize(numOfCols); + dataLen = blockDataGetSerialMetaSize(numOfCols); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -2255,26 +2257,23 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ } data += metaSize; - (*dataLen) += metaSize; + dataLen += metaSize; - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); } - *actualLen = *dataLen; + *actualLen = dataLen; *groupId = pBlock->info.groupId; - ASSERT(*dataLen > 0); - uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); + ASSERT(dataLen > 0); + + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols); + + return dataLen; } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 85a09b79fd..8a8561161b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -307,8 +307,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 637a3818e6..b0af98b933 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -303,8 +303,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a6e8767a4d..48c14bc758 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -27,9 +27,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->completed = 1; pRetrieve->numOfRows = htonl(pBlock->info.rows); - // TODO enable compress - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 76c84b3be9..1c2d7e1f66 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -39,8 +39,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 80a524496c..915dc08c14 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1610,8 +1610,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - int32_t len = 0; - blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ced668cf37..77915624dd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -842,10 +842,8 @@ typedef struct SJoinOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -void doDestroyExchangeOperatorInfo(void* param); - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain); +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -881,6 +879,8 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); +void doDestroyExchangeOperatorInfo(void* param); + void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 76866eedb7..41fcc44f97 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -102,7 +102,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, NULL, destroyLastrowScanOperator, NULL); + createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index bc4ab9c468..3d9757c96f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1cf01c8661..1aa9a3c613 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1106,3 +1106,24 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } return 0; } + +void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; + assert(pMsg->info.ahandle != NULL); + + SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; + + if (pMsg->contLen > 0) { + buf.pData = taosMemoryCalloc(1, pMsg->contLen); + if (buf.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(buf.pData, pMsg->pCont, pMsg->contLen); + } + } + + pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); + rpcFreeCont(pMsg->pCont); + destroySendMsgInfo(pSendInfo); +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7d662f8784..ca1d4cccab 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -32,7 +32,6 @@ #include "index.h" #include "query.h" #include "tcompare.h" -#include "tcompression.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" @@ -91,7 +90,7 @@ static void destroySortOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param); static void destroyIntervalOperatorInfo(void* param); -static void destroyExchangeOperatorInfo(void* param); + static void destroyOperatorInfo(SOperatorInfo* pOperator); @@ -109,8 +108,8 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, @@ -1652,596 +1651,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t return TSDB_CODE_SUCCESS; } -typedef struct SFetchRspHandleWrapper { - uint32_t exchangeId; - int32_t sourceIndex; -} SFetchRspHandleWrapper; - -int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { - SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; - - SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); - if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; - } - - int32_t index = pWrapper->sourceIndex; - SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - - if (code == TSDB_CODE_SUCCESS) { - pSourceDataInfo->pRsp = pMsg->pData; - - SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->compLen = htonl(pRsp->compLen); - pRsp->numOfCols = htonl(pRsp->numOfCols); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - - ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); - } else { - taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); - } - - pSourceDataInfo->status = EX_SOURCE_DATA_READY; - - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - - return TSDB_CODE_SUCCESS; -} - -void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); - - SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; - - if (pMsg->contLen > 0) { - buf.pData = taosMemoryCalloc(1, pMsg->contLen); - if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - memcpy(buf.pData, pMsg->pCont, pMsg->contLen); - } - } - - pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); - rpcFreeCont(pMsg->pCont); - destroySendMsgInfo(pSendInfo); -} - -static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); - - ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - - SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); - pWrapper->exchangeId = pExchangeInfo->self; - pWrapper->sourceIndex = sourceIndex; - - if (pSource->localExec) { - SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); - loadRemoteDataCallback(pWrapper, &pBuf, code); - taosMemoryFree(pWrapper); - } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - taosMemoryFree(pWrapper); - return pTaskInfo->code; - } - - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); - taosMemoryFree(pWrapper); - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - - pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = pSource->fetchMsgType; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator) { - pInfo->totalRows += numOfRows; - pInfo->totalSize += dataLen; - pInfo->totalElapsed += (taosGetTimestampUs() - startTs); - pOperator->resultInfo.totalRows += numOfRows; -} - -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { - if (pColList == NULL) { // data from other sources - blockDataCleanup(pRes); - *pNextStart = (char*)blockDecode(pRes, pData); - } else { // extract data according to pColList - char* pStart = pData; - - int32_t numOfCols = htonl(*(int32_t*)pStart); - pStart += sizeof(int32_t); - - // todo refactor:extract method - SSysTableSchema* pSchema = (SSysTableSchema*)pStart; - for (int32_t i = 0; i < numOfCols; ++i) { - SSysTableSchema* p = (SSysTableSchema*)pStart; - - p->colId = htons(p->colId); - p->bytes = htonl(p->bytes); - pStart += sizeof(SSysTableSchema); - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); - blockDataAppendColInfo(pBlock, &idata); - } - - blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, pBlock->info.rows); - - // data from mnode - pRes->info.rows = pBlock->info.rows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - blockDataDestroy(pBlock); - } - - // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. - blockDataUpdateTsWindow(pRes, 0); - return TSDB_CODE_SUCCESS; -} - -static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int64_t el = taosGetTimestampUs() - startTs; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - - pLoadInfo->totalElapsed += el; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - - doSetOperatorCompleted(pOperator); - return NULL; -} - - -static int32_t getCompletedSources(const SArray* pArray) { - size_t total = taosArrayGetSize(pArray); - - int32_t completed = 0; - for (int32_t k = 0; k < total; ++k) { - SSourceDataInfo* p = taosArrayGet(pArray, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - return completed; -} - -static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { - int32_t code = 0; - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); - int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (completed == totalSources) { - setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); - return; - } - - while (1) { - tsem_wait(&pExchangeInfo->ready); - - for (int32_t i = 0; i < totalSources; ++i) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { - continue; - } - - if (pDataInfo->status != EX_SOURCE_DATA_READY) { - continue; - } - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - code = pDataInfo->code; - goto _error; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - - // todo - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); - taosMemoryFreeClear(pDataInfo->pRsp); - break; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); - - if (pRsp->completed == 1) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - i + 1, totalSources); - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); - } - - taosMemoryFreeClear(pDataInfo->pRsp); - - if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { - pDataInfo->status = EX_SOURCE_DATA_NOT_READY; - code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - } - return; - } // end loop - - int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (complete1 == totalSources) { - qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); - return; - } - } - -_error: - pTaskInfo->code = code; -} - -static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return code; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), - totalSources, (endTs - startTs) / 1000.0); - - pOperator->status = OP_RES_TO_RETURN; - pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); - tsem_post(&pExchangeInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - while (1) { - if (pExchangeInfo->current >= totalSources) { - setAllSourcesCompleted(pOperator, startTs); - return TSDB_CODE_SUCCESS; - } - - doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); - tsem_wait(&pExchangeInfo->ready); - - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); - pOperator->pTaskInfo->code = pDataInfo->code; - return pOperator->pTaskInfo->code; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - taosMemoryFreeClear(pDataInfo->pRsp); - continue; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - - if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); - pDataInfo->totalRows += pRetrieveRsp->numOfRows; - - taosMemoryFreeClear(pDataInfo->pRsp); - return TSDB_CODE_SUCCESS; - } -} - -static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - - SExchangeInfo* pExchangeInfo = pOperator->info; - if (!pExchangeInfo->seqLoadData) { - int32_t code = prepareConcurrentlyLoad(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - pExchangeInfo->openedTs = taosGetTimestampUs(); - } - - OPTR_SET_OPENED(pOperator); - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return TSDB_CODE_SUCCESS; -} - -static void freeBlock(void* pParam) { - SSDataBlock* pBlock = *(SSDataBlock**)pParam; - blockDataDestroy(pBlock); -} - -static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pOperator->status == OP_EXEC_DONE) { - qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - return NULL; - } - - size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); - if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { - pExchangeInfo->rspBlockIndex = 0; - taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); - if (pExchangeInfo->seqLoadData) { - seqLoadRemoteData(pOperator); - } else { - concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } - - if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { - return NULL; - } - } - - // we have buffered retrieved datablock, return it directly - return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); -} - -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); - if (pBlock == NULL) { - return NULL; - } - - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; - if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); - if (status == PROJECT_RETRIEVE_CONTINUE) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pBlock->info.rows; - pExchangeInfo->limitInfo.numOfOutputRows += rows; - - if (rows == 0) { - doSetOperatorCompleted(pOperator); - return NULL; - } else { - return pBlock; - } - } - } else { - return pBlock; - } - } -} - -static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SSourceDataInfo dataInfo = {0}; - dataInfo.status = EX_SOURCE_DATA_NOT_READY; - dataInfo.taskId = id; - dataInfo.index = i; - SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); - if (pDs == NULL) { - taosArrayDestroy(pInfo->pSourceDataInfo); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { - size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); - - if (numOfSources == 0) { - qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); - return TSDB_CODE_INVALID_PARA; - } - - pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - if (pInfo->pSources == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); - taosArrayPush(pInfo->pSources, pNode); - } - - initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); - pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); - - return initDataSource(numOfSources, pInfo, id); -} - -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tsem_init(&pInfo->ready, 0, 0); - pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); - - pInfo->seqLoadData = false; - pInfo->pTransporter = pTransporter; - - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - doDestroyExchangeOperatorInfo(pInfo); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); @@ -2960,7 +2369,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, NULL); + createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -3024,33 +2433,6 @@ void destroyFillOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void destroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - taosRemoveRef(exchangeObjRefPool, pExInfo->self); -} - -void freeSourceDataInfo(void* p) { - SSourceDataInfo* pInfo = (SSourceDataInfo*)p; - taosMemoryFreeClear(pInfo->pRsp); -} - -void doDestroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - - taosArrayDestroy(pExInfo->pSources); - taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); - - if (pExInfo->pResultBlockList != NULL) { - taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); - pExInfo->pResultBlockList = NULL; - } - - blockDataDestroy(pExInfo->pDummyBlock); - - tsem_destroy(&pExInfo->ready); - taosMemoryFreeClear(param); -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { @@ -3190,7 +2572,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 891eb6e7a4..cfcdd0ee64 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -446,7 +446,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -831,7 +831,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1111,7 +1111,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 45d76dce74..e7fe2bfc12 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -121,8 +121,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,13 +371,13 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) if (leftTs == rightTs) { mergeJoinJoinDownstreamTsRanges(pOperator, leftTs, pRes, &nrows); - } else if (asc && leftTs < rightTs || !asc && leftTs > rightTs) { + } else if ((asc && leftTs < rightTs) || (!asc && leftTs > rightTs)) { pJoinInfo->leftPos += 1; if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { continue; } - } else if (asc && leftTs > rightTs || !asc && leftTs < rightTs) { + } else if ((asc && leftTs > rightTs) || (!asc && leftTs < rightTs)) { pJoinInfo->rightPos += 1; if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { continue; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index b2865d15f0..58cc082653 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -104,7 +104,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -406,8 +406,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, - destroyIndefinitOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6d13048123..903843bcc4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 @@ -993,7 +993,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1155,8 +1155,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; _error: @@ -2372,7 +2371,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; _end: @@ -2565,7 +2564,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); return pOperator; @@ -4207,7 +4206,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; _error: @@ -4346,7 +4345,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); return pOperator; @@ -4861,7 +4860,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index af87c4b2eb..700612e0ab 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -67,7 +67,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -517,7 +517,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); @@ -781,7 +781,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 9f06e639b3..a6ac28829c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1698,7 +1698,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7b6ed5b67d..6d8282ff5d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1785,7 +1785,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2565,7 +2565,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -2641,7 +2641,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2718,7 +2718,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3403,7 +3403,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -4196,7 +4196,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, @@ -4342,7 +4342,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } @@ -4713,7 +4713,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->pTaskInfo = pTaskInfo; pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); @@ -4995,7 +4995,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = miaInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5307,7 +5307,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pOperator->info = pMergeIntervalInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5540,7 +5540,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index ad342edfa0..573a1ea31f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -118,8 +118,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -200,8 +199,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From dc67c5041a360e495e4c49f13b9e7f3dd502180b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 13:46:19 +0800 Subject: [PATCH 066/101] refactor: do some internal refactor. --- source/libs/executor/src/exchangeoperator.c | 643 ++++++++++++++++++++ 1 file changed, 643 insertions(+) create mode 100644 source/libs/executor/src/exchangeoperator.c diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c new file mode 100644 index 0000000000..cb3330e9ca --- /dev/null +++ b/source/libs/executor/src/exchangeoperator.c @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" +#include "tref.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "tsort.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef struct SFetchRspHandleWrapper { + uint32_t exchangeId; + int32_t sourceIndex; +} SFetchRspHandleWrapper; + +static void destroyExchangeOperatorInfo(void* param); +static void freeBlock(void* pParam); +static void freeSourceDataInfo(void* param); +static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs); + +static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); +static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); +static int32_t getCompletedSources(const SArray* pArray); +static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); +static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); + +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { + int32_t code = 0; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); + int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (completed == totalSources) { + setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); + return; + } + + while (1) { + tsem_wait(&pExchangeInfo->ready); + + for (int32_t i = 0; i < totalSources; ++i) { + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { + continue; + } + + if (pDataInfo->status != EX_SOURCE_DATA_READY) { + continue; + } + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); + + // todo + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, + pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + taosMemoryFreeClear(pDataInfo->pRsp); + break; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); + + if (pRsp->completed == 1) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + ", total:%.2f Kb, try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + i + 1, totalSources); + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + } + + taosMemoryFreeClear(pDataInfo->pRsp); + + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { + pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + } + return; + } // end loop + + int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (complete1 == totalSources) { + qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); + return; + } + } + + _error: + pTaskInfo->code = code; +} + +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pOperator->status == OP_EXEC_DONE) { + qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + return NULL; + } + + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } + } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); +} + +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pBlock->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + doSetOperatorCompleted(pOperator); + return NULL; + } else { + return pBlock; + } + } + } else { + return pBlock; + } + } +} + +static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = EX_SOURCE_DATA_NOT_READY; + dataInfo.taskId = id; + dataInfo.index = i; + SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + if (pDs == NULL) { + taosArrayDestroy(pInfo->pSourceDataInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + + if (numOfSources == 0) { + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); + return TSDB_CODE_INVALID_PARA; + } + + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); + taosArrayPush(pInfo->pSources, pNode); + } + + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); + pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); + + return initDataSource(numOfSources, pInfo, id); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); + + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + + pOperator->name = "ExchangeOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + doDestroyExchangeOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + taosRemoveRef(exchangeObjRefPool, pExInfo->self); +} + +void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + +void freeSourceDataInfo(void* p) { + SSourceDataInfo* pInfo = (SSourceDataInfo*)p; + taosMemoryFreeClear(pInfo->pRsp); +} + +void doDestroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + + taosArrayDestroy(pExInfo->pSources); + taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; + } + + blockDataDestroy(pExInfo->pDummyBlock); + + tsem_destroy(&pExInfo->ready); + taosMemoryFreeClear(param); +} + +int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { + SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; + + SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); + if (pExchangeInfo == NULL) { + qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); + return TSDB_CODE_SUCCESS; + } + + int32_t index = pWrapper->sourceIndex; + SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + + if (code == TSDB_CODE_SUCCESS) { + pSourceDataInfo->pRsp = pMsg->pData; + + SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->compLen = htonl(pRsp->compLen); + pRsp->numOfCols = htonl(pRsp->numOfCols); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); + + ASSERT(pRsp != NULL); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); + } else { + taosMemoryFree(pMsg->pData); + pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + } + + pSourceDataInfo->status = EX_SOURCE_DATA_READY; + + tsem_post(&pExchangeInfo->ready); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + + return TSDB_CODE_SUCCESS; +} + +int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + + SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); + pWrapper->exchangeId = pExchangeInfo->self; + pWrapper->sourceIndex = sourceIndex; + + if (pSource->localExec) { + SDataBuf pBuf = {0}; + int32_t code = + (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + loadRemoteDataCallback(pWrapper, &pBuf, code); + taosMemoryFree(pWrapper); + } else { + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, + pSource->execId, sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pMsg->execId = htonl(pSource->execId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = pSource->fetchMsgType; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { + if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + *pNextStart = (char*)blockDecode(pRes, pData); + } else { // extract data according to pColList + char* pStart = pData; + + int32_t numOfCols = htonl(*(int32_t*)pStart); + pStart += sizeof(int32_t); + + // todo refactor:extract method + SSysTableSchema* pSchema = (SSysTableSchema*)pStart; + for (int32_t i = 0; i < numOfCols; ++i) { + SSysTableSchema* p = (SSysTableSchema*)pStart; + + p->colId = htons(p->colId); + p->bytes = htonl(p->bytes); + pStart += sizeof(SSysTableSchema); + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); + } + + blockDecode(pBlock, pStart); + blockDataEnsureCapacity(pRes, pBlock->info.rows); + + // data from mnode + pRes->info.rows = pBlock->info.rows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + blockDataDestroy(pBlock); + } + + // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. + blockDataUpdateTsWindow(pRes, 0); + return TSDB_CODE_SUCCESS; +} + +void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int64_t el = taosGetTimestampUs() - startTs; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + + pLoadInfo->totalElapsed += el; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + + doSetOperatorCompleted(pOperator); + return NULL; +} + +int32_t getCompletedSources(const SArray* pArray) { + size_t total = taosArrayGetSize(pArray); + + int32_t completed = 0; + for (int32_t k = 0; k < total; ++k) { + SSourceDataInfo* p = taosArrayGet(pArray, k); + if (p->status == EX_SOURCE_DATA_EXHAUSTED) { + completed += 1; + } + } + + return completed; +} + +int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for (int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return code; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), + totalSources, (endTs - startTs) / 1000.0); + + pOperator->status = OP_RES_TO_RETURN; + pOperator->cost.openCost = taosGetTimestampUs() - startTs; + + tsem_wait(&pExchangeInfo->ready); + tsem_post(&pExchangeInfo->ready); + return TSDB_CODE_SUCCESS; +} + +int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + while (1) { + if (pExchangeInfo->current >= totalSources) { + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; + } + + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); + tsem_wait(&pExchangeInfo->ready); + + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), + pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + pOperator->pTaskInfo->code = pDataInfo->code; + return pOperator->pTaskInfo->code; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, + pDataInfo->totalRows, pLoadInfo->totalRows); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + taosMemoryFreeClear(pDataInfo->pRsp); + continue; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); + + if (pRsp->completed == 1) { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, + totalSources); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pLoadInfo->totalRows, pLoadInfo->totalSize); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + + taosMemoryFreeClear(pDataInfo->pRsp); + return TSDB_CODE_SUCCESS; + } +} + +int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + int64_t st = taosGetTimestampUs(); + + SExchangeInfo* pExchangeInfo = pOperator->info; + if (!pExchangeInfo->seqLoadData) { + int32_t code = prepareConcurrentlyLoad(pOperator); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pExchangeInfo->openedTs = taosGetTimestampUs(); + } + + OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return TSDB_CODE_SUCCESS; +} From 411371ba488dd324f49f19c5ee82a59e094b7bd7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:16:21 +0800 Subject: [PATCH 067/101] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +-- source/libs/function/src/builtinsimpl.c | 40 ++++++++++++++++++------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9dab1e1d33..c157faecb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -973,7 +973,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int8_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } @@ -998,7 +998,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int32_t* pts = (int32_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int32_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6eb57c1a18..47977f9757 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3096,18 +3096,36 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } #else int64_t* pts = (int64_t*)pInput->pPTS->pData; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; + + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetVarData(pInputCol, i); + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } - numOfElems++; + numOfElems++; - char* data = colDataGetData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; + char* data = colDataGetNumData(pInputCol, i); + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } } } #endif @@ -3266,8 +3284,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. - // this is NOT always works if project operator exists in downstream. + // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. + // this assumption is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); From 92dc920f4c8b813962bc991e3e797026f35cfa74 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 14:22:58 +0800 Subject: [PATCH 068/101] perf: change default buffer sizea --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 6e1fa87854..b6a7f5c824 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -290,7 +290,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB -#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 +#define TSDB_DEFAULT_BUFFER_PER_VNODE 256 #define TSDB_MIN_PAGES_PER_VNODE 64 #define TSDB_MAX_PAGES_PER_VNODE (INT32_MAX - 1) #define TSDB_DEFAULT_PAGES_PER_VNODE 256 From caf5bddb0441bf30c094916a0c90a629c699f394 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Nov 2022 14:43:56 +0800 Subject: [PATCH 069/101] fix: taos_fetch_rows_a error --- source/client/src/clientMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 71a87a4b54..efa7d095c5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -990,7 +990,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // all data has returned to App already, no need to try again if (pResultInfo->completed) { // it is a local executed query, no need to do async fetch - if (QUERY_EXEC_MODE_LOCAL == pRequest->body.execMode) { + if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { if (pResultInfo->localResultFetched) { pResultInfo->numOfRows = 0; pResultInfo->current = 0; From 15e7e345dc40b83084cf5fbdbc7e1a215ca13a5d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 14:45:07 +0800 Subject: [PATCH 070/101] refact: move sync test code to single lib --- source/libs/sync/inc/syncInt.h | 2 - source/libs/sync/inc/syncRaftEntry.h | 34 +- source/libs/sync/inc/syncRaftLog.h | 22 +- source/libs/sync/inc/syncTools.h | 30 +- source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncMain.c | 354 ++--------- source/libs/sync/src/syncMessage.c | 587 ++---------------- source/libs/sync/src/syncRaftEntry.c | 291 +-------- source/libs/sync/src/syncRaftLog.c | 190 +----- source/libs/sync/src/syncReplication.c | 9 +- source/libs/sync/test/CMakeLists.txt | 164 ++--- .../sync/test/syncAppendEntriesBatchTest.cpp | 1 + .../sync/test/syncClientRequestBatchTest.cpp | 125 ---- .../libs/sync/test/syncClientRequestTest.cpp | 98 --- source/libs/sync/test/syncEncodeTest.cpp | 30 +- source/libs/sync/test/syncEntryCacheTest.cpp | 1 + source/libs/sync/test/syncEntryTest.cpp | 18 +- source/libs/sync/test/syncHashCacheTest.cpp | 1 + source/libs/sync/test/syncLogStoreCheck.cpp | 1 + source/libs/sync/test/syncLogStoreCheck2.cpp | 1 + source/libs/sync/test/syncLogStoreTest.cpp | 1 + source/libs/sync/test/syncRaftLogTest2.cpp | 1 + source/libs/sync/test/syncRaftLogTest3.cpp | 9 +- source/libs/sync/test/syncRespMgrTest.cpp | 20 +- source/libs/sync/test/syncRpcMsgTest.cpp | 10 +- source/libs/sync/test/syncSnapshotTest.cpp | 11 +- .../libs/sync/test/syncVotesGrantedTest.cpp | 1 + .../libs/sync/test/syncVotesRespondTest.cpp | 2 + source/libs/sync/test/syncWriteTest.cpp | 18 +- .../sync/test/sync_test_lib/CMakeLists.txt | 14 + .../{ => test/sync_test_lib}/inc/syncIO.h | 2 +- .../sync/test/sync_test_lib/inc/syncTest.h | 90 +++ .../{ => test/sync_test_lib}/src/syncIO.c | 5 +- .../test/sync_test_lib/src/syncMainDebug.c | 255 ++++++++ .../test/sync_test_lib/src/syncMessageDebug.c | 508 +++++++++++++++ .../test/sync_test_lib/src/syncRaftCfgDebug.c | 19 + .../sync_test_lib/src/syncRaftEntryDebug.c | 222 +++++++ .../test/sync_test_lib/src/syncRaftLogDebug.c | 182 ++++++ 38 files changed, 1562 insertions(+), 1769 deletions(-) delete mode 100644 source/libs/sync/test/syncClientRequestBatchTest.cpp delete mode 100644 source/libs/sync/test/syncClientRequestTest.cpp create mode 100644 source/libs/sync/test/sync_test_lib/CMakeLists.txt rename source/libs/sync/{ => test/sync_test_lib}/inc/syncIO.h (95%) create mode 100644 source/libs/sync/test/sync_test_lib/inc/syncTest.h rename source/libs/sync/{ => test/sync_test_lib}/src/syncIO.c (98%) create mode 100644 source/libs/sync/test/sync_test_lib/src/syncMainDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3085b6d8f4..8a951ba38d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -268,8 +268,6 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index bab1dcc661..8b8ab41b53 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -42,25 +42,14 @@ typedef struct SSyncRaftEntry { char data[]; // origin RpcMsg.pCont } SSyncRaftEntry; -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild(int32_t dataLen); +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg); SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId); void syncEntryDestory(SSyncRaftEntry* pEntry); -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); -char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj); -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); -void syncEntryLog(const SSyncRaftEntry* pObj); -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); - -//----------------------------------- typedef struct SRaftEntryHashCache { SHashObj* pEntryHash; int32_t maxCount; @@ -78,14 +67,6 @@ int32_t raftCacheDelEntry(struct SRaftEntryHashCache* pCache, SyncI int32_t raftCacheGetAndDel(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftCacheClear(struct SRaftEntryHashCache* pCache); -cJSON* raftCache2Json(SRaftEntryHashCache* pObj); -char* raftCache2Str(SRaftEntryHashCache* pObj); -void raftCachePrint(SRaftEntryHashCache* pObj); -void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); -void raftCacheLog(SRaftEntryHashCache* pObj); -void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); - -//----------------------------------- typedef struct SRaftEntryCache { SSkipList* pSkipList; int32_t maxCount; @@ -102,13 +83,6 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncInde int32_t raftEntryCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count); -cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); -char* raftEntryCache2Str(SRaftEntryCache* pObj); -void raftEntryCachePrint(SRaftEntryCache* pObj); -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); -void raftEntryCacheLog(SRaftEntryCache* pObj); -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index ff59189a9d..c25d4ae34e 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -40,25 +40,19 @@ typedef struct SSyncLogStoreData { SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); void logStoreDestory(SSyncLogStore* pLogStore); -cJSON* logStore2Json(SSyncLogStore* pLogStore); -char* logStore2Str(SSyncLogStore* pLogStore); -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); -char* logStoreSimple2Str(SSyncLogStore* pLogStore); SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore); -// for debug -void logStorePrint(SSyncLogStore* pLogStore); -void logStorePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreLog(SSyncLogStore* pLogStore); -void logStoreLog2(char* s, SSyncLogStore* pLogStore); - -void logStoreSimplePrint(SSyncLogStore* pLogStore); -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreSimpleLog(SSyncLogStore* pLogStore); -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 193579030f..6a760ecd87 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -180,17 +180,13 @@ typedef struct SyncClientRequest { } SyncClientRequest; SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pMsg, uint64_t seqNum, bool isWeak, int32_t vgId); // step 1 -void syncClientRequestDestroy(SyncClientRequest* pMsg); -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 -cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); -char* syncClientRequest2Str(const SyncClientRequest* pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId); +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); +char* syncClientRequest2Str(const SyncClientRequest* pMsg); // for debug ---------------------- void syncClientRequestPrint(const SyncClientRequest* pMsg); @@ -381,14 +377,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchDeserialize2(const char* buf, uint void syncAppendEntriesBatch2RpcMsg(const SyncAppendEntriesBatch* pMsg, SRpcMsg* pRpcMsg); void syncAppendEntriesBatchFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesBatch* pMsg); SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); // --------------------------------------------- typedef struct SyncAppendEntriesReply { @@ -739,14 +727,14 @@ int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg); int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); -typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7d6e358511..4f3f1c2c00 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -179,7 +179,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->success = true; bool hasAppendEntries = pMsg->dataLen > 0; if (hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + SSyncRaftEntry* pAppendEntry = syncEntryBuildFromAppendEntries(pMsg); ASSERT(pAppendEntry != NULL); SyncIndex appendIndex = pMsg->prevLogIndex + 1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 14823be098..67d5c5afdd 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -154,9 +154,7 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { code = syncNodeOnPingReply(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote* pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); @@ -225,7 +223,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { sError("sync begin snapshot error"); return -1; } - + int32_t code = 0; if (syncNodeIsMnode(pSyncNode)) { @@ -601,44 +599,42 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { return -1; } - int32_t ret = 0; - SyncClientRequest* pSyncMsg; - // optimized one replica if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { - pSyncMsg = syncClientRequestBuild(pMsg, 0, isWeak, pSyncNode->vgId); - SyncIndex retIndex; - int32_t code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, &retIndex); + int32_t code = syncNodeOnClientRequest(pSyncNode, pMsg, &retIndex); if (code == 0) { pMsg->info.conn.applyIndex = retIndex; pMsg->info.conn.applyTerm = pSyncNode->pRaftStore->currentTerm; - ret = 1; - sTrace("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); - } else { - ret = -1; - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, + sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); + return 1; + } else { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d, failed to propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType)); + return -1; } } else { SRespStub stub = {.createTime = taosGetTimestampMs(), .rpcMsg = *pMsg}; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); - - pSyncMsg = syncClientRequestBuild(pMsg, seqNum, isWeak, pSyncNode->vgId); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - - sNTrace(pSyncNode, "propose message, type:%s", TMSG_INFO(pMsg->msgType)); - ret = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); - if (ret != 0) { - sError("vgId:%d, failed to enqueue msg since %s", pSyncNode->vgId, terrstr()); - syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromRpcMsg(&rpcMsg, pMsg, seqNum, isWeak, pSyncNode->vgId); + if (code != 0) { + sError("vgId:%d, failed to propose msg while serialize since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + return -1; } - } - syncClientRequestDestroy(pSyncMsg); - return ret; + sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); + code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + } + + return code; + } } static int32_t syncHbTimerInit(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer, SRaftId destId) { @@ -1084,53 +1080,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; } -// ping -------------- -int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - syncPingLog2((char*)"==syncNodePing==", pMsg); - int32_t ret = 0; - - SRpcMsg rpcMsg; - syncPing2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); - - ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); - return ret; -} - -int32_t syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret = 0; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - ASSERT(ret == 0); - - syncPingDestroy(pMsg); - return ret; -} - -int32_t syncNodePingPeers(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId* destId = &(pSyncNode->peersId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - -int32_t syncNodePingAll(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - SRaftId* destId = &(pSyncNode->replicasId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; @@ -1293,196 +1242,6 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S return 0; } -cJSON* syncNode2Json(const SSyncNode* pSyncNode) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pSyncNode != NULL) { - // init by SSyncInfo - cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); - cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); - cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); - cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); - cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); - cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "queue", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); - cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); - - // init internal - cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); - cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); - cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); - - cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); - cJSON* pPeers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); - } - cJSON* pPeersId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersId", pPeersId); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); - } - - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); - cJSON* pReplicasId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); - } - - // raft algorithm - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); - cJSON_AddStringToObject(pRoot, "pFsm", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); - cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); - cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); - - // life cycle - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - - // tla+ server vars - cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); - cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); - cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); - - // tla+ candidate vars - cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); - cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); - - // tla+ leader vars - cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); - cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); - - // tla+ log vars - cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - // timer ms init - cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); - cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); - cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); - - // ping timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); - cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); - cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); - cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); - - // elect timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); - cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); - cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); - cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); - cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); - - // heartbeat timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); - cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); - - // callback - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); - cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); - cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); - cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); - cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); - cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); - - // restoreFinish - cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); - - // snapshot senders - cJSON* pSenders = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "senders", pSenders); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); - } - - // snapshot receivers - cJSON* pReceivers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); - - // changing - cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); - return pJson; -} - -char* syncNode2Str(const SSyncNode* pSyncNode) { - cJSON* pJson = syncNode2Json(pSyncNode); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { - int32_t len = 256; - char* s = (char*)taosMemoryMalloc(len); - - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); - - snprintf(s, len, - "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 - ", sby:%d, " - "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, - logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); - - return s; -} - inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { bool b1 = false; bool b2 = false; @@ -2056,8 +1815,6 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex return 0; } -// ------ local funciton --------- -// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { @@ -2065,7 +1822,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { pSyncNode->pingTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue ping timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2141,7 +1898,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue heartbeat timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2230,34 +1987,28 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { } } -static int32_t syncNodeEqNoop(SSyncNode* ths) { - int32_t ret = 0; - ASSERT(ths->state == TAOS_SYNC_STATE_LEADER); - - SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - ASSERT(pEntry != NULL); - - uint32_t entryLen; - char* serialized = syncEntrySerialize(pEntry, &entryLen); - SyncClientRequest* pSyncMsg = syncClientRequestAlloc(entryLen); - ASSERT(pSyncMsg->dataLen == entryLen); - memcpy(pSyncMsg->data, serialized, entryLen); - - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - if (ths->syncEqMsg != NULL) { - ths->syncEqMsg(ths->msgcb, &rpcMsg); - } else { - sTrace("syncNodeEqNoop pSyncNode->syncEqMsg is NULL"); +static int32_t syncNodeEqNoop(SSyncNode* pNode) { + if (pNode->state == TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + return -1; } - syncEntryDestory(pEntry); - taosMemoryFree(serialized); - syncClientRequestDestroy(pSyncMsg); + SyncIndex index = pNode->pLogStore->syncLogWriteIndex(pNode->pLogStore); + SyncTerm term = pNode->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, pNode->vgId); + if (pEntry == NULL) return -1; - return ret; + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromNoopEntry(&rpcMsg, pEntry, pNode->vgId); + syncEntryDestory(pEntry); + + sNTrace(pNode, "propose msg, type:noop"); + code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg); + if (code != 0) { + sNError(pNode, "failed to propose noop msg while enqueue since %s", terrstr()); + } + + return code; } static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } @@ -2438,7 +2189,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg) { // leaderVars, commitIndex>> // -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { sNTrace(ths, "on client request"); int32_t ret = 0; @@ -2446,8 +2197,13 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuild2(pMsg, term, index); - ASSERT(pEntry != NULL); + SSyncRaftEntry* pEntry; + + if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index); + } else { + pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index); + } LRUHandle* h = NULL; syncCacheEntry(ths->pLogStore, pEntry, &h); @@ -2646,7 +2402,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde } } - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); // user commit diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 6453693576..3fcb563f3b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -19,146 +19,6 @@ #include "syncUtil.h" #include "tcoding.h" -// --------------------------------------------- -cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { - cJSON* pRoot; - - // in compiler optimization, switch case = if else constants - if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncTimeout2Json(pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { - SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPing2Json(pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPingReply2Json(pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncClientRequest2Json(pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { - pRoot = syncRpcUnknownMsg2Json(); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVote2Json(pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVoteReply2Json(pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntries2Json(pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntriesReply2Json(pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotSend2Json(pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotRsp2Json(pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { - SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncLeaderTransfer2Json(pSyncMsg); - syncLeaderTransferDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - - } else { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - } - - cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); - cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); - cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); - // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); - return pJson; -} - -cJSON* syncRpcUnknownMsg2Json() { - cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); - cJSON_AddStringToObject(pRoot, "data", "unknown message"); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); - return pJson; -} - -char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { - cJSON* pJson = syncRpcMsg2Json(pRpcMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncRpcMsgPrint(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); @@ -840,69 +700,49 @@ SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen) { return pMsg; } -// step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, int32_t vgId) { - SyncClientRequest* pMsg = syncClientRequestAlloc(pOriginalRpcMsg->contLen); - pMsg->vgId = vgId; - pMsg->originalRpcType = pOriginalRpcMsg->msgType; - pMsg->seqNum = seqNum; - pMsg->isWeak = isWeak; - memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); - return pMsg; -} - -void syncClientRequestDestroy(SyncClientRequest* pMsg) { - if (pMsg != NULL) { - taosMemoryFree(pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pOriginalRpcMsg->contLen; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = pOriginalRpcMsg->msgType; + pClientRequest->seqNum = seqNum; + pClientRequest->isWeak = isWeak; + pClientRequest->dataLen = pOriginalRpcMsg->contLen; + memcpy(pClientRequest->data, (char*)pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); + + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) { - ASSERT(pMsg->bytes <= bufLen); - memcpy(buf, pMsg, pMsg->bytes); -} - -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) { - memcpy(pMsg, buf, len); - ASSERT(len == pMsg->bytes); -} - -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = taosMemoryMalloc(pMsg->bytes); - ASSERT(buf != NULL); - syncClientRequestSerialize(pMsg, buf, pMsg->bytes); - if (len != NULL) { - *len = pMsg->bytes; +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pEntry->bytes; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - return buf; -} -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = taosMemoryMalloc(bytes); - ASSERT(pMsg != NULL); - syncClientRequestDeserialize(buf, len, pMsg); - ASSERT(len == pMsg->bytes); - return pMsg; -} + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = TDMT_SYNC_NOOP; + pClientRequest->dataLen = pEntry->bytes; + memcpy(pClientRequest->data, (char*)pEntry, pEntry->bytes); -// step 2. SyncClientRequest => RpcMsg, to queue -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { - pRpcMsg->msgType = pMsg->msgType; - pRpcMsg->contLen = pMsg->bytes; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - syncClientRequestSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); -} - -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg) { - syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); -} - -// step 3. RpcMsg => SyncClientRequest, from queue -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { - SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - ASSERT(pMsg != NULL); - return pMsg; + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { @@ -940,35 +780,6 @@ char* syncClientRequest2Str(const SyncClientRequest* pMsg) { return serialized; } -// for debug ---------------------- -void syncClientRequestPrint(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestLog(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncClientRequestBatch---- // block1: @@ -1059,86 +870,6 @@ SyncClientRequestBatch* syncClientRequestBatchFromRpcMsg(const SRpcMsg* pRpcMsg) return pSyncMsg; } -cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - - SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); - SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); - cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pMsgArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); - cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); - return pJson; -} - -char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { - cJSON* pJson = syncClientRequestBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncRequestVote); @@ -1716,138 +1447,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg return pMsg; } -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); - cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - - int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // - int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; - - cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); - cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); - - SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); - cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pEntryArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); - cJSON* pEntryJson = syncEntry2Json(pEntry); - cJSON_AddItemToArray(pEntryArr, pEntryJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); - return pJson; -} - -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { - cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncAppendEntriesReply); @@ -2827,118 +2426,6 @@ SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { return pMsg; } -cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { - char u64buf[128]; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); - cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); - cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); - cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); - cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); - return pJson; -} - -char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { - cJSON* pJson = syncSnapshotSend2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// --------------------------------------------- SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncSnapshotRsp); SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 520ecd95db..c5b1399c8c 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,31 +13,28 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { - uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; +SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { + int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memset(pEntry, 0, bytes); + if (pEntry == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pEntry->bytes = bytes; pEntry->dataLen = dataLen; pEntry->rid = -1; - return pEntry; -} - -// step 4. SyncClientRequest => SSyncRaftEntry, add term, index -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index); - ASSERT(pEntry != NULL); return pEntry; } -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); - ASSERT(pEntry != NULL); + if (pEntry == NULL) return NULL; pEntry->msgType = pMsg->msgType; pEntry->originalRpcType = pMsg->originalRpcType; @@ -45,42 +42,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild(pOriginalMsg->contLen); - ASSERT(pEntry != NULL); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->contLen); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; - pEntry->originalRpcType = pOriginalMsg->msgType; + pEntry->originalRpcType = pMsg->msgType; pEntry->seqNum = 0; pEntry->isWeak = 0; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pOriginalMsg->contLen; - memcpy(pEntry->data, pOriginalMsg->pCont, pOriginalMsg->contLen); + memcpy(pEntry->data, pMsg->pCont, pMsg->contLen); return pEntry; } -SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { - // init rpcMsg - SMsgHead head; - head.vgId = vgId; - head.contLen = sizeof(SMsgHead); - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(SRpcMsg)); - rpcMsg.contLen = head.contLen; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - rpcMsg.msgType = TDMT_SYNC_NOOP; - memcpy(rpcMsg.pCont, &head, sizeof(head)); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + if (pEntry == NULL) return NULL; - SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); - ASSERT(pEntry != NULL); + memcpy(pEntry, pMsg->data, pMsg->dataLen); + return pEntry; +} + +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { + SSyncRaftEntry* pEntry = syncEntryBuild(sizeof(SMsgHead)); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = TDMT_SYNC_NOOP; @@ -89,9 +81,9 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) pEntry->term = term; pEntry->index = index; - ASSERT(pEntry->dataLen == rpcMsg.contLen); - memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen); - rpcFreeCont(rpcMsg.pCont); + SMsgHead* pHead = (SMsgHead*)pEntry->data; + pHead->vgId = vgId; + pHead->contLen = sizeof(SMsgHead); return pEntry; } @@ -102,104 +94,13 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { } } -// step 5. SSyncRaftEntry => bin, to raft log -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = taosMemoryMalloc(pEntry->bytes); - ASSERT(buf != NULL); - memcpy(buf, pEntry, pEntry->bytes); - if (len != NULL) { - *len = pEntry->bytes; - } - return buf; -} - -// step 6. bin => SSyncRaftEntry, from raft log -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memcpy(pEntry, buf, len); - ASSERT(len == pEntry->bytes); - return pEntry; -} - -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pEntry != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); - cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); - cJSON_AddStringToObject(pRoot, "seqNum", u64buf); - cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); - cJSON_AddStringToObject(pRoot, "index", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); - - char* s; - s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - - s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); - return pJson; -} - -char* syncEntry2Str(const SSyncRaftEntry* pEntry) { - cJSON* pJson = syncEntry2Json(pEntry); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// step 7. SSyncRaftEntry => original SRpcMsg, commit to user, delete seqNum, isWeak, term, index void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { - memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pEntry->originalRpcType; pRpcMsg->contLen = pEntry->dataLen; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryLog(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); -} - //----------------------------------- SRaftEntryHashCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { SRaftEntryHashCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryHashCache)); @@ -354,76 +255,6 @@ int32_t raftCacheClear(struct SRaftEntryHashCache* pCache) { return 0; } -//----------------------------------- -cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - while (pIter) { - pIter = taosHashIterate(pCache->pEntryHash, pIter); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - } - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); - return pJson; -} - -char* raftCache2Str(SRaftEntryHashCache* pCache) { - cJSON* pJson = raftCache2Json(pCache); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftCachePrint(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCacheLog(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { - if (gRaftDetailLog) { - char* serialized = raftCache2Str(pCache); - sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} //----------------------------------- static char* keyFn(const void* pData) { @@ -612,69 +443,3 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) { taosThreadMutexUnlock(&pCache->mutex); return returnCnt; } - -cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); - while (tSkipListIterNext(pIter)) { - SSkipListNode* pNode = tSkipListIterGet(pIter); - ASSERT(pNode != NULL); - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - tSkipListDestroyIter(pIter); - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); - return pJson; -} - -char* raftEntryCache2Str(SRaftEntryCache* pObj) { - cJSON* pJson = raftEntryCache2Json(pObj); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftEntryCachePrint(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { - if (gRaftDetailLog) { - char* serialized = raftEntryCache2Str(pObj); - sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d3d69b288b..85c94884f2 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -22,15 +22,8 @@ // public function static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex); -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); + static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index); static int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); @@ -126,29 +119,29 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI return 0; } -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex firstVer = walGetFirstVer(pWal); return firstVer; } -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; return walIsEmpty(pWal); } -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); int32_t count = endIndex - beginIndex + 1; return count > 0 ? count : 0; } -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SyncIndex lastIndex; SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -157,7 +150,7 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { return lastVer; } -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); @@ -174,7 +167,7 @@ static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index) { // if success, return last term // if not log, return 0 // if error, return SYNC_TERM_INVALID -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (walIsEmpty(pWal)) { @@ -225,7 +218,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr // entry found, return 0 // entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST // other error, return -1 -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; int32_t code = 0; @@ -364,111 +357,6 @@ SyncIndex raftlogCommitIndex(SSyncLogStore* pLogStore) { return pData->pSyncNode->commitIndex; } -cJSON* logStore2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - - if (!raftLogIsEmpty(pLogStore)) { - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - SSyncRaftEntry* pEntry = NULL; - raftLogGetEntry(pLogStore, i, &pEntry); - - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); - } - } - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); - return pJson; -} - -char* logStore2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStore2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); - return pJson; -} - -char* logStoreSimple2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStoreSimple2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -480,63 +368,3 @@ SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore) { SWal* pWal = pData->pWal; return walGetCommittedVer(pWal); } - -// for debug ----------------- -void logStorePrint(SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStorePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreLog(SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); - } -} - -void logStoreLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// for debug ----------------- -void logStoreSimplePrint(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7b4ef63406..8c3f7e5384 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -78,14 +78,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); - - // add pEntry into msg - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - ASSERT(len == pEntry->bytes); - memcpy(pMsg->data, serialized, len); - - taosMemoryFree(serialized); + memcpy(pMsg->data, pEntry->data, pEntry->bytes); syncEntryDestory(pEntry); } else { diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index e584893ae0..7a22d96972 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(sync_test_lib) add_executable(syncTest "") add_executable(syncRaftIdCheck "") add_executable(syncEnvTest "") @@ -24,8 +25,6 @@ add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") add_executable(syncAppendEntriesBatchTest "") add_executable(syncAppendEntriesReplyTest "") -add_executable(syncClientRequestTest "") -add_executable(syncClientRequestBatchTest "") add_executable(syncTimeoutTest "") add_executable(syncPingTest "") add_executable(syncPingReplyTest "") @@ -167,14 +166,6 @@ target_sources(syncAppendEntriesReplyTest PRIVATE "syncAppendEntriesReplyTest.cpp" ) -target_sources(syncClientRequestTest - PRIVATE - "syncClientRequestTest.cpp" -) -target_sources(syncClientRequestBatchTest - PRIVATE - "syncClientRequestBatchTest.cpp" -) target_sources(syncTimeoutTest PRIVATE "syncTimeoutTest.cpp" @@ -287,10 +278,6 @@ target_sources(syncLeaderTransferTest PRIVATE "syncLeaderTransferTest.cpp" ) -target_sources(syncReconfigFinishTest - PRIVATE - "syncReconfigFinishTest.cpp" -) target_sources(syncRestoreFromSnapshot PRIVATE "syncRestoreFromSnapshot.cpp" @@ -451,16 +438,6 @@ target_include_directories(syncAppendEntriesReplyTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncClientRequestTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncClientRequestBatchTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncTimeoutTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -601,11 +578,6 @@ target_include_directories(syncLeaderTransferTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncReconfigFinishTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncRestoreFromSnapshot PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -644,259 +616,247 @@ target_include_directories(syncPreSnapshotReplyTest target_link_libraries(syncTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftIdCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnvTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickQTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOSendMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOClientTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOServerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnqTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncInitTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncUtilTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesGrantedTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesRespondTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHashCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesReplyTest - sync - gtest_main -) -target_link_libraries(syncClientRequestTest - sync - gtest_main -) -target_link_libraries(syncClientRequestBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTimeoutTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRpcMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingSelfTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncElectTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEncodeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncWriteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncReplicateTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRefTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRespMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncApplyMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSendTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotRspTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSenderTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotReceiverTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTestTool - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest3 - sync + sync_test_lib gtest_main ) target_link_libraries(syncLeaderTransferTest - sync - gtest_main -) -target_link_libraries(syncReconfigFinishTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRestoreFromSnapshot - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLocalCmdTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotReplyTest - sync + sync_test_lib gtest_main ) diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index f2544d8fec..98b8654734 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftEntry.h" #include "syncUtil.h" #include "trpc.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncClientRequestBatchTest.cpp b/source/libs/sync/test/syncClientRequestBatchTest.cpp deleted file mode 100644 index f07ee08b2b..0000000000 --- a/source/libs/sync/test/syncClientRequestBatchTest.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SRpcMsg *createRpcMsg(int32_t i, int32_t dataLen) { - SyncPing *pSyncMsg = syncPingBuild(20); - snprintf(pSyncMsg->data, pSyncMsg->dataLen, "value_%d", i); - - SRpcMsg *pRpcMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pRpcMsg, 0, sizeof(SRpcMsg)); - pRpcMsg->code = 10 * i; - syncPing2RpcMsg(pSyncMsg, pRpcMsg); - - syncPingDestroy(pSyncMsg); - return pRpcMsg; -} - -SyncClientRequestBatch *createMsg() { - SRpcMsg *rpcMsgPArr[5]; - memset(rpcMsgPArr, 0, sizeof(rpcMsgPArr)); - for (int32_t i = 0; i < 5; ++i) { - SRpcMsg *pRpcMsg = createRpcMsg(i, 20); - rpcMsgPArr[i] = pRpcMsg; - // taosMemoryFree(pRpcMsg); - } - - SRaftMeta raftArr[5]; - memset(raftArr, 0, sizeof(raftArr)); - for (int32_t i = 0; i < 5; ++i) { - raftArr[i].seqNum = i * 10; - raftArr[i].isWeak = i % 2; - } - - SyncClientRequestBatch *pMsg = syncClientRequestBatchBuild(rpcMsgPArr, raftArr, 5, 1234); - return pMsg; -} - -void test1() { - SyncClientRequestBatch *pMsg = createMsg(); - syncClientRequestBatchLog2((char *)"==test1==", pMsg); - syncClientRequestBatchDestroyDeep(pMsg); -} - -/* -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char * serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} -*/ - -int main() { - gRaftDetailLog = true; - tsAsyncLog = 0; - sDebugFlag = DEBUG_DEBUG + DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - - /* -test2(); -test3(); -test4(); -test5(); -*/ - - return 0; -} diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp deleted file mode 100644 index b6bfcc2da5..0000000000 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SyncClientRequest *createMsg() { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = 12345; - rpcMsg.contLen = 20; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); - rpcFreeCont(rpcMsg.pCont); - return pMsg; -} - -void test1() { - SyncClientRequest *pMsg = createMsg(); - syncClientRequestLog2((char *)"test1:", pMsg); - syncClientRequestDestroy(pMsg); -} - -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char *serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char *serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -int main() { - tsAsyncLog = 0; - sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - test2(); - test3(); - test4(); - test5(); - - return 0; -} diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index c60176fbf8..216291c1d8 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -7,9 +7,11 @@ #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" +#if 0 void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); @@ -118,17 +120,7 @@ SyncClientRequest *step3(const SRpcMsg *pMsg) { } SSyncRaftEntry *step4(const SyncClientRequest *pMsg) { - SSyncRaftEntry *pRetMsg = syncEntryBuild2((SyncClientRequest *)pMsg, 100, 0); - return pRetMsg; -} - -char *step5(const SSyncRaftEntry *pMsg, uint32_t *len) { - char *pRetMsg = syncEntrySerialize(pMsg, len); - return pRetMsg; -} - -SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { - SSyncRaftEntry *pRetMsg = syncEntryDeserialize(pMsg, len); + SSyncRaftEntry *pRetMsg = syncEntryBuildFromClientRequest((SyncClientRequest *)pMsg, 100, 0); return pRetMsg; } @@ -137,13 +129,14 @@ SRpcMsg *step7(const SSyncRaftEntry *pMsg) { syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } - +#endif int main(int argc, char **argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; void logTest(); +#if 0 myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); @@ -188,20 +181,9 @@ int main(int argc, char **argv) { syncEntryLog2((char *)"==pEntry==", pEntry); - // step5 - uint32_t len; - char *pMsg5 = step5(pMsg4, &len); - char *s = syncUtilPrintBin(pMsg5, len); - printf("==step5== [%s] \n", s); - taosMemoryFree(s); - - // step6 - SSyncRaftEntry *pMsg6 = step6(pMsg5, len); - syncEntryLog2((char *)"==step6==", pMsg6); - // step7 SRpcMsg *pMsg7 = step7(pMsg6); syncRpcMsgLog2((char *)"==step7==", pMsg7); - +#endif return 0; } diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 3ee28ce96b..56b3d6da2e 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -7,6 +7,7 @@ #include "syncUtil.h" #include "tref.h" #include "tskiplist.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index b274408c01..e94755195b 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -38,10 +39,10 @@ void test2() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test2"); - SSyncRaftEntry* pEntry = syncEntryBuild2(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -52,10 +53,10 @@ void test3() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test3"); - SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -71,14 +72,7 @@ void test4() { strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - assert(serialized != NULL); - SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); - syncEntryPrint(pEntry2); - - taosMemoryFree(serialized); - syncEntryDestory(pEntry2); + // syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 7d822971da..2f5bb07f9a 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -4,6 +4,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "tskiplist.h" diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 431b291ca7..0161160a75 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 80679bc85c..29ad0610e7 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -7,6 +7,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 9ff0ed2089..832b42bf80 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 78c08a6d8d..3d50b63ff9 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -5,6 +5,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index cf862feb2a..31c06625aa 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp index cad6eec91d..8d709e8c81 100644 --- a/source/libs/sync/test/syncRespMgrTest.cpp +++ b/source/libs/sync/test/syncRespMgrTest.cpp @@ -1,8 +1,4 @@ -#include "syncRespMgr.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -64,13 +60,13 @@ void syncRespMgrGetTest(uint64_t i) { void syncRespMgrGetAndDelTest(uint64_t i) { printf("------syncRespMgrGetAndDelTest-------%" PRIu64 "-- \n", i); - SRespStub stub; - int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); - if (ret == 1) { - printStub(&stub); - } else if (ret == 0) { - printf("%" PRId64 " notFound \n", i); - } + // SRespStub stub; + // int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); + // if (ret == 1) { + // printStub(&stub); + // } else if (ret == 0) { + // printf("%" PRId64 " notFound \n", i); + // } } SSyncNode *createSyncNode() { diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 941fa7eab5..127d8e1c41 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -47,7 +47,11 @@ SyncClientRequest *createSyncClientRequest() { rpcMsg.contLen = 20; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); + + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, &rpcMsg, 123, true, 1000); + SyncClientRequest *pMsg = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg->data, clientRequestMsg.pCont, clientRequestMsg.contLen); return pMsg; } @@ -155,11 +159,13 @@ void test7() { } void test8() { +#if 0 SyncClientRequest *pMsg = createSyncClientRequest(); SRpcMsg rpcMsg = {0}; syncClientRequest2RpcMsg(pMsg, &rpcMsg); syncRpcMsgLog2((char *)"test8", &rpcMsg); - syncClientRequestDestroy(pMsg); + taosMemoryFree(pMsg); +#endif } int main() { diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index a1cedd624a..e2264fd08c 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -162,8 +162,11 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); - return pRetMsg; + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, pMsg, 123, true, 1000); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg2->data, clientRequestMsg.pCont, clientRequestMsg.contLen); + return pMsg2; } int main(int argc, char **argv) { @@ -207,8 +210,8 @@ int main(int argc, char **argv) { for (int i = 0; i < 10; ++i) { SyncClientRequest *pSyncClientRequest = pMsg1; SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index bbf14c604e..6a8404308b 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index adebfe1be2..d5e7d71030 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -6,6 +6,8 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncTest.h" + void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 7c5334f668..2e5c26719b 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -7,6 +7,7 @@ #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncTest.h" #include "syncUtil.h" #include "wal.h" @@ -140,7 +141,8 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); + SyncClientRequest *pRetMsg = NULL; + // syncClientRequestBuild(pMsg, 123, true, 1000); return pRetMsg; } @@ -179,14 +181,14 @@ int main(int argc, char **argv) { SyncClientRequest *pMsg1 = step1(pMsg0); syncClientRequestLog2((char *)"==step1==", pMsg1); - for (int i = 0; i < 10; ++i) { - SyncClientRequest *pSyncClientRequest = pMsg1; - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // for (int i = 0; i < 10; ++i) { + // SyncClientRequest *pSyncClientRequest = pMsg1; + // SRpcMsg rpcMsg = {0}; + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); - taosMsleep(1000); - } + // taosMsleep(1000); + // } while (1) { sTrace("while 1 sleep"); diff --git a/source/libs/sync/test/sync_test_lib/CMakeLists.txt b/source/libs/sync/test/sync_test_lib/CMakeLists.txt new file mode 100644 index 0000000000..db775c3662 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SYNC_TEST_SRC) +add_library(sync_test_lib STATIC ${SYNC_TEST_SRC}) + +target_link_libraries( + sync_test_lib + PUBLIC sync +) + +target_include_directories( + sync_test_lib + PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/test/sync_test_lib/inc/syncIO.h similarity index 95% rename from source/libs/sync/inc/syncIO.h rename to source/libs/sync/test/sync_test_lib/inc/syncIO.h index cfc4dd2472..955a832b68 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncIO.h @@ -50,7 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); - int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SRpcMsg *pMsg, SyncIndex *pRetIndex); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h new file mode 100644 index 0000000000..3a49074272 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TEST_H +#define _TD_LIBS_SYNC_TEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "syncInt.h" + +#include "wal.h" + +#include "syncEnv.h" +#include "syncIO.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftCfg.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncRespMgr.h" +#include "syncSnapshot.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); +char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntryPrint(const SSyncRaftEntry* pObj); +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); +void syncEntryLog(const SSyncRaftEntry* pObj); +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); + +cJSON* raftCache2Json(SRaftEntryHashCache* pObj); +char* raftCache2Str(SRaftEntryHashCache* pObj); +void raftCachePrint(SRaftEntryHashCache* pObj); +void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); +void raftCacheLog(SRaftEntryHashCache* pObj); +void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); + +cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); +char* raftEntryCache2Str(SRaftEntryCache* pObj); +void raftEntryCachePrint(SRaftEntryCache* pObj); +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); +void raftEntryCacheLog(SRaftEntryCache* pObj); +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); + +cJSON* logStore2Json(SSyncLogStore* pLogStore); +char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); +void logStorePrint(SSyncLogStore* pLogStore); +void logStorePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreLog(SSyncLogStore* pLogStore); +void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c similarity index 98% rename from source/libs/sync/src/syncIO.c rename to source/libs/sync/test/sync_test_lib/src/syncIO.c index afa2d43e13..14adc18c66 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -279,10 +279,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - ASSERT(pSyncMsg != NULL); - io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pRpcMsg, NULL); } } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { diff --git a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c new file mode 100644 index 0000000000..99d124580b --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncNode2Json(const SSyncNode* pSyncNode) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pSyncNode != NULL) { + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); + cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); + cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); + cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } + + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // life cycle + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + + // tla+ server vars + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); + cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); + + // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); + + // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); + + // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + // timer ms init + cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); + cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); + cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); + + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + // restoreFinish + cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); + + // snapshot senders + cJSON* pSenders = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "senders", pSenders); + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); + } + + // snapshot receivers + cJSON* pReceivers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); + + // changing + cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); + return pJson; +} + +char* syncNode2Str(const SSyncNode* pSyncNode) { + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { + int32_t len = 256; + char* s = (char*)taosMemoryMalloc(len); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + snprintf(s, len, + "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", sby:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d", + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + + return s; +} + +// ping -------------- +int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + syncPingLog2((char*)"==syncNodePing==", pMsg); + int32_t ret = 0; + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); + + ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + return ret; +} + +int32_t syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + ASSERT(ret == 0); + + syncPingDestroy(pMsg); + return ret; +} + +int32_t syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* destId = &(pSyncNode->peersId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +int32_t syncNodePingAll(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + SRaftId* destId = &(pSyncNode->replicasId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c new file mode 100644 index 0000000000..012382d69d --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +// --------------------------------------------- +cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { + cJSON* pRoot; + + // in compiler optimization, switch case = if else constants + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncTimeout2Json(pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { + SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPing2Json(pSyncMsg); + syncPingDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPingReply2Json(pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest* pSyncMsg = pRpcMsg->pCont; + pRoot = syncClientRequest2Json(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { + pRoot = syncRpcUnknownMsg2Json(); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVote2Json(pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVoteReply2Json(pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntries2Json(pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntriesReply2Json(pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotSend2Json(pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotRsp2Json(pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { + SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncLeaderTransfer2Json(pSyncMsg); + syncLeaderTransferDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + + } else { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + } + + cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); + cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); + cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); + // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); + return pJson; +} + +cJSON* syncRpcUnknownMsg2Json() { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); + cJSON_AddStringToObject(pRoot, "data", "unknown message"); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); + return pJson; +} + +char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { + cJSON* pJson = syncRpcMsg2Json(pRpcMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncRpcMsgPrint(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + + int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // + int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; + + cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); + cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); + + SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); + cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pEntryArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); + cJSON* pEntryJson = syncEntry2Json(pEntry); + cJSON_AddItemToArray(pEntryArr, pEntryJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); + return pJson; +} + +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { + cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); + cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); + cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); + return pJson; +} + +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { + cJSON* pJson = syncSnapshotSend2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + + SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); + SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); + cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pMsgArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); + cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); + return pJson; +} + +char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { + cJSON* pJson = syncClientRequestBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ---------------------- +void syncClientRequestPrint(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestLog(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c new file mode 100644 index 0000000000..88b697fbf1 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + + diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c new file mode 100644 index 0000000000..8179b24d29 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pEntry != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); + cJSON_AddStringToObject(pRoot, "index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); + + char* s; + s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + + s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); + return pJson; +} + +char* syncEntry2Str(const SSyncRaftEntry* pEntry) { + cJSON* pJson = syncEntry2Json(pEntry); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncEntryPrint(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryLog(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryHashCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); + while (tSkipListIterNext(pIter)) { + SSkipListNode* pNode = tSkipListIterGet(pIter); + ASSERT(pNode != NULL); + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + tSkipListDestroyIter(pIter); + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftEntryCache2Str(SRaftEntryCache* pObj) { + cJSON* pJson = raftEntryCache2Json(pObj); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftEntryCachePrint(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { + if (gRaftDetailLog) { + char* serialized = raftEntryCache2Str(pObj); + sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c new file mode 100644 index 0000000000..53e414ccfd --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + + if (!raftLogIsEmpty(pLogStore)) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = NULL; + raftLogGetEntry(pLogStore, i, &pEntry); + + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + } + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} + +char* logStore2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStore2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ----------------- +void logStorePrint(SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStorePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreLog(SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); + } +} + +void logStoreLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} From a9167beb91be8a50154b6e0d9909a9f2635a9ea7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:52:37 +0800 Subject: [PATCH 071/101] fix(query): expand buffer size. --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a03c04ed6e..e3714be594 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -586,7 +586,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; + char tmpBuf[128] = {0}; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; From 65c813f735d0c8b98975bacfac97bf8de5fdce14 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 14:59:09 +0800 Subject: [PATCH 072/101] fix(query): fix some syntax errors. --- source/util/src/tlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e3714be594..a3bfd6e3d2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -317,10 +317,10 @@ static void taosGetLogFileName(char *fn) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { char fileName[LOG_FILE_NAME_LEN]; - sprintf(fileName, "%s%d.0", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN,"%s%d.0", fn, i); bool file1open = taosCheckFileIsOpen(fileName); - sprintf(fileName, "%s%d.1", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN,"%s%d.1", fn, i); bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { From 8659382c0eecafe3af30c9ed2fa1f38c40f957b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:11:52 +0800 Subject: [PATCH 073/101] refactor: update cmake file. --- cmake/cmake.version | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 92f7f8b895..d0a83155be 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,13 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "build type:" ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) From f942e319b92d4545226321fca22caf3b72ff9c24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:15:35 +0800 Subject: [PATCH 074/101] fix(query): fix some syntax errors. --- cmake/cmake.version | 16 ++++++++-------- source/util/src/tlog.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index d0a83155be..03598519ed 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,14 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) -MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "build type:" ${CMAKE_BUILD_TYPE}) -MESSAGE(STATUS "type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) +MESSAGE(STATUS "build date: " ${TD_VER_DATE}) +MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type: " ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3bfd6e3d2..c72be76a95 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -324,7 +324,7 @@ static void taosGetLogFileName(char *fn) { bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { - sprintf(tsLogObj.logName, "%s%d", fn, i); + snprintf(tsLogObj.logName, tListLen(tsLogObj.logName), "%s%d", fn, i); return; } } From 2ee0cf6701b4b62648ff70bd99de44921e7ab5a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:19:27 +0800 Subject: [PATCH 075/101] fix(query): fix the string copy error. --- source/libs/function/src/tudf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 1dedfe8364..8715aa0be1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -131,7 +131,8 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); - strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib)); + udfdPathLdLib[udfdLdLibPathLen] = ':'; strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { From 683e2cff7c08393804bd4dd3c4e0d0d6331b7179 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 15:49:39 +0800 Subject: [PATCH 076/101] change test cases --- tests/script/tsim/db/alter_option.sim | 4 +-- tests/script/tsim/db/create_all_options.sim | 4 +-- tests/system-test/1-insert/alter_database.py | 29 ++++++++++++-------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index b6139cea13..63018aea8c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -38,7 +38,7 @@ endi print ============= create database #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | WAL_FSYNC_PERIOD value [0 ~ 180000 ms] @@ -78,7 +78,7 @@ endi if $data7_db != 1440000m,1440000m,1440000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 5ac2ee6c4e..7012fbac6c 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -37,7 +37,7 @@ endi print ============= create database with all options #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] @@ -98,7 +98,7 @@ endi if $data7_db != 5256000m,5256000m,5256000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 1918c0ef76..aec18a1cf1 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -10,37 +10,43 @@ from util.sql import * from util.cases import * from util.dnodes import * + class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) - self.buffer_boundary = [3,4097,8193,12289,16384] - self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + tdSql.init(conn.cursor(), logSql) + self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + self.buffer_error = [self.buffer_boundary[0] - + 1, self.buffer_boundary[-1]+1, 12289, 256] # pages_boundary >= 64 - self.pages_boundary = [64,128,512] + self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: tdSql.execute(f'alter database db buffer {buffer}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8], buffer) tdSql.execute('drop database db') tdSql.execute('create database db vgroups 10') for buffer in self.buffer_error: tdSql.error(f'alter database db buffer {buffer}') tdSql.execute('drop database db') - + def alter_pages(self): tdSql.execute('create database db') for pages in self.pages_boundary: tdSql.execute(f'alter database db pages {pages}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10], pages) tdSql.execute('drop database db') tdSql.execute('create database db') - tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') @@ -55,5 +61,6 @@ class TDTestCase: tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From fa45cd2ce89f0214f17c157d7105d30a7b36cee8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 15:50:31 +0800 Subject: [PATCH 077/101] enh(query): unfold the loop. --- source/libs/function/src/builtinsimpl.c | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 47977f9757..fbcaac95df 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3113,6 +3113,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } } else { +#if 0 for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; @@ -3127,6 +3128,64 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { pResInfo->numOfRes = 1; } } +#else + + if (!pInputCol->hasNull) { + int32_t round = pInput->numOfRows >> 2; + int32_t reminder = pInput->numOfRows & 0x03; + + int32_t tick = 0; + for (int32_t i = pInput->startRowIndex; tick < round; i += 4, tick += 1) { + int64_t cts = pts[i]; + int32_t chosen = i; + + if (cts < pts[i + 1]) { + cts = pts[i + 1]; + chosen = i + 1; + } + + if (cts < pts[i + 2]) { + cts = pts[i + 2]; + chosen = i + 2; + } + + if (cts < pts[i + 3]) { + cts = pts[i + 3]; + chosen = i + 3; + } + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetNumData(pInputCol, chosen); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + + for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + numOfElems++; + TSKEY cts = pts[i]; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetNumData(pInputCol, i); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetNumData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } + } +#endif } #endif From 86ed5e5674d28da33c3eea30466acc544412e324 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 16:02:04 +0800 Subject: [PATCH 078/101] more optimize --- source/dnode/vnode/src/inc/tsdb.h | 44 +++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 25 ------------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 +----- source/dnode/vnode/src/vnd/vnodeModule.c | 6 +++ 4 files changed, 45 insertions(+), 42 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a5257b32c0..ba9d68ee7f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -110,7 +110,6 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { #define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // SRowIter void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); @@ -210,11 +209,10 @@ void tsdbRefMemTable(SMemTable *pMemTable); void tsdbUnrefMemTable(SMemTable *pMemTable); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter -int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); -void *tsdbTbDataIterDestroy(STbDataIter *pIter); -void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter); -bool tsdbTbDataIterNext(STbDataIter *pIter); +int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); +void *tsdbTbDataIterDestroy(STbDataIter *pIter); +void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); +bool tsdbTbDataIterNext(STbDataIter *pIter); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== @@ -772,6 +770,40 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +static FORCE_INLINE int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = tGetI64(p, &pRow->version); + pRow->pTSRow = (STSRow *)(p + n); + n += pRow->pTSRow->len; + return n; +} + +static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { + if (pIter == NULL) return NULL; + + if (pIter->pRow) { + return pIter->pRow; + } + + if (pIter->backward) { + if (pIter->pNode == pIter->pTbData->sl.pHead) { + return NULL; + } + } else { + if (pIter->pNode == pIter->pTbData->sl.pTail) { + return NULL; + } + } + + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + pIter->pRow = &pIter->row; + + return pIter->pRow; +} + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index c663e2b526..52c7a07c49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -294,31 +294,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { - // we add here for commit usage - if (pIter == NULL) return NULL; - - if (pIter->pRow) { - goto _exit; - } - - if (pIter->backward) { - if (pIter->pNode == pIter->pTbData->sl.pHead) { - goto _exit; - } - } else { - if (pIter->pNode == pIter->pTbData->sl.pTail) { - goto _exit; - } - } - - tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); - pIter->pRow = &pIter->row; - -_exit: - return pIter->pRow; -} - static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 52b74aea3f..9ac9a7d66d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -575,16 +575,6 @@ int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { return n; } -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { - int32_t n = 0; - - n += tGetI64(p, &pRow->version); - pRow->pTSRow = (STSRow *)(p + n); - n += pRow->pTSRow->len; - - return n; -} - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } @@ -1053,7 +1043,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS tRowIterInit(&rIter, pRow, pTSchema); pColVal = tRowIterNext(&rIter); for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; while (pColVal && pColVal->cid < pColData->cid) { pColVal = tRowIterNext(&rIter); diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 9fe37505ad..99d6decfc9 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,6 +37,12 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); +static sem_t canCommit = {0}; + +static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; +void vnode_wait_commit() { tsem_wait(&canCommit); } +void vnode_done_commit() { tsem_wait(&canCommit); } + int vnodeInit(int nthreads) { int8_t init; int ret; From d672e7a05c063496ab57618bbaf4f9d9049dd274 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 16:18:28 +0800 Subject: [PATCH 079/101] refactor(sync): add FpApplyQueueItems in fsm --- include/libs/sync/sync.h | 1 + source/dnode/mnode/impl/src/mndSync.c | 22 +++++- source/dnode/vnode/src/vnd/vnodeSync.c | 21 +++++- source/libs/sync/src/syncMain.c | 96 +++++++++++++------------- source/libs/sync/src/syncUtil.c | 4 +- 5 files changed, 90 insertions(+), 54 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 74a73f6b10..a27be95049 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -139,6 +139,7 @@ typedef struct SSyncFSM { void (*FpReConfigCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); + int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm); void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm); void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index ebab83cf6d..450d7a30ed 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -205,8 +205,23 @@ static void mndBecomeLeader(const SSyncFSM *pFsm) { static bool mndApplyQueueEmpty(const SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; - int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); - return (itemSize == 0); + if (pMnode != NULL && pMnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); + return (itemSize == 0); + } else { + return true; + } +} + +static int32_t mndApplyQueueItems(const SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + + if (pMnode != NULL && pMnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); + return itemSize; + } else { + return -1; + } } SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { @@ -218,6 +233,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpLeaderTransferCb = NULL; pFsm->FpApplyQueueEmptyCb = mndApplyQueueEmpty; + pFsm->FpApplyQueueItems = mndApplyQueueItems; pFsm->FpReConfigCb = NULL; pFsm->FpBecomeLeaderCb = mndBecomeLeader; pFsm->FpBecomeFollowerCb = mndBecomeFollower; @@ -291,7 +307,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; if (req.contLen <= 0) return -1; - + req.pCont = rpcMallocCont(req.contLen); if (req.pCont == NULL) return -1; memcpy(req.pCont, pRaw, req.contLen); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e7f8c9f562..38cb534d7f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -438,8 +438,24 @@ static void vnodeBecomeLeader(const SSyncFSM *pFsm) { static bool vnodeApplyQueueEmpty(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; - int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); - return (itemSize == 0); + + if (pVnode != NULL && pVnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); + return (itemSize == 0); + } else { + return true; + } +} + +static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + + if (pVnode != NULL && pVnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); + return itemSize; + } else { + return -1; + } } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { @@ -452,6 +468,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = NULL; pFsm->FpApplyQueueEmptyCb = vnodeApplyQueueEmpty; + pFsm->FpApplyQueueItems = vnodeApplyQueueItems; pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; pFsm->FpReConfigCb = NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b4a3d30f7d..f6925946e9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1381,37 +1381,37 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerLogicClock); cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerCounter); cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); // elect timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->electTimerLogicClock); cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->electTimerCounter); cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); // heartbeat timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerLogicClock); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerCounter); cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); // callback @@ -1472,7 +1472,7 @@ inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); snprintf(s, len, - "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + "vgId:%d, sync %s, tm:%" PRId64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 ", sby:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d", @@ -1629,7 +1629,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "snapshot sender reset for: %" PRIu64 ", newIndex:%d, %s:%d, %p", + sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p", (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); (pSyncNode->senders)[i] = oldSenders[j]; @@ -1698,7 +1698,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, term); char tmpBuf[64]; - snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRIu64, term); + snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRId64, term); syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); } @@ -1712,20 +1712,20 @@ void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm) { if (pSyncNode->pRaftStore->currentTerm > newTerm) { - sNTrace(pSyncNode, "step down, ignore, new-term:%" PRIu64 ", current-term:%" PRIu64, newTerm, + sNTrace(pSyncNode, "step down, ignore, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, pSyncNode->pRaftStore->currentTerm); return; } do { - sNTrace(pSyncNode, "step down, new-term:%" PRIu64 ", current-term:%" PRIu64, newTerm, + sNTrace(pSyncNode, "step down, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, pSyncNode->pRaftStore->currentTerm); } while (0); if (pSyncNode->pRaftStore->currentTerm < newTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, newTerm); char tmpBuf[64]; - snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRIu64, newTerm); + snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRId64, newTerm); syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); @@ -2044,7 +2044,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { } } - sNError(pSyncNode, "sync node get pre term error, index:%" PRId64 ", snap-index:%" PRId64 ", snap-term:%" PRIu64, + sNError(pSyncNode, "sync node get pre term error, index:%" PRId64 ", snap-index:%" PRId64 ", snap-term:%" PRId64, index, snapshot.lastApplyIndex, snapshot.lastApplyTerm); return SYNC_TERM_INVALID; } @@ -2087,7 +2087,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } } else { - sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64, + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRId64 ", pingTimerLogicClockUser:%" PRId64, pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -2109,7 +2109,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { taosMemoryFree(pElectTimer); return; } - sNTrace(pSyncNode, "eq elect timer lc:%" PRIu64, pSyncMsg->logicClock); + sNTrace(pSyncNode, "eq elect timer lc:%" PRId64, pSyncMsg->logicClock); } else { sTrace("syncNodeEqElectTimer syncEqMsg is NULL"); } @@ -2162,7 +2162,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { sError("sync env is stop, syncNodeEqHeartbeatTimer"); } } else { - sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRId64 ", heartbeatTimerLogicClockUser:%" PRId64 "", pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } @@ -2224,7 +2224,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { } } else { - sTrace("==syncNodeEqPeerHeartbeatTimer== timerLogicClock:%" PRIu64 ", msgLogicClock:%" PRIu64 "", timerLogicClock, + sTrace("==syncNodeEqPeerHeartbeatTimer== timerLogicClock:%" PRId64 ", msgLogicClock:%" PRId64 "", timerLogicClock, msgLogicClock); } } @@ -2360,7 +2360,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg) { sError("vgId:%d, sync enqueue fc-commit msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index: %" PRIu64, ths->vgId, pSyncMsg->fcIndex); + sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index:%" PRId64, ths->vgId, pSyncMsg->fcIndex); } } } @@ -2381,7 +2381,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg) { sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRIu64, ths->vgId, pSyncMsg->sdNewTerm); + sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRId64, ths->vgId, pSyncMsg->sdNewTerm); } } @@ -2553,7 +2553,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p } if (pEntry->term < ths->pRaftStore->currentTerm) { - sNTrace(ths, "little term:%" PRIu64 ", can not do leader transfer", pEntry->term); + sNTrace(ths, "little term:%" PRId64 ", can not do leader transfer", pEntry->term); return 0; } @@ -2583,7 +2583,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p int32_t ret = syncNodeRestartElectTimer(ths, electMS); ASSERT(ret == 0); - sNTrace(ths, "maybe leader transfer to %s:%d %" PRIu64, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, + sNTrace(ths, "maybe leader transfer to %s:%d %" PRId64, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); } @@ -2831,7 +2831,7 @@ const char* syncTimerTypeStr(enum ESyncTimeoutType timerType) { } void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s) { - sNTrace(pSyncNode, "recv sync-timer {type:%s, lc:%" PRIu64 ", ms:%d, data:%p}, %s", + sNTrace(pSyncNode, "recv sync-timer {type:%s, lc:%" PRId64 ", ms:%d, data:%p}, %s", syncTimerTypeStr(pMsg->timeoutType), pMsg->logicClock, pMsg->timerMS, pMsg->data, s); } @@ -2839,7 +2839,7 @@ void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, c char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", + sNTrace(pSyncNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } @@ -2848,7 +2848,7 @@ void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, c char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", + sNTrace(pSyncNode, "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } @@ -2856,7 +2856,7 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, pMsg->term, + sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, pMsg->voteGranted, s); } @@ -2864,7 +2864,7 @@ void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, pMsg->term, + sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, pMsg->voteGranted, s); } @@ -2873,8 +2873,8 @@ void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d}, %s", + "send sync-append-entries to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, s); } @@ -2885,8 +2885,8 @@ void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 - ", cmt:%" PRIu64 ", pterm:%" PRIu64 ", datalen:%d}, %s", + "recv sync-append-entries from %s:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", cmt:%" PRId64 ", pterm:%" PRId64 ", datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, pMsg->dataLen, s); } @@ -2897,8 +2897,8 @@ void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + "send sync-append-entries-batch to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); } @@ -2909,8 +2909,8 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + "recv sync-append-entries-batch from %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); } @@ -2921,7 +2921,7 @@ void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "send sync-append-entries-reply to %s:%d, {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, match:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); } @@ -2932,7 +2932,7 @@ void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "recv sync-append-entries-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, match:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); } @@ -2943,7 +2943,7 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRIu64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRIu64 + "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); } @@ -2954,7 +2954,7 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat from %s:%d {term:%" PRIu64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRIu64 + "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); } @@ -2964,7 +2964,7 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 "}, %s", host, port, + sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, s); } @@ -2972,34 +2972,34 @@ void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 "}, %s", host, port, + sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, s); } void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s) { - sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRIu64 "}, %s", pMsg->cmd, - syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, s); + sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRId64 ", fc-index:%" PRId64 "}, %s", pMsg->cmd, + syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, pMsg->fcIndex, s); } void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-pre-snapshot to %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s); + sNTrace(pSyncNode, "send sync-pre-snapshot to %s:%d {term:%" PRId64 "}, %s", host, port, pMsg->term, s); } void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-pre-snapshot from %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s); + sNTrace(pSyncNode, "recv sync-pre-snapshot from %s:%d {term:%" PRId64 "}, %s", host, port, pMsg->term, s); } void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-pre-snapshot-reply to %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port, + sNTrace(pSyncNode, "send sync-pre-snapshot-reply to %s:%d {term:%" PRId64 ", snap-start:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->snapStart, s); } @@ -3007,7 +3007,7 @@ void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshot char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-pre-snapshot-reply from %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, + sNTrace(pSyncNode, "recv sync-pre-snapshot-reply from %s:%d {term:%" PRId64 ", snap-start:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->snapStart, s); } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index e82578ef4d..590cb21866 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -205,7 +205,7 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { if (i < pSyncNode->replicaNum - 1) { len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 ", ", i, pState->lastSendIndex, - pState->lastSendTime); + pState->lastSendTime); } else { len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 "}", i, pState->lastSendIndex, pState->lastSendTime); @@ -242,6 +242,8 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); + // int32_t aq = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + taosPrintLog(flags, level, dflag, "vgId:%d, sync %s " "%s" From 18363f52e50d0ee8716f3a66a2ebcf8c552519d2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 16:35:26 +0800 Subject: [PATCH 080/101] refactor(sync): add FpApplyQueueItems in fsm --- source/libs/sync/src/syncAppendEntries.c | 3 ++- source/libs/sync/src/syncUtil.c | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7d6e358511..a72746b5a4 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -246,7 +246,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { } else { // error char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "ignore, get local entry error, append-index:%" PRId64, appendIndex); + snprintf(logBuf, sizeof(logBuf), "ignore, get local entry error, append-index:%" PRId64 " err:%d", appendIndex, + terrno); syncLogRecvAppendEntries(ths, pMsg, logBuf); syncEntryDestory(pLocalEntry); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 590cb21866..24f961d1ce 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -242,18 +242,21 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - // int32_t aq = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + // save error code, otherwise it will be overwritten by FpApplyQueueItems + int32_t errCode = terrno; + int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + terrno = errCode; taosPrintLog(flags, level, dflag, "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64 + ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, bch:%d, r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->pRaftCfg->isStandBy, pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize, - pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, + pNode->pRaftCfg->isStandBy, aqItems, pNode->pRaftCfg->batchSize, pNode->replicaNum, + pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); } From 652c6388f2aeb5a52849ef6a3af3264b7b672b8c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 9 Nov 2022 16:40:17 +0800 Subject: [PATCH 081/101] fix: fix valgrind invalid read error --- source/libs/scalar/src/sclfunc.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c89072233f..22fa99d75d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1028,11 +1028,11 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t type = GET_PARAM_TYPE(pInput); bool tzPresent = (inputNum == 2) ? true : false; - char *tz; - int32_t tzLen; + char tz[20] = {0}; + int32_t tzLen = 0; if (tzPresent) { - tz = varDataVal(pInput[1].columnData->pData); tzLen = varDataLen(pInput[1].columnData->pData); + memcpy(tz, varDataVal(pInput[1].columnData->pData), tzLen); } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -1071,8 +1071,10 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t len = (int32_t)strlen(buf); // add timezone string - snprintf(buf + len, tzLen + 1, "%s", tz); - len += tzLen; + if (tzLen > 0) { + snprintf(buf + len, tzLen + 1, "%s", tz); + len += tzLen; + } if (hasFraction) { int32_t fracLen = (int32_t)strlen(fraction) + 1; From 2391391621914d9b3215afdf2124151afec4d0ed Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 16:44:05 +0800 Subject: [PATCH 082/101] refactor(sync): save error code in syncPrintNodeLog --- source/libs/sync/src/syncUtil.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 24f961d1ce..1a00b0f5a4 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -216,6 +216,9 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) { if (pNode == NULL || pNode->pRaftCfg != NULL && pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + // save error code, otherwise it will be overwritten + int32_t errCode = terrno; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot); @@ -242,9 +245,9 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - // save error code, otherwise it will be overwritten by FpApplyQueueItems - int32_t errCode = terrno; int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + + // restore error code terrno = errCode; taosPrintLog(flags, level, dflag, From 3f62f14cee3a952c9155a235991e69fc5e1e7797 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 16:54:33 +0800 Subject: [PATCH 083/101] refactor: do some internal refactor. --- source/libs/function/src/builtinsimpl.c | 38 ++++++------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fbcaac95df..55642fa1a1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3097,22 +3097,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { #else int64_t* pts = (int64_t*)pInput->pPTS->pData; - if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; - } - - numOfElems++; - - char* data = colDataGetVarData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; - } - } - } else { #if 0 for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { @@ -3120,16 +3104,13 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - - char* data = colDataGetNumData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } #else - if (!pInputCol->hasNull) { int32_t round = pInput->numOfRows >> 2; int32_t reminder = pInput->numOfRows & 0x03; @@ -3155,7 +3136,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - char* data = colDataGetNumData(pInputCol, chosen); + char* data = colDataGetData(pInputCol, chosen); doSaveCurrentVal(pCtx, i, cts, type, data); pResInfo->numOfRes = 1; } @@ -3163,10 +3144,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { numOfElems++; - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - char* data = colDataGetNumData(pInputCol, i); - doSaveCurrentVal(pCtx, i, cts, type, data); + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } @@ -3179,14 +3159,14 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { - char* data = colDataGetNumData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); doSaveCurrentVal(pCtx, i, pts[i], type, data); pResInfo->numOfRes = 1; } } } #endif - } + #endif // save selectivity value for column consisted of all null values From f4a5cb995d889b9cae32fc536ddebc939250ab95 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 9 Nov 2022 17:04:22 +0800 Subject: [PATCH 084/101] Revert "test:refact unsuccessful case" This reverts commit d550e43c5aaf535de40534f5954ee1570536f43b. --- tests/system-test/fulltest.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4228d1e419..0729d36f12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -76,8 +76,8 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/count_partition.py -R python3 ./test.py -f 2-query/count.py python3 ./test.py -f 2-query/count.py -R -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/db.py -R python3 ./test.py -f 2-query/diff.py @@ -393,7 +393,7 @@ python3 ./test.py -f 2-query/max.py -Q 2 python3 ./test.py -f 2-query/min.py -Q 2 python3 ./test.py -f 2-query/mode.py -Q 2 python3 ./test.py -f 2-query/count.py -Q 2 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 python3 ./test.py -f 2-query/last.py -Q 2 python3 ./test.py -f 2-query/first.py -Q 2 python3 ./test.py -f 2-query/To_iso8601.py -Q 2 @@ -490,7 +490,7 @@ python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 python3 ./test.py -f 2-query/mode.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 python3 ./test.py -f 2-query/To_iso8601.py -Q 3 @@ -589,7 +589,7 @@ python3 ./test.py -f 2-query/max.py -Q 4 python3 ./test.py -f 2-query/min.py -Q 4 python3 ./test.py -f 2-query/mode.py -Q 4 python3 ./test.py -f 2-query/count.py -Q 4 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 python3 ./test.py -f 2-query/last.py -Q 4 python3 ./test.py -f 2-query/first.py -Q 4 python3 ./test.py -f 2-query/To_iso8601.py -Q 4 From 464f108e48c3c14e1622fb1369e19628eae205c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 17:14:01 +0800 Subject: [PATCH 085/101] fix(query): fix syntax errors. --- source/libs/function/src/tudf.c | 3 ++- source/util/src/tlog.c | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 1dedfe8364..8715aa0be1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -131,7 +131,8 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); - strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib)); + udfdPathLdLib[udfdLdLibPathLen] = ':'; strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a03c04ed6e..1bd2c6f0a0 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -317,14 +317,14 @@ static void taosGetLogFileName(char *fn) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { char fileName[LOG_FILE_NAME_LEN]; - sprintf(fileName, "%s%d.0", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.0", fn, i); bool file1open = taosCheckFileIsOpen(fileName); - sprintf(fileName, "%s%d.1", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.1", fn, i); bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { - sprintf(tsLogObj.logName, "%s%d", fn, i); + snprintf(tsLogObj.logName, LOG_FILE_NAME_LEN, "%s%d", fn, i); return; } } @@ -598,7 +598,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(pLogBuf) - end - 1); if (lostLine > 0) { - sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine); + snprintf(tmpBuf, tListLen(tmpBuf), "...Lost %" PRId64 " lines here...\n", lostLine); tmpBufLen = (int32_t)strlen(tmpBuf); } From fcc50467f43f20abd303bd80fbc37d6af23b6ff2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 17:33:22 +0800 Subject: [PATCH 086/101] fix(sync): fix AddressSanitizer error --- source/libs/sync/src/syncMain.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f6925946e9..85668068a5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1040,6 +1040,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); + pSyncNode->pRaftStore = NULL; syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2174,10 +2175,18 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = pData->pSyncNode; SSyncTimer* pSyncTimer = pData->pTimer; + if (pSyncNode == NULL) { + return; + } + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return; } + if (pSyncNode->pRaftStore == NULL) { + return; + } + // sNTrace(pSyncNode, "eq peer hb timer"); int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); From f2fd847414f7841eec4e792fe61ee3063265f8d2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 17:35:22 +0800 Subject: [PATCH 087/101] refact: move sync test code to single lib --- source/libs/sync/src/syncMain.c | 8 +------- source/libs/sync/src/syncReplication.c | 2 +- tests/script/tsim/vnode/replica3_repeat.sim | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d05e1608e2..b4ca3f5b01 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2223,10 +2223,6 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn } else { // del resp mgr, call FpCommitCb - - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SFsmCbMeta cbMeta = { .index = pEntry->index, .lastConfigIndex = SYNC_INDEX_INVALID, @@ -2238,9 +2234,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn .currentTerm = ths->pRaftStore->currentTerm, .flag = 0, }; - - syncRespMgrGetAndDel(ths->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, &cbMeta); + ths->pFsm->FpCommitCb(ths->pFsm, pMsg, &cbMeta); if (h) { taosLRUCacheRelease(ths->pLogStore->pCache, h, false); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 8c3f7e5384..3f22a1f2cd 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -78,7 +78,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); - memcpy(pMsg->data, pEntry->data, pEntry->bytes); + memcpy(pMsg->data, pEntry, pEntry->bytes); syncEntryDestory(pEntry); } else { diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index ccff06819b..8efba515ae 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -118,7 +118,7 @@ step7: sql select count(*) from db.tb -x step7 print select count(*) from db.tb ==> $data00 $lastRows -if $data00 <= $lastRows then +if $data00 < $lastRows then return -1 endi From 153c01ee7328678101f434edf896a66dbdcec90c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 17:38:46 +0800 Subject: [PATCH 088/101] fix(sync): fix AddressSanitizer error --- source/libs/sync/src/syncMain.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b4a3d30f7d..09be3e45c3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1040,6 +1040,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); + pSyncNode->pRaftStore = NULL; syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2174,10 +2175,18 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = pData->pSyncNode; SSyncTimer* pSyncTimer = pData->pTimer; + if (pSyncNode == NULL) { + return; + } + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return; } + if (pSyncNode->pRaftStore == NULL) { + return; + } + // sNTrace(pSyncNode, "eq peer hb timer"); int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); From 6298f92d563f768909d3abde1b4f864b5bdeef7d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 9 Nov 2022 17:39:28 +0800 Subject: [PATCH 089/101] fix: some problems of parser --- source/libs/parser/src/parTranslater.c | 34 +++++++++++++++++++++-- source/libs/parser/test/parSelectTest.cpp | 2 ++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dea3b959ed..82170a27d8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2393,6 +2393,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { pCxt->stableQuery = true; } + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { + ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + } code = addNamespace(pCxt, pRealTable); } break; @@ -3428,6 +3431,19 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return (SNode*)pCol; } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); @@ -3440,7 +3456,8 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) { SExprNode* pLeftExpr = (SExprNode*)pLeft; SExprNode* pRightExpr = (SExprNode*)pRight; - if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { + int32_t comp = dataTypeComp(&pLeftExpr->resType, &pRightExpr->resType); + if (comp > 0) { SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); if (TSDB_CODE_SUCCESS != code) { @@ -3448,9 +3465,20 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } REPLACE_LIST2_NODE(pRightFunc); pRightExpr = (SExprNode*)pRightFunc; + } else if (comp < 0) { + SNode* pLeftFunc = NULL; + int32_t code = createCastFunc(pCxt, pLeft, pRightExpr->resType, &pLeftFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST1_NODE(pLeftFunc); + SExprNode* pLeftFuncExpr = (SExprNode*)pLeftFunc; + snprintf(pLeftFuncExpr->aliasName, sizeof(pLeftFuncExpr->aliasName), "%s", pLeftExpr->aliasName); + snprintf(pLeftFuncExpr->userAlias, sizeof(pLeftFuncExpr->userAlias), "%s", pLeftExpr->userAlias); + pLeft = pLeftFunc; + pLeftExpr = pLeftFuncExpr; } - strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); - pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; + snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pSetOperator->stmtName, pLeft))) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 0027ac9ca1..fcd8dd1f26 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -425,6 +425,8 @@ TEST_F(ParserSelectTest, informationSchema) { run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); + + run("SELECT * FROM (SELECT table_name FROM ins_tables) t WHERE table_name = 'a'"); } TEST_F(ParserSelectTest, withoutFrom) { From 9f7e06721c3009f1f57dabc2462f7b0aa752809d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 9 Nov 2022 18:48:52 +0800 Subject: [PATCH 090/101] enh: support vgroup epset update --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 3 ++- include/libs/qcom/query.h | 1 + source/client/src/clientHb.c | 4 +++- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 27 +++++++++++++++++---- source/libs/catalog/src/catalog.c | 4 ++-- source/libs/catalog/src/ctgAsync.c | 1 + source/libs/catalog/src/ctgCache.c | 29 +++++++++++++---------- source/libs/catalog/src/ctgDbg.c | 26 +++++++++++++++++--- source/libs/catalog/test/catalogTests.cpp | 3 ++- source/libs/parser/inc/parUtil.h | 2 +- source/libs/parser/src/parTranslater.c | 8 +++---- source/libs/parser/src/parUtil.c | 3 ++- source/libs/parser/test/mockCatalog.cpp | 2 +- source/libs/qcom/src/querymsg.c | 4 +++- 16 files changed, 86 insertions(+), 34 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b840053b08..75c34f88a2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -190,6 +190,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SBuildUseDBInput; typedef struct SField { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 7717d3bccc..6154882756 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -56,6 +56,7 @@ typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; int64_t dbId; + int64_t stateTs; } SDbInfo; typedef struct STablesReq { @@ -153,7 +154,7 @@ int32_t catalogInit(SCatalogCfg* cfg); */ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* stateTs); /** * Get a DB's all vgroup info. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 74c1a8c07c..40b7e97fc7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -127,6 +127,7 @@ typedef struct SDBVgInfo { int16_t hashSuffix; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index fa0e3212a3..bf34d3e2df 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -61,7 +61,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); @@ -72,6 +72,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } vgInfo->vgVersion = rsp->vgVersion; + vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; vgInfo->hashPrefix = rsp->hashPrefix; vgInfo->hashSuffix = rsp->hashSuffix; @@ -486,6 +487,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); db->numOfTable = htonl(db->numOfTable); + db->stateTs = htobe64(db->stateTs); } SKv kv = { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e17e7e79c6..095bc2d709 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -181,7 +181,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); struct SCatalog* pCatalog = NULL; - if (usedbRsp.vgVersion >= 0) { + if (usedbRsp.vgVersion >= 0) { // cached in local uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2a4462bf5..9973f02bb9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1175,7 +1175,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { + if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); } @@ -1275,12 +1275,31 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, for (int32_t i = 0; i < numOfDbs; ++i) { SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); + pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); SUseDbRsp usedbRsp = {0}; + if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); + if (pDbVgVersion->vgVersion < vgVersion) { + usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); + usedbRsp.vgVersion = vgVersion++; + } else { + usedbRsp.vgVersion = pDbVgVersion->vgVersion; + } + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + + continue; + } + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { mTrace("db:%s, no exist", pDbVgVersion->dbFName); @@ -1293,8 +1312,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable /* && - pDbVgVersion->stateTs == pDb->stateTs */) { + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && + pDbVgVersion->stateTs == pDb->stateTs) { mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1f87066c82..2dcd681205 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -715,10 +715,10 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* pStateTs) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId || NULL == tableNum || NULL == pStateTs) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b601865306..ff3bd4d33c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1998,6 +1998,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + pInfo->stateTs = dbCache->vgCache.vgInfo->stateTs; ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 51807a145a..e99beca69a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1231,14 +1231,16 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + } return TSDB_CODE_SUCCESS; @@ -1563,7 +1565,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1579,15 +1581,15 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SDBVgInfo *vgInfo = vgCache->vgInfo; if (dbInfo->vgVersion < vgInfo->vgVersion) { - ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgDebug("db updateVgroup is ignored, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); goto _return; } - if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { - ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, - dbInfo->numOfTable); + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable && dbInfo->stateTs == vgInfo->stateTs) { + ctgDebug("no new db vgroup update info, dbFName:%s, vgVer:%d, numOfTable:%d, stateTs:%" PRId64, dbFName, dbInfo->vgVersion, + dbInfo->numOfTable, dbInfo->stateTs); ctgWUnlockVgInfo(dbCache); goto _return; @@ -1599,15 +1601,17 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + } _return: @@ -2170,7 +2174,6 @@ void *ctgUpdateThreadFunc(void *param) { CTG_RT_STAT_INC(numOfOpDequeue, 1); ctgdShowCacheInfo(); - ctgdShowClusterCache(pCtg); } qInfo("catalog update thread stopped"); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 26b5903bb0..b6ff1c8b98 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -78,7 +78,7 @@ void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { num = taosArrayGetSize(pResult->pDbInfo); for (int32_t i = 0; i < num; ++i) { SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i); - qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); + qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, stateTs:%" PRId64 " dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->stateTs, pDb->dbId); } } else { qDebug("empty db info"); @@ -462,6 +462,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { int32_t hashMethod = -1; int16_t hashPrefix = 0; int16_t hashSuffix = 0; + int64_t stateTs = 0; int32_t vgNum = 0; if (dbCache->vgCache.vgInfo) { @@ -469,16 +470,35 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { hashMethod = dbCache->vgCache.vgInfo->hashMethod; hashPrefix = dbCache->vgCache.vgInfo->hashPrefix; hashSuffix = dbCache->vgCache.vgInfo->hashSuffix; + stateTs = dbCache->vgCache.vgInfo->stateTs; if (dbCache->vgCache.vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } ctgDebug("[%d] db [%.*s][0x%" PRIx64 - "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, + "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs, hashMethod, hashPrefix, hashSuffix, vgNum); + if (dbCache->vgCache.vgInfo) { + int32_t i = 0; + void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL); + while (pVgIter) { + SVgroupInfo * pVg = (SVgroupInfo *)pVgIter; + + ctgDebug("The %04dth VG [id:%d, hashBegin:%u, hashEnd:%u, numOfTable:%d, epNum:%d, inUse:%d]", + i++, pVg->vgId, pVg->hashBegin, pVg->hashEnd, pVg->numOfTable, pVg->epSet.numOfEps, pVg->epSet.inUse); + + for (int32_t n = 0; n < pVg->epSet.numOfEps; ++n) { + SEp *pEp = &pVg->epSet.eps[n]; + ctgDebug("\tEp %d [fqdn:%s, port:%d]", n, pEp->fqdn, pEp->port); + } + + pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter); + } + } + pIter = taosHashIterate(dbHash, pIter); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebf7c7baeb..5e543384ac 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2486,7 +2486,8 @@ TEST(dbVgroup, getSetDbVgroupCase) { int32_t dbVer = 0; int64_t dbId = 0; int32_t tbNum = 0; - code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum); + int64_t stateTs = 0; + code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum, &stateTs); ASSERT_EQ(code, 0); ASSERT_EQ(dbVer, ctgTestVgVersion); ASSERT_EQ(dbId, ctgTestDbId); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 10a86866d5..c53d3f9320 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -108,7 +108,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum); + int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dea3b959ed..dc43985a70 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -466,14 +466,14 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SParseContext* pParCxt = pCxt->pParseCxt; int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { - code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum); + code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } else { - code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); + code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } } if (TSDB_CODE_SUCCESS != code) { @@ -4941,7 +4941,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); + int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 466c6edd24..00a72a1946 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -876,13 +876,14 @@ int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCac } int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SDbInfo* pDbInfo = NULL; int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo); if (TSDB_CODE_SUCCESS == code) { *pVersion = pDbInfo->vgVer; *pDbId = pDbInfo->dbId; *pTableNum = pDbInfo->tbNum; + *pStateTs = pDbInfo->stateTs; } return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 4f5ddd9a51..8f051c67a0 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -249,7 +249,7 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con } int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, - int32_t* tableNum) { + int32_t* tableNum, int64_t* stateTs) { return 0; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 181abc2247..fadc39f21d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -41,8 +41,9 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix; pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix; + pOut->dbVgroup->stateTs = usedbRsp->stateTs; - qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); + qDebug("Got %d vgroup for db %s, vgVersion:%d, stateTs:%" PRId64, usedbRsp->vgNum, usedbRsp->db, usedbRsp->vgVersion, usedbRsp->stateTs); if (usedbRsp->vgNum <= 0) { return TSDB_CODE_SUCCESS; @@ -103,6 +104,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms usedbReq.vgVersion = pInput->vgVersion; usedbReq.dbId = pInput->dbId; usedbReq.numOfTable = pInput->numOfTable; + usedbReq.stateTs = pInput->stateTs; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); From 0c92bd6e376bc747b12007c2742ccae4afdeea84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 9 Nov 2022 18:58:35 +0800 Subject: [PATCH 091/101] make pass ci --- source/dnode/vnode/src/vnd/vnodeModule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 99d6decfc9..782ffd788d 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,7 +37,7 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); -static sem_t canCommit = {0}; +static tsem_t canCommit = {0}; static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; void vnode_wait_commit() { tsem_wait(&canCommit); } From 98b77fe8b473b093b3136d29e414352c72c95529 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:02:47 +0800 Subject: [PATCH 092/101] refact: adjust head file --- source/libs/sync/inc/syncAppendEntries.h | 4 ---- source/libs/sync/inc/syncAppendEntriesReply.h | 4 ---- source/libs/sync/inc/syncCommit.h | 4 ---- source/libs/sync/inc/syncElection.h | 4 ---- source/libs/sync/inc/syncIndexMgr.h | 4 ---- source/libs/sync/inc/syncMessage.h | 5 ----- source/libs/sync/inc/syncRaftCfg.h | 5 ----- source/libs/sync/inc/syncRaftEntry.h | 5 ----- source/libs/sync/inc/syncRaftLog.h | 4 ---- source/libs/sync/inc/syncRaftStore.h | 5 ----- source/libs/sync/inc/syncReplication.h | 4 ---- source/libs/sync/inc/syncRequestVote.h | 4 ---- source/libs/sync/inc/syncRequestVoteReply.h | 4 ---- source/libs/sync/inc/syncRespMgr.h | 4 ---- source/libs/sync/inc/syncSnapshot.h | 5 ----- source/libs/sync/inc/syncTimeout.h | 4 ---- source/libs/sync/src/syncRaftEntry.c | 1 + 17 files changed, 1 insertion(+), 69 deletions(-) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index dc40e2fc72..0a67939d35 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 0227d832fc..d2dff92d43 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index 7458ce28ab..d3ba556f82 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 9ccd9dd28f..662c17094e 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // RequestVote(i, j) == diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index e8f17537b4..bd88f5cdce 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 19079d518d..936081c7b2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 15ca82664a..6b6ca21399 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define CONFIG_FILE_LEN 2048 diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 8b8ab41b53..2a73b95e4d 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" -#include "tref.h" #include "tskiplist.h" typedef struct SSyncRaftEntry { diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index c25d4ae34e..17ea36c640 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncRaftEntry.h" -#include "taosdef.h" #include "wal.h" typedef struct SSyncLogStoreData { diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index ac298c5375..077e6a7658 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 4f15a45cec..e08430327b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // AppendEntries(i, j) == diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 73b4a0efae..c51ea4eea6 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 6bef18405c..1adab30aaa 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index 9026ecb66e..1d41c0a85f 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" typedef struct SRespStub { SRpcMsg rpcMsg; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 5594fe46ed..d688a3c1bd 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 112a3d8610..e1a6050d92 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index c5b1399c8c..262a7b0879 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" +#include "tref.h" SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; From fb9e62afe24cdab219a71cc7ad9fac229c82e362 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 9 Nov 2022 19:02:50 +0800 Subject: [PATCH 093/101] refactor(sync): if index less than wal.commitVer, do not truncate, otherwise it will print much log --- source/libs/sync/src/syncRaftLog.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d3d69b288b..7b1158319e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -301,6 +301,12 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return 0; } + // need not truncate + SyncIndex walCommitVer = walGetCommittedVer(pWal); + if (fromIndex <= walCommitVer) { + return 0; + } + int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; From 2402fd1d4867bbf5a67a87a8964eab5646b01beb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 19:14:27 +0800 Subject: [PATCH 094/101] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/cachescanoperator.c | 13 +- source/libs/executor/src/exchangeoperator.c | 11 +- source/libs/executor/src/executorimpl.c | 35 +++-- source/libs/executor/src/groupoperator.c | 27 +--- source/libs/executor/src/joinoperator.c | 8 +- source/libs/executor/src/projectoperator.c | 23 +--- source/libs/executor/src/scanoperator.c | 80 +++-------- source/libs/executor/src/sortoperator.c | 31 +---- source/libs/executor/src/tfill.c | 12 +- source/libs/executor/src/timewindowoperator.c | 129 ++++++------------ source/libs/function/src/builtinsimpl.c | 6 +- 12 files changed, 117 insertions(+), 262 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 77915624dd..62146b6048 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -881,7 +881,9 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doDestroyExchangeOperatorInfo(void* param); -void doSetOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, + SExecTaskInfo* pTaskInfo); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo * pCache); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 41fcc44f97..6f9084ce52 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -93,12 +93,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - pOperator->name = "LastrowScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = @@ -126,7 +121,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); if (size == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -182,7 +177,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes += 1; return pRes; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } else { @@ -234,7 +229,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index cb3330e9ca..049de727df 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -213,7 +213,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { pExchangeInfo->limitInfo.numOfOutputRows += rows; if (rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } else { return pBlock; @@ -289,13 +289,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); return pOperator; @@ -506,7 +501,7 @@ void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, pLoadInfo->totalElapsed / 1000.0); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ca1d4cccab..709e981a1f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -94,7 +94,7 @@ static void destroyIntervalOperatorInfo(void* param); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void doSetOperatorCompleted(SOperatorInfo* pOperator) { +void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); @@ -102,6 +102,16 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo) { + pOperator->name = (char*)name; + pOperator->operatorType = type; + pOperator->blocking = blocking; + pOperator->status = status; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; +} + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; @@ -1795,7 +1805,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1805,7 +1815,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2054,7 +2064,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2131,7 +2141,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { while (true) { fillResult = doFillImpl(pOperator); if (fillResult == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2361,13 +2371,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupId = UINT64_MAX; - pOperator->name = "TableAggregate"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_AGG; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); @@ -2564,14 +2569,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pOperator->name = "FillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; + setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index cfcdd0ee64..cf8dd110b6 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -316,7 +316,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -438,12 +438,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "GroupbyAggOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); @@ -654,7 +649,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { // try next group data ++pInfo->groupIndex; if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); clearPartitionOperator(pInfo); return NULL; } @@ -821,14 +816,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; + setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); @@ -963,7 +953,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->pInputDataBlock = NULL; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } printDataBlock(pBlock, "stream partitionby recv"); @@ -1103,14 +1093,9 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; + setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index e7fe2bfc12..4e1daac643 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -73,14 +73,10 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + + setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 58cc082653..ce1d13775c 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -98,12 +98,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); @@ -153,7 +149,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return PROJECT_RETRIEVE_DONE; } @@ -187,7 +183,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS // TODO: optimize it later when partition by + limit if ((pLimitInfo->slimit.limit == -1 && pLimitInfo->currentGroupId == 0) || (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -252,7 +248,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { @@ -400,12 +396,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - pOperator->name = "IndefinitOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -498,7 +489,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -627,7 +618,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 903843bcc4..3da54509ce 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -820,7 +820,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -843,7 +843,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -865,7 +865,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -947,13 +947,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; - pOperator->name = "TableScanOperator"; // for debug purpose - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024*128, -1, .5); if (pInfo->metaCache.pTableMetaEntryCache == NULL) { @@ -986,13 +981,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1148,13 +1137,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - pOperator->name = "DataBlockDistScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; @@ -2367,9 +2350,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; - pOperator->name = "RawScanOperator"; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; @@ -2555,13 +2536,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->partitionSup.needCalc = false; - pOperator->name = "StreamScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); @@ -2899,7 +2875,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2952,7 +2928,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3742,7 +3718,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } if (i >= taosArrayGetSize(pIdx->uids)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } else { pIdx->lastIdx = i + 1; } @@ -3924,7 +3900,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3946,7 +3922,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { if (pInfo->showRewrite == false) { @@ -4198,14 +4174,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; @@ -4282,7 +4252,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { count += 1; if (++pInfo->curPos >= size) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -4334,14 +4304,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -4712,7 +4675,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; if (tableListSize == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->tableStartIndex = 0; @@ -4731,7 +4694,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } else { stopGroupTableMergeScan(pOperator); if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; @@ -4852,13 +4815,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - pOperator->name = "TableMergeScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 700612e0ab..fc53623d44 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -53,11 +53,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); - pOperator->name = "SortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; @@ -214,7 +210,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -428,7 +424,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); if (pInfo->prefetchedSortInput == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; @@ -453,7 +449,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { beginSortGroup(pOperator); } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { finishSortGroup(pOperator); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -509,14 +505,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort } pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - - pOperator->name = "GroupSortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); @@ -705,7 +694,7 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } return pBlock; @@ -774,13 +763,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->bufPageSize = getProperSortPageSize(rowSize); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. - pOperator->name = "MultiwayMerge"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index a6ac28829c..7c9d73ad13 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1443,7 +1443,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRes, "stream fill"); return pInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1512,7 +1512,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } if (pInfo->pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1690,13 +1690,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->srcRowIndex = 0; - - pOperator->name = "StreamFillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6d8282ff5d..ca250f304a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1221,7 +1221,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1232,7 +1232,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1269,7 +1269,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1739,7 +1739,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pOperator->pTaskInfo = pTaskInfo; pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -1777,12 +1776,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); @@ -1890,7 +1884,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1933,7 +1927,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2281,7 +2275,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2330,7 +2324,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2342,7 +2336,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2365,7 +2359,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2386,7 +2380,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } else { @@ -2448,7 +2442,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2463,7 +2457,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2557,13 +2551,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pScanInfo->cond.twindows = pInfo->win; pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; - pOperator->name = "TimeSliceOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); @@ -2633,13 +2621,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); @@ -2711,12 +2694,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW goto _error; } - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; @@ -3134,7 +3112,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -4024,7 +4002,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4124,7 +4102,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4191,13 +4169,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); @@ -4248,7 +4224,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -4321,7 +4297,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4332,20 +4308,21 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pOperator == NULL) { goto _error; } + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pInfo->isFinal = true; - pOperator->name = "StreamSessionFinalAggOperator"; - } else { - pInfo->isFinal = false; + pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); + char* name = (pInfo->isFinal)? "StreamSessionFinalAggOperator":"StreamSessionSemiAggOperator"; + + if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } + setOperatorInfo(pOperator, name, pPhyNode->type , false, OP_NOT_OPENED, pInfo, pTaskInfo); + pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); @@ -4575,7 +4552,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4641,7 +4618,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4706,12 +4683,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamStateAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, @@ -4861,7 +4833,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -4986,13 +4958,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); @@ -5239,7 +5205,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } size_t rows = pRes->info.rows; @@ -5298,14 +5264,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge } initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - - pOperator->name = "TimeMergeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; - + setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); @@ -5351,7 +5310,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, &pInfo->delKey); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); return NULL; } @@ -5535,11 +5494,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamIntervalOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 55642fa1a1..079e553b07 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3112,6 +3112,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } #else if (!pInputCol->hasNull) { + numOfElems = 1; + int32_t round = pInput->numOfRows >> 2; int32_t reminder = pInput->numOfRows & 0x03; @@ -3143,7 +3145,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { - numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { char* data = colDataGetData(pInputCol, i); doSaveCurrentVal(pCtx, i, pts[i], type, data); @@ -3173,7 +3174,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); } - SET_VAL(pResInfo, numOfElems, 1); + +// SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 808ee3775d9edb086d30f931cb7f3328fb3d1bf4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:42:30 +0800 Subject: [PATCH 095/101] fix: wait until db created while process use db req --- source/dnode/mnode/impl/src/mndDb.c | 10 +++++++++- source/dnode/mnode/impl/src/mndTrans.c | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2a4462bf5..8de1e976fc 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -606,7 +606,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } else { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and response after trans finished", createReq.db); + mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); code = TSDB_CODE_ACTION_IN_PROGRESS; goto _OVER; } else { @@ -1225,6 +1225,14 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { usedbRsp.vgVersion = usedbReq.vgVersion; usedbRsp.errCode = terrno; + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { + mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } + } + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ac05598bdc..db878d72b9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -938,11 +938,15 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - mInfo("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pInfo->ahandle); if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } + if (i != 0 && code == 0) { + code = TSDB_CODE_RPC_REDIRECT; + } + mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { From b595c07149f298c3fcbc8b9b266582d209aad401 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 9 Nov 2022 19:47:22 +0800 Subject: [PATCH 096/101] fix: wait until db created while process use db req --- source/dnode/mnode/impl/src/mndDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8de1e976fc..cdd0ce9fc5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1263,7 +1263,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; _OVER: - if (code != 0) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } From a2cec238ec255d1e423a408e7acb334c6f5923b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Nov 2022 22:16:56 +0800 Subject: [PATCH 097/101] fix(query): set correct operator type. --- source/libs/executor/src/groupoperator.c | 2 +- tests/script/tsim/parser/having_child.sim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 34398f9672..e31c8f588f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -438,7 +438,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - setOperatorInfo(pOperator, "GroupbyAggOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index ae78c806ca..db9a25365e 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -733,6 +733,7 @@ sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 group by f1 having sum(f1) > 2 order by f1; if $rows != 3 then + print expect 3 , actual: $rows return -1 endi if $data00 != 2.000000000 then From 8d6f129154fb2fc44ca59261ac52eabe8dd3ccf7 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Thu, 10 Nov 2022 08:14:21 +0800 Subject: [PATCH 098/101] fix: skip empty data in delete file --- source/dnode/vnode/src/tsdb/tsdbCache.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 76236e5078..291a5ab1eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -475,7 +475,7 @@ _err: } return code; } - +/* static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { int32_t code = 0; SArray *pDelIdxArray = NULL; @@ -499,7 +499,7 @@ _err: } return code; } - +*/ typedef enum { SFSLASTNEXTROW_FS, SFSLASTNEXTROW_FILESET, @@ -997,8 +997,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); - SDelIdx delIdx; - SDelFile *pDelFile = pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader *pDelFReader; @@ -1006,18 +1004,20 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; - code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); + SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); + + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); + if (code) goto _err; + + SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + + code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); if (code) { tsdbDelFReaderClose(&pDelFReader); goto _err; } + taosArrayDestroy(pDelIdxArray); tsdbDelFReaderClose(&pDelFReader); } else { code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); From f7d263862b3d8b508cd114669def31432ba3045e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 10:04:05 +0800 Subject: [PATCH 099/101] refact: remove assert and adjust log --- source/libs/sync/inc/syncSnapshot.h | 14 +- source/libs/sync/inc/syncTimeout.h | 1 - source/libs/sync/inc/syncTools.h | 1 - source/libs/sync/inc/syncUtil.h | 8 +- source/libs/sync/inc/syncVoteMgr.h | 4 - source/libs/sync/src/syncMain.c | 16 +- source/libs/sync/src/syncSnapshot.c | 136 +---------------- source/libs/sync/src/syncTimeout.c | 27 ++-- source/libs/sync/src/syncUtil.c | 32 +--- source/libs/sync/src/syncVoteMgr.c | 140 ++++-------------- .../sync/test/sync_test_lib/inc/syncTest.h | 12 ++ .../sync_test_lib/src/syncSnapshotDebug.c | 132 +++++++++++++++++ .../test/sync_test_lib/src/syncUtilDebug.c | 36 +++++ .../test/sync_test_lib/src/syncVoteMgrDebug.c | 102 +++++++++++++ 14 files changed, 349 insertions(+), 312 deletions(-) create mode 100644 source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c create mode 100644 source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index d688a3c1bd..963fedce31 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "syncInt.h" -#include "syncMessage.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 @@ -31,7 +30,6 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 -//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; @@ -60,12 +58,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finis int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); - -int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); - -//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; @@ -78,7 +70,6 @@ typedef struct SSyncSnapshotReceiver { // init when create SSyncNode *pSyncNode; - } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); @@ -88,13 +79,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); - -//--------------------------------------------------- // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode *ths, SyncSnapshotRsp *pMsg); +int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index e1a6050d92..3139707d54 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "syncInt.h" -#include "syncMessage.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 6a760ecd87..3fb4a5ba0c 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -728,7 +728,6 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); -int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7f241e827d..076101ef43 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -24,17 +24,15 @@ extern "C" { uint64_t syncUtilAddr2U64(const char* host, uint16_t port); void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilEmptyId(const SRaftId* pId); int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); -char* syncUtilRaftId2Str(const SRaftId* p); const char* syncStr(ESyncState state); char* syncUtilPrintBin(char* ptr, uint32_t len); char* syncUtilPrintBin2(char* ptr, uint32_t len); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index d894e91600..066a4dd76f 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -39,8 +39,6 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncN bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted); -char *voteGranted2Str(SVotesGranted *pVotesGranted); typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; @@ -56,8 +54,6 @@ void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSync bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond); -char *votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 252d5d0219..25f6c5e6f1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -757,7 +757,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); goto _error; } @@ -772,7 +772,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { sError("vgId:%d, failed to determine raft member id, peer:%d", pSyncNode->vgId, i); goto _error; } @@ -781,7 +781,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; } @@ -1213,7 +1213,7 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); + syncUtilRaftId2EpSet(destRaftId, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1230,7 +1230,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + syncUtilNodeInfo2EpSet(nodeInfo, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1344,7 +1344,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; @@ -1356,13 +1356,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // update quorum first diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 78413bbeff..3273718f7d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -13,33 +13,25 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncSnapshot.h" #include "syncIndexMgr.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "wal.h" -//---------------------------------- -static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg); -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); -static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); - -//---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); SSyncSnapshotSender *pSender = NULL; if (condition) { - pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pSender, 0, sizeof(*pSender)); pSender->start = false; pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; @@ -249,64 +241,6 @@ static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnaps ++(pSender->seq); } -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pSender != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pSender->start); - cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); - cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); - cJSON_AddStringToObject(pRoot, "pReader", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); - cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); - - if (pSender->pCurrentBlock != NULL) { - char *s; - s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); - taosMemoryFree(s); - } - - cJSON *pSnapshot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); - cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); - cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); - // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); - return pJson; -} - -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { - cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { sNTrace(pSyncNode, "starting snapshot ..."); @@ -335,16 +269,17 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } -// ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); SSyncSnapshotReceiver *pReceiver = NULL; if (condition) { - pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); - ASSERT(pReceiver != NULL); - memset(pReceiver, 0, sizeof(*pReceiver)); + pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); + if (pReceiver == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pReceiver->start = false; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; @@ -530,63 +465,6 @@ static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapsh } } -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pReceiver != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); - cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); - cJSON_AddStringToObject(pRoot, "pWriter", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - cJSON *pFromId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); - cJSON_AddStringToObject(pFromId, "addr", u64buf); - { - uint64_t u64 = pReceiver->fromId.addr; - cJSON *pTmp = pFromId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); - cJSON_AddItemToObject(pRoot, "fromId", pFromId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); - return pJson; -} - -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { - cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { SyncIndex snapStart = SYNC_INDEX_INVALID; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 30e0617f43..91d807319b 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -13,28 +13,23 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncTimeout.h" #include "syncElection.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncReplication.h" -#include "syncRespMgr.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { int32_t newArrIndex = 0; - SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT]; - memset(newConfigIndexArr, 0, sizeof(newConfigIndexArr)); - + SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0}; SSnapshot snapshot = {0}; - if (ths->pFsm != NULL && ths->pFsm->FpGetSnapshotInfo != NULL) { - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - } - + + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) { - for (int i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { + for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) { // pass - ; } else { // save newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i]; @@ -47,13 +42,15 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); int32_t code = raftCfgPersist(ths->pRaftCfg); - ASSERT(code == 0); - - sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + if (code != 0) { + sNFatal(ths, "failed to persist cfg"); + } else { + sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + } } } -int32_t syncNodeTimerRoutine(SSyncNode* ths) { +static int32_t syncNodeTimerRoutine(SSyncNode* ths) { sNTrace(ths, "timer routines"); // timer replicate @@ -71,7 +68,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); + sNError(ths, "timer wal snapshot end error since:%s", terrstr()); return -1; } else { sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1a00b0f5a4..894aa05aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -41,15 +41,15 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort); } -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { char host[TSDB_FQDN_LEN] = {0}; - uint16_t port; + uint16_t port = 0; syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); pEpSet->inUse = 0; @@ -57,7 +57,7 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { addEpIntoEpSet(pEpSet, host, port); } -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { sError("failed to resolve ipv4 addr, fqdn: %s", pInfo->nodeFqdn); @@ -73,8 +73,7 @@ bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - bool ret = pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; - return ret; + return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; } bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } @@ -90,18 +89,6 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); - cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); - return pJson; -} - cJSON* syncUtilRaftId2Json(const SRaftId* p) { char u64buf[128] = {0}; cJSON* pRoot = cJSON_CreateObject(); @@ -120,13 +107,6 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { return pJson; } -char* syncUtilRaftId2Str(const SRaftId* p) { - cJSON* pJson = syncUtilRaftId2Json(p); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - static inline bool syncUtilCanPrint(char c) { if (c >= 32 && c <= 126) { return true; @@ -165,14 +145,12 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) { } void syncUtilMsgHtoN(void* msg) { - // htonl SMsgHead* pHead = msg; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); } void syncUtilMsgNtoH(void* msg) { - // ntohl SMsgHead* pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index ee1f83ee6a..b36e5c0288 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,12 +23,11 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryCalloc(1, sizeof(SVotesGranted)); if (pVotesGranted == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); pVotesGranted->replicaNum = pSyncNode->replicaNum; @@ -59,20 +58,24 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode) { pVotesGranted->pSyncNode = pSyncNode; } -bool voteGrantedMajority(SVotesGranted *pVotesGranted) { - bool ret = pVotesGranted->votes >= pVotesGranted->quorum; - return ret; -} +bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->votes >= pVotesGranted->quorum; } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { - ASSERT(pMsg->voteGranted == true); - - if (pMsg->term != pVotesGranted->term) { - sNTrace(pVotesGranted->pSyncNode, "vote grant vnode error"); + if (!pMsg->voteGranted) { + sNFatal(pVotesGranted->pSyncNode, " vote granted should be true"); return; } - ASSERT(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + if (pMsg->term != pVotesGranted->term) { + sNTrace(pVotesGranted->pSyncNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, + pVotesGranted->term, pMsg->term); + return; + } + + if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { + sNFatal(pVotesGranted->pSyncNode, " vote granted raftId not matched with msg"); + return; + } int32_t j = -1; for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -81,14 +84,21 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - ASSERT(j != -1); - ASSERT(j >= 0 && j < pVotesGranted->replicaNum); + if ((j == -1) || (j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, " invalid msg srcId, index:%d", j); + return; + } if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); pVotesGranted->isGranted[j] = true; } - ASSERT(pVotesGranted->votes <= pVotesGranted->replicaNum); + + if (pVotesGranted->votes <= pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, " votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + pVotesGranted->replicaNum); + return; + } } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { @@ -97,53 +107,12 @@ void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->toLeader = false; } -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesGranted != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); - } - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - arr[i] = pVotesGranted->isGranted[i]; - } - cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); - - cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); - cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - bool majority = voteGrantedMajority(pVotesGranted); - cJSON_AddNumberToObject(pRoot, "majority", majority); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); - return pJson; -} - -char *voteGranted2Str(SVotesGranted *pVotesGranted) { - cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); - ASSERT(pVotesRespond != NULL); - memset(pVotesRespond, 0, sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryCalloc(1, sizeof(SVotesRespond)); + if (pVotesRespond == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pVotesRespond->replicas = &(pSyncNode->replicasId); pVotesRespond->replicaNum = pSyncNode->replicaNum; @@ -185,62 +154,15 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { - // ASSERT(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } } - ASSERT(0); + + sNFatal(pVotesRespond->pSyncNode, "votes respond not found"); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); - /* - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; - } - */ -} - -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesRespond != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); - } - int32_t respondNum = 0; - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - arr[i] = pVotesRespond->isRespond[i]; - if (pVotesRespond->isRespond[i]) { - respondNum++; - } - } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); - cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); - return pJson; -} - -char *votesRespond2Str(SVotesRespond *pVotesRespond) { - cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; } diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index 3a49074272..241a4a5e02 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -82,6 +82,18 @@ void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); +cJSON* voteGranted2Json(SVotesGranted* pVotesGranted); +char* voteGranted2Str(SVotesGranted* pVotesGranted); +cJSON* votesRespond2Json(SVotesRespond* pVotesRespond); +char* votesRespond2Str(SVotesRespond* pVotesRespond); + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +char* syncUtilRaftId2Str(const SRaftId* p); + +cJSON* snapshotSender2Json(SSyncSnapshotSender* pSender); +char* snapshotSender2Str(SSyncSnapshotSender* pSender); +cJSON* snapshotReceiver2Json(SSyncSnapshotReceiver* pReceiver); +char* snapshotReceiver2Str(SSyncSnapshotReceiver* pReceiver); #ifdef __cplusplus } diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c new file mode 100644 index 0000000000..fdaf4b8b34 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pFromId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); + cJSON_AddStringToObject(pFromId, "addr", u64buf); + { + uint64_t u64 = pReceiver->fromId.addr; + cJSON *pTmp = pFromId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); + cJSON_AddItemToObject(pRoot, "fromId", pFromId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c new file mode 100644 index 0000000000..b12f4e76a5 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c new file mode 100644 index 0000000000..f1c26a97aa --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesGranted != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesRespond != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); + } + int32_t respondNum = 0; + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} From fba45da22555f28a5ae9b703e6021c0a5a708b32 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 10 Nov 2022 12:43:23 +0800 Subject: [PATCH 100/101] refact: remove assert and adjust log --- source/libs/sync/src/syncVoteMgr.c | 12 ++++++------ source/libs/sync/test/syncAppendEntriesBatchTest.cpp | 7 ------- source/libs/sync/test/syncAppendEntriesReplyTest.cpp | 6 +----- source/libs/sync/test/syncAppendEntriesTest.cpp | 6 +----- source/libs/sync/test/syncApplyMsgTest.cpp | 6 +----- .../libs/sync/test/syncConfigChangeSnapshotTest.cpp | 8 +------- source/libs/sync/test/syncConfigChangeTest.cpp | 8 +------- source/libs/sync/test/syncElectTest.cpp | 7 +------ source/libs/sync/test/syncEncodeTest.cpp | 10 ---------- source/libs/sync/test/syncEnqTest.cpp | 7 +------ source/libs/sync/test/syncEntryCacheTest.cpp | 9 --------- source/libs/sync/test/syncEntryTest.cpp | 8 -------- source/libs/sync/test/syncEnvTest.cpp | 7 +------ source/libs/sync/test/syncHashCacheTest.cpp | 8 -------- source/libs/sync/test/syncHeartbeatReplyTest.cpp | 6 +----- source/libs/sync/test/syncHeartbeatTest.cpp | 6 +----- source/libs/sync/test/syncIOClientTest.cpp | 8 +------- source/libs/sync/test/syncIOSendMsgTest.cpp | 10 ++-------- source/libs/sync/test/syncIOServerTest.cpp | 6 +----- source/libs/sync/test/syncIOTickPingTest.cpp | 5 +---- source/libs/sync/test/syncIOTickQTest.cpp | 5 +---- source/libs/sync/test/syncIndexMgrTest.cpp | 10 +--------- source/libs/sync/test/syncIndexTest.cpp | 6 +----- source/libs/sync/test/syncInitTest.cpp | 8 +------- source/libs/sync/test/syncLeaderTransferTest.cpp | 6 +----- source/libs/sync/test/syncLocalCmdTest.cpp | 6 +----- source/libs/sync/test/syncLogStoreCheck.cpp | 8 -------- source/libs/sync/test/syncLogStoreCheck2.cpp | 8 -------- source/libs/sync/test/syncLogStoreTest.cpp | 8 -------- source/libs/sync/test/syncPingReplyTest.cpp | 6 +----- source/libs/sync/test/syncPingSelfTest.cpp | 7 +------ source/libs/sync/test/syncPingTest.cpp | 6 +----- source/libs/sync/test/syncPingTimerTest.cpp | 7 +------ source/libs/sync/test/syncPingTimerTest2.cpp | 7 +------ source/libs/sync/test/syncPreSnapshotReplyTest.cpp | 6 +----- source/libs/sync/test/syncPreSnapshotTest.cpp | 6 +----- source/libs/sync/test/syncRaftCfgIndexTest.cpp | 7 +------ source/libs/sync/test/syncRaftCfgTest.cpp | 8 +------- source/libs/sync/test/syncRaftIdCheck.cpp | 5 +---- source/libs/sync/test/syncRaftLogTest.cpp | 9 +-------- source/libs/sync/test/syncRaftLogTest2.cpp | 8 -------- source/libs/sync/test/syncRaftStoreTest.cpp | 6 +----- source/libs/sync/test/syncRefTest.cpp | 6 +----- source/libs/sync/test/syncReplicateTest.cpp | 7 +------ source/libs/sync/test/syncRequestVoteReplyTest.cpp | 6 +----- source/libs/sync/test/syncRequestVoteTest.cpp | 6 +----- source/libs/sync/test/syncRestoreFromSnapshot.cpp | 9 +-------- source/libs/sync/test/syncRpcMsgTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotReceiverTest.cpp | 8 +------- source/libs/sync/test/syncSnapshotRspTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotSendTest.cpp | 6 +----- source/libs/sync/test/syncSnapshotSenderTest.cpp | 8 +------- source/libs/sync/test/syncSnapshotTest.cpp | 11 +---------- source/libs/sync/test/syncTest.cpp | 4 +--- source/libs/sync/test/syncTestTool.cpp | 9 +-------- source/libs/sync/test/syncTimeoutTest.cpp | 6 +----- source/libs/sync/test/syncUtilTest.cpp | 7 +------ source/libs/sync/test/syncVotesGrantedTest.cpp | 8 -------- source/libs/sync/test/syncVotesRespondTest.cpp | 8 -------- source/libs/sync/test/syncWriteTest.cpp | 10 ---------- source/libs/sync/test/sync_test_lib/inc/syncTest.h | 4 ++++ 61 files changed, 58 insertions(+), 380 deletions(-) diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index b36e5c0288..4ca4e26bec 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -62,7 +62,7 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->v void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { if (!pMsg->voteGranted) { - sNFatal(pVotesGranted->pSyncNode, " vote granted should be true"); + sNFatal(pVotesGranted->pSyncNode, "vote granted should be true"); return; } @@ -73,7 +73,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { } if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { - sNFatal(pVotesGranted->pSyncNode, " vote granted raftId not matched with msg"); + sNFatal(pVotesGranted->pSyncNode, "vote granted raftId not matched with msg"); return; } @@ -84,8 +84,8 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - if ((j == -1) || (j >= 0 && j < pVotesGranted->replicaNum)) { - sNFatal(pVotesGranted->pSyncNode, " invalid msg srcId, index:%d", j); + if ((j == -1) || !(j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, "invalid msg srcId, index:%d", j); return; } @@ -94,8 +94,8 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { pVotesGranted->isGranted[j] = true; } - if (pVotesGranted->votes <= pVotesGranted->replicaNum) { - sNFatal(pVotesGranted->pSyncNode, " votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + if (pVotesGranted->votes > pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, pVotesGranted->replicaNum); return; } diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index 98b8654734..947a2cb05c 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -1,11 +1,4 @@ //#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncUtil.h" -#include "trpc.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a5e96233c2..1eb803e846 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 256c13e267..7b9e04a814 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp index ce129015c6..a8a1931d1d 100644 --- a/source/libs/sync/test/syncApplyMsgTest.cpp +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index dcf7249d30..a6d33aa674 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c5548d25d4..cf498933e1 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 5cdbb2cc88..59d731c823 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 216291c1d8..528cc1614d 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" #if 0 void logTest() { diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index bb5f968dc6..54bfb1b387 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 56b3d6da2e..b86422b0b1 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -1,12 +1,3 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" -#include "tskiplist.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e94755195b..369306b857 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index a220c2aed4..3cd6048428 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -1,9 +1,4 @@ -#include "syncEnv.h" -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "ttime.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 2f5bb07f9a..14a29c9a1e 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -1,12 +1,4 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "tskiplist.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp index 1fac03652b..6f7f4d18f3 100644 --- a/source/libs/sync/test/syncHeartbeatReplyTest.cpp +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp index b0c0554355..c8c38c0c38 100644 --- a/source/libs/sync/test/syncHeartbeatTest.cpp +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index bd0221114a..f66a854657 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" -#include "tdatablock.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index f88e4f240d..a082f9373a 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -101,7 +95,7 @@ int main(int argc, char** argv) { syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); SEpSet epSet; - syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + syncUtilNodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); rpcMsg.info.noResp = 1; pSyncNode->syncSendMSg(&epSet, &rpcMsg); diff --git a/source/libs/sync/test/syncIOServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp index 1d7402e461..add7d6df60 100644 --- a/source/libs/sync/test/syncIOServerTest.cpp +++ b/source/libs/sync/test/syncIOServerTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 9c2342828e..71274f5ca3 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 64b65f25c8..9ef8a7ad40 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 23b8693a0b..f84119045f 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -1,12 +1,4 @@ -#include "syncIndexMgr.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp index 763117c0c9..a44d9c4e31 100644 --- a/source/libs/sync/test/syncIndexTest.cpp +++ b/source/libs/sync/test/syncIndexTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void print(SHashObj *pNextIndex) { printf("----------------\n"); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 2fe5dd7e18..b15a767db5 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLeaderTransferTest.cpp b/source/libs/sync/test/syncLeaderTransferTest.cpp index bd1f22edff..fd8cdfaac0 100644 --- a/source/libs/sync/test/syncLeaderTransferTest.cpp +++ b/source/libs/sync/test/syncLeaderTransferTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index b42626df29..fa0d91d45f 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 0161160a75..e02c484729 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 29ad0610e7..fce0a21a47 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -1,12 +1,4 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 832b42bf80..1b898803ef 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index e5158a7932..661509e9a2 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 781ffc31db..19f377e037 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 1777b9b9e2..54cc506b03 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 5a0b5e9953..ded7e22eac 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 09f56815cd..4f1bb64798 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp index a30dcc2c54..e5175f1101 100644 --- a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp @@ -1,10 +1,6 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotTest.cpp b/source/libs/sync/test/syncPreSnapshotTest.cpp index 03894dfa84..2ce95abea3 100644 --- a/source/libs/sync/test/syncPreSnapshotTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgIndexTest.cpp b/source/libs/sync/test/syncRaftCfgIndexTest.cpp index 522da159b5..e6d3f23b58 100644 --- a/source/libs/sync/test/syncRaftCfgIndexTest.cpp +++ b/source/libs/sync/test/syncRaftCfgIndexTest.cpp @@ -1,10 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index b1f820c78e..c841a68fde 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -1,10 +1,4 @@ -#include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index e7ef69da20..08716c876b 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void usage(char* exe) { printf("Usage: %s host port \n", exe); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp index 278113919a..e309a2e432 100644 --- a/source/libs/sync/test/syncRaftLogTest.cpp +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -1,12 +1,5 @@ #include "syncRaftLog.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 3d50b63ff9..a7752dcb8b 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 070f8c0f07..87798a7d80 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,9 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index c5132018a2..47bc290abe 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "tref.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 7506ae19bd..2419ec0974 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 3d9db17725..973ccf040c 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 94e1504add..275f7804bf 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp index 470dd678b0..44444a7f5e 100644 --- a/source/libs/sync/test/syncRestoreFromSnapshot.cpp +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 127d8e1c41..3f66cd9d1c 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 6eb8548740..49b06a7d1b 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 63905c2182..0a06b512ea 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 83f1dfebb3..46f33e0c56 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 7b2c13463c..9a234d412e 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index e2264fd08c..40eac0b55f 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -1,14 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 97de81572a..55480802da 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" /* typedef enum { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index e6f6006410..b0fe612b7f 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -1,12 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index bae5e0ea88..ed0742f499 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 411915fe22..bbd4fa7d18 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -1,9 +1,4 @@ -#include "syncUtil.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 6a8404308b..2d2a12c656 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index d5e7d71030..712e97edc9 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -1,14 +1,6 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" - void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2e5c26719b..56dd0a5845 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index 241a4a5e02..a1881bf1b3 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -22,8 +22,10 @@ extern "C" { #include "syncInt.h" +#include "tref.h" #include "wal.h" +#include "tref.h" #include "syncEnv.h" #include "syncIO.h" #include "syncIndexMgr.h" @@ -38,6 +40,8 @@ extern "C" { #include "syncUtil.h" #include "syncVoteMgr.h" +extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); + cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntryPrint(const SSyncRaftEntry* pObj); From 6f7d43c3715f1da83425c6fe86e5752e03ad5eaf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 10 Nov 2022 13:33:55 +0800 Subject: [PATCH 101/101] fix: case when type error --- source/libs/parser/src/parTranslater.c | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f7dce1637a..05fc651618 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1252,6 +1252,19 @@ static bool dataTypeEqual(const SDataType* l, const SDataType* r) { return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); @@ -1876,10 +1889,15 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW } pWhenThen->pWhen = pIsTrue; } - if (first) { - first = false; + if (first || dataTypeComp(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType) < 0) { pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; - } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { + } + first = false; + } + + FOREACH(pNode, pCaseWhen->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pWhenThen->pThen, pCaseWhen->node.resType, &pCastFunc); if (TSDB_CODE_SUCCESS != pCxt->errCode) { @@ -1889,6 +1907,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->node.resType = pCaseWhen->node.resType; } } + if (NULL != pCaseWhen->pElse && !dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pCaseWhen->pElse)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pCaseWhen->pElse, pCaseWhen->node.resType, &pCastFunc); @@ -3431,19 +3450,6 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return (SNode*)pCol; } -// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail -static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { - if (l->type != r->type) { - return 1; - } - - if (l->bytes != r->bytes) { - return l->bytes > r->bytes ? 1 : -1; - } - - return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; -} - static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); @@ -4969,7 +4975,8 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); + int32_t code = + getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); }