From 5097329bf707305611741c4eb55f2e9b1c3174e3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 13 Apr 2022 10:38:02 +0800 Subject: [PATCH 01/20] enh(test): optimize the multi-threaded exit mode of tsim --- tests/script/tmp/back.sim | 6 ++++++ tests/script/tmp/data.sim | 14 ++++++++++++++ tests/tsim/src/simSystem.c | 26 +++++++++++++------------- 3 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 tests/script/tmp/back.sim create mode 100644 tests/script/tmp/data.sim diff --git a/tests/script/tmp/back.sim b/tests/script/tmp/back.sim new file mode 100644 index 0000000000..9202e90594 --- /dev/null +++ b/tests/script/tmp/back.sim @@ -0,0 +1,6 @@ +sql connect +$x = 1 +begin: + sql insert into db.tb values(now, $x ) -x begin + $x = $x + 1 +goto begin diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim new file mode 100644 index 0000000000..faac5b2828 --- /dev/null +++ b/tests/script/tmp/data.sim @@ -0,0 +1,14 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database db +sql create table db.tb (ts timestamp, i int) +sql insert into db.tb values(now, 1) + +print ======== start back +run_back tmp/back.sim + +sleep 2000 +return -1 diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index eb5fb68264..5bbcceada5 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -80,24 +80,24 @@ SScript *simProcessCallOver(SScript *script) { simExecSuccess = false; simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", script->fileName, script->error); - return NULL; } else { simExecSuccess = true; simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, script->fileName); - simCloseTaosdConnect(script); - simScriptSucced++; - simScriptPos--; - - simFreeScript(script); - if (simScriptPos == -1) { - simInfo("----------------------------------------------------------------------"); - simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); - return NULL; - } - - return simScriptList[simScriptPos]; } + + simCloseTaosdConnect(script); + simScriptSucced++; + simScriptPos--; + simFreeScript(script); + + if (simScriptPos == -1 && simExecSuccess) { + simInfo("----------------------------------------------------------------------"); + simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); + return NULL; + } + + return simScriptList[simScriptPos]; } else { simDebug("script:%s, is stopped", script->fileName); simFreeScript(script); From 0aab0cd7f1bf227b4e91ab95ed6b33e17b6a6b08 Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 13 Apr 2022 11:10:38 +0800 Subject: [PATCH 02/20] test: fix the where sim case --- tests/script/tsim/query/complex_where.sim | 80 +++++++++++------------ 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/tests/script/tsim/query/complex_where.sim b/tests/script/tsim/query/complex_where.sim index 7a5de85a3c..312c1d98ed 100644 --- a/tests/script/tsim/query/complex_where.sim +++ b/tests/script/tsim/query/complex_where.sim @@ -192,14 +192,14 @@ if $data01 != 1 then return -1 endi -sql select c1 from stb1 where stb1 > 5 and c1 <= 6 +sql select c1 from stb1 where c1 > 5 and c1 <= 6 print ====> sql : select c1 from stb1 where c1 > 5 and c1 <= 6 print ====> rows: $rows print ====> rows0: $data00 -if $rows != 4 then +if $rows != 3 then return -1 endi -if $data01 != 6 then +if $data00 != 6 then return -1 endi @@ -210,7 +210,7 @@ print ====> rows0: $data00 if $rows != 32 then return -1 endi -if $data01 != 1 then +if $data00 != 1 then return -1 endi @@ -221,7 +221,7 @@ print ====> rows0: $data00 if $rows != 17 then return -1 endi -if $data01 != 5 then +if $data00 != 5 then return -1 endi @@ -240,7 +240,7 @@ if $rows != 12 then return -1 endi -sql_error select c1 from stb1 where c7 between false and true +sql select c1 from stb1 where c7 between false and true sql select c1 from stb1 where c1 in (1,2,3) print ====> sql : select c1 from stb1 where c1 in (1,2,3) @@ -272,98 +272,98 @@ endi print ================ query 2 complex with where -sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select count(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then @@ -510,14 +510,14 @@ if $data01 != 1 then return -1 endi -sql select c1 from stb1 where stb1 > 5 and c1 <= 6 +sql select c1 from stb1 where c1 > 5 and c1 <= 6 print ====> sql : select c1 from stb1 where c1 > 5 and c1 <= 6 print ====> rows: $rows print ====> rows0: $data00 -if $rows != 4 then +if $rows != 3 then return -1 endi -if $data01 != 6 then +if $data00 != 6 then return -1 endi @@ -528,7 +528,7 @@ print ====> rows0: $data00 if $rows != 32 then return -1 endi -if $data01 != 1 then +if $data00 != 1 then return -1 endi @@ -539,7 +539,7 @@ print ====> rows0: $data00 if $rows != 17 then return -1 endi -if $data01 != 5 then +if $data00 != 5 then return -1 endi @@ -558,7 +558,7 @@ if $rows != 12 then return -1 endi -sql_error select c1 from stb1 where c7 between false and true +sql select c1 from stb1 where c7 between false and true sql select c1 from stb1 where c1 in (1,2,3) print ====> sql : select c1 from stb1 where c1 in (1,2,3) @@ -590,98 +590,98 @@ endi print ================ query 2 complex with where -sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select count(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 +sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then From c0b3607756564ff4e8b67f3e9e9c3c2d76717c58 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 12 Apr 2022 20:02:33 +0800 Subject: [PATCH 03/20] feat(query): add to_unixtimestamp function TD-14243 --- include/common/ttime.h | 1 + include/libs/function/functionMgt.h | 2 +- include/libs/scalar/scalar.h | 1 + source/common/src/ttime.c | 26 +++++++++++++++++++++++- source/libs/function/inc/builtins.h | 2 +- source/libs/function/src/builtins.c | 13 ++++++++++++ source/libs/scalar/inc/sclInt.h | 5 +++-- source/libs/scalar/src/sclfunc.c | 31 +++++++++++++++++++++++++++++ 8 files changed, 76 insertions(+), 5 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 306f54bedb..15450c31ca 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -64,6 +64,7 @@ char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); +int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal); void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 9d1b2c9ea0..148a9f77d6 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -85,8 +85,8 @@ typedef enum EFunctionType { // conversion function FUNCTION_TYPE_CAST = 2000, FUNCTION_TYPE_TO_ISO8601, + FUNCTION_TYPE_TO_UNIXTIMESTAMP, FUNCTION_TYPE_TO_JSON, - FUNCTION_TYPE_UNIXTIMESTAMP, // date and time function FUNCTION_TYPE_NOW = 2500, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index cd9c6f5bf2..226909a187 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -75,6 +75,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp /* Time related functions */ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index a65352f2b9..1baf393e9a 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -406,7 +406,31 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char default: { return -1; } - } + } +} + +int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { + int32_t charLen = varDataLen(inputData); + char *newColData; + if (type == TSDB_DATA_TYPE_BINARY) { + newColData = taosMemoryCalloc(1, charLen + 1); + memcpy(newColData, varDataVal(inputData), charLen); + taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + taosMemoryFree(newColData); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + newColData = taosMemoryCalloc(1, charLen / TSDB_NCHAR_SIZE + 1); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inputData), charLen, newColData); + if (len < 0){ + taosMemoryFree(newColData); + return TSDB_CODE_FAILED; + } + newColData[len] = 0; + taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + taosMemoryFree(newColData); + } else { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index ab5a02c438..624a2953c9 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -22,7 +22,7 @@ extern "C" { #include "functionMgt.h" -#define FUNCTION_NAME_MAX_LENGTH 16 +#define FUNCTION_NAME_MAX_LENGTH 32 #define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 55f38e9005..44888fa793 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -403,6 +403,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = toISO8601Function, .finalizeFunc = NULL }, + { + .name = "to_unixtimestamp", + .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = checkAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toUnixtimestampFunction, + .finalizeFunc = NULL + }, { .name = "_rowts", .type = FUNCTION_TYPE_ROWTS, @@ -622,6 +632,9 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_TO_ISO8601: { pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; } + case FUNCTION_TYPE_TO_UNIXTIMESTAMP: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + } case FUNCTION_TYPE_TBNAME: { // todo diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 58b62bb05e..3d59ffd93d 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -46,8 +46,9 @@ typedef struct SScalarCtx { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); -#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) -#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) +#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) +#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) +#define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam *param); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0ab0052624..e2d28fe454 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,6 +1,7 @@ #include "function.h" #include "scalar.h" #include "tdatablock.h" +#include "ttime.h" #include "sclInt.h" #include "sclvector.h" @@ -872,6 +873,36 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * return TSDB_CODE_SUCCESS; } +int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + int32_t timePrec = GET_PARAM_PRECISON(pInput); + if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (inputNum != 1) { + return TSDB_CODE_FAILED; + } + + char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + int64_t timeVal = 0; + convertStringToTimestamp(type, input, timePrec, &timeVal); + + colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + input += varDataTLen(input); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } From ebb5f20169ab2cc0081ef65f87880900d3180c8a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 13:58:02 +0800 Subject: [PATCH 04/20] test: del test cases --- tests/script/tsim/tmq/consumerMain.sim | 267 ------------------------- 1 file changed, 267 deletions(-) delete mode 100644 tests/script/tsim/tmq/consumerMain.sim diff --git a/tests/script/tsim/tmq/consumerMain.sim b/tests/script/tsim/tmq/consumerMain.sim deleted file mode 100644 index 51b90971fd..0000000000 --- a/tests/script/tsim/tmq/consumerMain.sim +++ /dev/null @@ -1,267 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 2 then - return -1 - endi - if $data02 != 1 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 3 then - return -1 - endi - if $data00 == d1 then - if $data02 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data12 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 2 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 3 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertDataV1.sim -else - run_back tsim/tmq/insertDataV4.sim -endi - -sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$consumeDelay = 5000 -$expectConsumeMsgCnt = 1000 -print expectConsumeMsgCnt: $expectConsumeMsgCnt, consumeDelay: $consumeDelay - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != $expect_result then -# return -1 -#endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content >= $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -#system sh/exec.sh -n dnode1 -s stop -x SIGINT From eac9d2d5fe4fb3412eda7995efe9d17e75b0d0b3 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 14:07:00 +0800 Subject: [PATCH 05/20] test: package scripte --- packaging/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/install.sh b/packaging/install.sh index 3aae074af5..49386ba05a 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -199,6 +199,8 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/* + ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so From 61b39204b5a22523d4aaeff0af10923b8048bc5d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:38:57 +0800 Subject: [PATCH 06/20] feat(query): add timetruncate function TD-14243 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 16 +++ source/libs/scalar/src/sclfunc.c | 201 ++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 226909a187..caea52a5ef 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp /* Time related functions */ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 44888fa793..36de0d1149 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -413,6 +413,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = toUnixtimestampFunction, .finalizeFunc = NULL }, + { + .name = "timetruncate", + .type = FUNCTION_TYPE_TIMETRUNCATE, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = checkAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timeTruncateFunction, + .finalizeFunc = NULL + }, { .name = "_rowts", .type = FUNCTION_TYPE_ROWTS, @@ -631,9 +641,15 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { } case FUNCTION_TYPE_TO_ISO8601: { pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; + break; } case FUNCTION_TYPE_TO_UNIXTIMESTAMP: { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + break; + } + case FUNCTION_TYPE_TIMETRUNCATE: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + break; } case FUNCTION_TYPE_TBNAME: { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e2d28fe454..b53dc955de 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -903,6 +903,207 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } +int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(&pInput[0]); + int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); + if (inputNum != 2) { + return TSDB_CODE_FAILED; + } + + if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && + type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (GET_PARAM_TYPE(&pInput[1]) != TSDB_DATA_TYPE_BIGINT) { //time_unit + return TSDB_CODE_FAILED; + } + + int64_t timeUnit, timeVal = 0; + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + + int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : + (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + + char *input = NULL; + if (IS_VAR_DATA_TYPE(type)) { + input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + } else { + input = pInput[0].columnData->pData; + } + + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ + convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); + //If converted value is less than 10digits in second, use value in second instead + int64_t timeValSec = timeVal / 1000000000; + if (timeValSec < 1000000000) { + timeVal = timeValSec; + } + } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */ + GET_TYPED_DATA(timeVal, int64_t, type, input); + } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/ + GET_TYPED_DATA(timeVal, int64_t, type, input); + int64_t timeValSec = timeVal / factor; + if (timeValSec < 1000000000) { + timeVal = timeValSec; + } + } + + char buf[20] = {0}; + NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal, sizeof(buf), buf); + int32_t tsDigits = (int32_t)strlen(buf); + timeUnit = timeUnit * 1000 / factor; + + switch (timeUnit) { + case 0: { /* 1u */ + if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000 * 1000; + //} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + // //timeVal = timeVal / 1000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor; + } else { + timeVal = timeVal * 1; + } + break; + } + case 1: { /* 1a */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal * 1; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000 * 1000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS){ + timeVal = timeVal * factor; + } else { + assert(0); + } + break; + } + case 1000: { /* 1s */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal / 1000 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000000 * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000000 * 1000000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor; + } else { + assert(0); + } + break; + } + case 60000: { /* 1m */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal / 1000 / 60 * 60 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000000 / 60 * 60 * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000000 / 60 * 60 * 1000000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor / factor / 60 * 60 * factor; + } else { + assert(0); + } + break; + } + case 3600000: { /* 1h */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal / 1000 / 3600 * 3600 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000000 / 3600 * 3600 * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000000 / 3600 * 3600 * 1000000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor / factor / 3600 * 3600 * factor; + } else { + assert(0); + } + break; + } + case 86400000: { /* 1d */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal / 1000 / 86400 * 86400 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor / factor / 86400* 86400 * factor; + } else { + assert(0); + } + break; + } + case 604800000: { /* 1w */ + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal / 1000 / 604800 * 604800 * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000; + } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal = timeVal * factor / factor / 604800 * 604800* factor; + } else { + assert(0); + } + break; + } + default: { + timeVal = timeVal * 1; + break; + } + } + + //truncate the timestamp to db precision + switch (timePrec) { + case TSDB_TIME_PRECISION_MILLI: { + if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal / 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000000; + } + break; + } + case TSDB_TIME_PRECISION_MICRO: { + if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal = timeVal / 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal * 1000; + } + break; + } + case TSDB_TIME_PRECISION_NANO: { + if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal = timeVal * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal = timeVal * 1000000; + } + break; + } + } + + colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + if (IS_VAR_DATA_TYPE(type)) { + input += varDataTLen(input); + } else { + input += tDataTypes[type].bytes; + } + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } From b2108e64221ed8409f1d0ff0ff5a9335f402299d Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 13 Apr 2022 14:19:14 +0800 Subject: [PATCH 07/20] test : fix group sim case --- tests/script/tsim/query/complex_group.sim | 150 +++++++++++----------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/tests/script/tsim/query/complex_group.sim b/tests/script/tsim/query/complex_group.sim index 3a1ec523fe..d6ce26b69a 100644 --- a/tests/script/tsim/query/complex_group.sim +++ b/tests/script/tsim/query/complex_group.sim @@ -97,167 +97,167 @@ print ================ query 1 group by filter sql select count(*) from ct3 group by c1 print ====> sql : select count(*) from ct3 group by c1 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c2 print ====> sql : select count(*) from ct3 group by c2 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c3 print ====> sql : select count(*) from ct3 group by c3 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c4 print ====> sql : select count(*) from ct3 group by c4 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c5 print ====> sql : select count(*) from ct3 group by c5 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c6 print ====> sql : select count(*) from ct3 group by c6 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c7 print ====> sql : select count(*) from ct3 group by c7 print ====> rows: $rows -if $rows != 2 then +if $rows != 3 then return -1 endi sql select count(*) from ct3 group by c8 print ====> sql : select count(*) from ct3 group by c8 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c9 print ====> sql : select count(*) from ct3 group by c9 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c10 print ====> sql : select count(*) from ct3 group by c10 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi print ================ query 2 complex with group by sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 @@ -307,170 +307,170 @@ print ================ query 1 group by filter sql select count(*) from ct3 group by c1 print ====> sql : select count(*) from ct3 group by c1 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c2 print ====> sql : select count(*) from ct3 group by c2 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c3 print ====> sql : select count(*) from ct3 group by c3 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c4 print ====> sql : select count(*) from ct3 group by c4 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c5 print ====> sql : select count(*) from ct3 group by c5 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c6 print ====> sql : select count(*) from ct3 group by c6 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c7 print ====> sql : select count(*) from ct3 group by c7 print ====> rows: $rows -if $rows != 2 then +if $rows != 3 then return -1 endi sql select count(*) from ct3 group by c8 print ====> sql : select count(*) from ct3 group by c8 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c9 print ====> sql : select count(*) from ct3 group by c9 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi sql select count(*) from ct3 group by c10 print ====> sql : select count(*) from ct3 group by c10 print ====> rows: $rows -if $rows != 8 then +if $rows != 9 then return -1 endi print ================ query 2 complex with group by sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1 -print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2 +sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1 +print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2 print ====> rows: $rows if $rows != 1 then return -1 endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From c4f1adc1bba96d1e288162e89ca561814f1529c9 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 14:31:42 +0800 Subject: [PATCH 08/20] test:package script --- packaging/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/install.sh b/packaging/install.sh index 49386ba05a..bfd5fcd3a9 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -157,7 +157,7 @@ function install_main_path() { ${csudo} mkdir -p ${install_main_dir}/cfg ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector - ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/lib ${csudo} mkdir -p ${install_main_dir}/examples ${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d From 0076463ddac8ac80fa33246d1b4b37e97061a921 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 15:05:23 +0800 Subject: [PATCH 09/20] test:package script --- packaging/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/install.sh b/packaging/install.sh index bfd5fcd3a9..c9748d223b 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -198,6 +198,8 @@ function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib_link_dir}/libtdb.* || : + ${csudo} rm -f ${lib64_link_dir}/libtdb.* || : ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/* From 03d7201d47047c7f969df503edcebe3f343b5e74 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 13 Apr 2022 16:00:30 +0800 Subject: [PATCH 10/20] feat: support tmq msg parse --- example/src/tmq.c | 18 +- include/client/taos.h | 93 +++++----- include/common/tdatablock.h | 12 +- include/common/tmsg.h | 49 +++++- source/client/inc/clientInt.h | 4 +- source/client/src/clientMain.c | 2 +- source/client/src/tmq.c | 180 +++++++++++++------ source/dnode/vnode/src/tq/tq.c | 201 +++++++++++++++++++++- source/libs/executor/src/dataDispatcher.c | 2 +- tests/test/c/tmqDemo.c | 12 +- tests/test/c/tmqSim.c | 8 +- 11 files changed, 452 insertions(+), 129 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index ca80c8fe5a..85dea5b382 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -19,8 +19,8 @@ #include #include "taos.h" -static int running = 1; -static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } +static int running = 1; +/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ int32_t init_env() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -166,11 +166,11 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { int32_t cnt = 0; /*clock_t startTime = clock();*/ while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { cnt++; - printf("get data\n"); - msg_process(tmqmessage); + /*printf("get data\n");*/ + /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); /*} else {*/ /*break;*/ @@ -198,9 +198,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1000); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { - msg_process(tmqmessage); + /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); @@ -226,10 +226,10 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) { int32_t skipLogNum = 0; clock_t startTime = clock(); while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { batchCnt++; - skipLogNum += tmqGetSkipLogNum(tmqmessage); + /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); } else { diff --git a/include/client/taos.h b/include/client/taos.h index d3856d432e..87948e7824 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -30,7 +30,7 @@ typedef void **TAOS_ROW; #if 0 typedef void TAOS_STREAM; #endif -typedef void TAOS_SUB; +typedef void TAOS_SUB; // Data type definition #define TSDB_DATA_TYPE_NULL 0 // 1 bytes @@ -138,13 +138,13 @@ typedef enum { #define RET_MSG_LENGTH 1024 typedef struct setConfRet { SET_CONF_RET_CODE retCode; - char retMsg[RET_MSG_LENGTH]; + char retMsg[RET_MSG_LENGTH]; } setConfRet; -DLL_EXPORT void taos_cleanup(void); -DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); -DLL_EXPORT setConfRet taos_set_config(const char *config); -DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT void taos_cleanup(void); +DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); +DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); @@ -152,34 +152,34 @@ DLL_EXPORT void taos_close(TAOS *taos); const char *taos_data_type(int type); -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); @@ -188,14 +188,14 @@ DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData); +DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT void taos_reset_current_db(TAOS *taos); -DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); -DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); +DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); @@ -237,9 +237,9 @@ typedef struct tmq_t tmq_t; typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; -typedef struct tmq_conf_t tmq_conf_t; -typedef struct tmq_list_t tmq_list_t; -typedef struct tmq_message_t tmq_message_t; +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; +// typedef struct tmq_message_t tmq_message_t; typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); @@ -259,7 +259,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); -DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); +DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); #if 0 DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); @@ -268,8 +268,8 @@ DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** v DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); -#endif DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); +#endif /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { @@ -285,21 +285,24 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); +#if 0 // temporary used function for demo only void tmqShowMsg(tmq_message_t *tmq_message); int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); +#endif /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ +DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +#if 0 DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); -DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); -DLL_EXPORT int32_t tmq_get_vgroup_id(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic); DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic); -DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); - +#endif +DLL_EXPORT void tmq_message_destroy(TAOS_RES *res); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ #if 0 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); @@ -308,7 +311,7 @@ DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char * DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql); /* ------------------------------ TMQ END -------------------------------- */ -#if 1 // Shuduo: temporary enable for app build +#if 1 // Shuduo: temporary enable for app build typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); #endif diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 5e5a8826e5..6f933c663e 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -199,11 +199,11 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); -void blockDataCleanup(SSDataBlock* pDataBlock); +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); +void blockDataCleanup(SSDataBlock* pDataBlock); -size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); -void* blockDataDestroy(SSDataBlock* pBlock); +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); +void* blockDataDestroy(SSDataBlock* pBlock); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); @@ -211,8 +211,8 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); void blockDebugShowData(const SArray* dataBlocks); -static FORCE_INLINE int32_t blockEstimateEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(pBlock) + (int32_t)ceil(blockDataGetSerialRowSize(pBlock) * pBlock->info.rows); +static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { + return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); } static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ba9147dcdd..cb38c694ce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -484,7 +484,7 @@ typedef struct { char intervalUnit; char slidingUnit; char - offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. int8_t precision; int64_t interval; int64_t sliding; @@ -2379,6 +2379,53 @@ typedef struct { SArray* pBlockData; // SArray } SMqPollRsp; +typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t dataLen; + SArray* blockPos; // beginning pos for each SRetrieveTableRsp + void* blockData; // serialized batched SRetrieveTableRsp +} SMqPollRspV2; + +static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->dataLen); + if (pRsp->dataLen != 0) { + int32_t sz = taosArrayGetSize(pRsp->blockPos); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i); + tlen += taosEncodeFixedI32(buf, blockPos); + } + tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->dataLen); + if (pRsp->dataLen != 0) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t)); + for (int32_t i = 0; i < sz; i++) { + int32_t blockPos; + buf = taosDecodeFixedI32(buf, &blockPos); + taosArrayPush(pRsp->blockPos, &blockPos); + } + buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen); + } + return (void*)buf; +} + typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 185b5824d9..b7d88c277b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -194,12 +194,12 @@ enum { #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) -typedef struct SMqRspObj { +typedef struct { int8_t resType; char* topic; - void* vg; SArray* res; // SArray int32_t resIter; + int32_t vgId; } SMqRspObj; typedef struct SRequestObj { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 040ddde630..c93260be37 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -465,7 +465,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { return 0; } - doFetchRow(pRequest, false, false); + doFetchRow(pRequest, false, true); SReqResultInfo *pResultInfo = &pRequest->body.resInfo; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index dbe78782f5..ea31170b1f 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -27,11 +27,22 @@ struct tmq_message_t { SMqPollRsp msg; char* topic; - void* vg; SArray* res; // SArray + int32_t vgId; int32_t resIter; }; +typedef struct { + int8_t tmqRspType; + int32_t epoch; +} SMqRspWrapper; + +typedef struct { + int8_t tmqRspType; + int32_t epoch; + SMqCMGetSubEpRsp msg; +} SMqAskEpRspWrapper; + struct tmq_list_t { SArray container; }; @@ -118,6 +129,14 @@ typedef struct { TAOS_FIELD* fields; } SMqClientTopic; +typedef struct { + int8_t tmqRspType; + int32_t epoch; + SMqClientVg* vgHandle; + SMqClientTopic* topicHandle; + SMqPollRspV2 msg; +} SMqPollRspWrapper; + typedef struct { tmq_t* tmq; tsem_t rspSem; @@ -133,10 +152,10 @@ typedef struct { typedef struct { tmq_t* tmq; SMqClientVg* pVg; + SMqClientTopic* pTopic; int32_t epoch; int32_t vgId; tsem_t rspSem; - tmq_message_t** msg; int32_t sync; } SMqPollCbParam; @@ -244,7 +263,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { } void tmqClearUnhandleMsg(tmq_t* tmq) { - tmq_message_t* msg = NULL; + SMqRspWrapper* msg = NULL; while (1) { taosGetQitem(tmq->qall, (void**)&msg); if (msg) @@ -777,7 +796,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } - +#if 0 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; SMqPollRsp* pRsp = &tmq_message->msg; @@ -827,11 +846,13 @@ void tmqShowMsg(tmq_message_t* tmq_message) { } } } +#endif int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { /*printf("recv poll\n");*/ SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; + SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = pParam->tmq; if (code != 0) { tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); @@ -874,18 +895,22 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { #endif /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ - tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); - if (pRsp == NULL) { + /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/ + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); + if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); goto CREATE_MSG_FAIL; } - memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg); - /*pRsp->iter.curBlock = 0;*/ - /*pRsp->iter.curRow = 0;*/ + pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP; + pRspWrapper->vgHandle = pVg; + pRspWrapper->topicHandle = pTopic; + /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/ + memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); // TODO: alloc mem /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ + #if 0 if (pRsp->msg.numOfTopics == 0) { /*printf("no data\n");*/ @@ -894,11 +919,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, - pRsp->msg.reqOffset, pRsp->msg.rspOffset); + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, + pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); - pRsp->vg = pParam->pVg; - taosWriteQitem(tmq->mqueue, pRsp); + taosWriteQitem(tmq->mqueue, pRspWrapper); atomic_add_fetch_32(&tmq->readyRequest, 1); /*tsem_post(&tmq->rspSem);*/ return 0; @@ -1015,16 +1039,19 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } tDeleteSMqCMGetSubEpRsp(&rsp); } else { - SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp)); - if (pRsp == NULL) { + /*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/ + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); + if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; goto END; } - memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); + pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; + pWrapper->epoch = head->epoch; + memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); - taosWriteQitem(tmq->mqueue, pRsp); + taosWriteQitem(tmq->mqueue, pWrapper); /*tsem_post(&tmq->rspSem);*/ } @@ -1152,6 +1179,25 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo return pReq; } +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + pRspObj->resType = RES_TYPE__TMQ; + pRspObj->topic = strdup(pWrapper->topicHandle->topicName); + pRspObj->resIter = -1; + pRspObj->vgId = pWrapper->vgHandle->vgId; + SMqPollRspV2* pRsp = &pWrapper->msg; + int32_t blockNum = taosArrayGetSize(pRsp->blockPos); + pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo)); + for (int32_t i = 0; i < blockNum; i++) { + int32_t pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i); + SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos); + SReqResultInfo resInfo; + setQueryResultFromRsp(&resInfo, pRetrieve, true); + taosArrayPush(pRspObj->res, &resInfo); + } + return pRspObj; +} + #if 0 tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { tmq_message_t* msg = NULL; @@ -1258,6 +1304,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { } pParam->tmq = tmq; pParam->pVg = pVg; + pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; pParam->sync = 0; @@ -1296,13 +1343,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { return 0; } -// return -int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) { - if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) { +int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) { + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ - if (rspHead->epoch > atomic_load_32(&tmq->epoch)) { - SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead; - tmqUpdateEp(tmq, rspHead->epoch, rspMsg); + if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { + SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; + SMqCMGetSubEpRsp* rspMsg = &pEpRspWrapper->msg; + tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1314,41 +1361,43 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) { return 0; } -tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { +SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { while (1) { - SMqRspHead* rspHead = NULL; - taosGetQitem(tmq->qall, (void**)&rspHead); - if (rspHead == NULL) { + SMqRspWrapper* rspWrapper = NULL; + taosGetQitem(tmq->qall, (void**)&rspWrapper); + if (rspWrapper == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); - taosGetQitem(tmq->qall, (void**)&rspHead); - if (rspHead == NULL) return NULL; + taosGetQitem(tmq->qall, (void**)&rspWrapper); + if (rspWrapper == NULL) return NULL; } - if (rspHead->mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { - tmq_message_t* rspMsg = (tmq_message_t*)rspHead; + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; atomic_sub_fetch_32(&tmq->readyRequest, 1); /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ - if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) { + if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ - SMqClientVg* pVg = rspMsg->vg; + SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = rspMsg->msg.rspOffset; + pVg->currentOffset = pollRspWrapper->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (rspMsg->msg.numOfTopics == 0) { - taosFreeQitem(rspMsg); - rspHead = NULL; + if (pollRspWrapper->msg.dataLen == 0) { + taosFreeQitem(pollRspWrapper); + rspWrapper = NULL; continue; } - return rspMsg; + // build msg + SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + return pRsp; } else { /*printf("epoch mismatch\n");*/ - taosFreeQitem(rspMsg); + taosFreeQitem(pollRspWrapper); } } else { /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ bool reset = false; - tmqHandleNoPollRsp(tmq, rspHead, &reset); - taosFreeQitem(rspHead); + tmqHandleNoPollRsp(tmq, rspWrapper, &reset); + taosFreeQitem(rspWrapper); if (pollIfReset && reset) { tscDebug("consumer %ld reset and repoll", tmq->consumerId); tmqPollImpl(tmq, blockingTime); @@ -1382,17 +1431,17 @@ tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { } #endif -tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* rspMsg; - int64_t startTime = taosGetTimestampMs(); +TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + SMqRspObj* rspObj; + int64_t startTime = taosGetTimestampMs(); // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); - rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); - if (rspMsg) { - return rspMsg; + rspObj = tmqHandleAllRsp(tmq, blocking_time, false); + if (rspObj) { + return (TAOS_RES*)rspObj; } while (1) { @@ -1402,9 +1451,9 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*tsem_wait(&tmq->rspSem);*/ - rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); - if (rspMsg) { - return rspMsg; + rspObj = tmqHandleAllRsp(tmq, blocking_time, false); + if (rspObj) { + return (TAOS_RES*)rspObj; } if (blocking_time != 0) { int64_t endTime = taosGetTimestampMs(); @@ -1546,6 +1595,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v } #endif +#if 0 void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; SMqPollRsp* pRsp = &tmq_message->msg; @@ -1553,6 +1603,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { /*taosMemoryFree(tmq_message);*/ taosFreeQitem(tmq_message); } +#endif tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } @@ -1563,4 +1614,27 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } -char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; } +char* tmq_get_topic_name(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + return pRspObj->topic; + } else { + return NULL; + } +} + +int32_t tmq_get_vgroup_id(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + return pRspObj->vgId; + } else { + return -1; + } +} + +void tmq_message_destroy(TAOS_RES* res) { + if (res == NULL) return; + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + } +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0d4acd4594..654d7c3a65 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -255,7 +255,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } - +#if 0 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; @@ -433,6 +433,205 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } +#endif + +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t fetchOffset; + int64_t blockingTime = pReq->blockingTime; + int32_t reqEpoch = pReq->epoch; + + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + + vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + pTq->pVnode->vgId, pReq->currentOffset, fetchOffset); + + SMqPollRspV2 rspV2 = {0}; + rspV2.dataLen = 0; + + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + if (pConsumer == NULL) { + vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, pTq->pVnode->vgId); + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + tmsgSendRsp(pMsg); + return 0; + } + + int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch); + while (consumerEpoch < reqEpoch) { + consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch); + } + + STqTopic* pTopic = NULL; + int32_t topicSz = taosArrayGetSize(pConsumer->topics); + for (int32_t i = 0; i < topicSz; i++) { + STqTopic* topic = taosArrayGet(pConsumer->topics, i); + // TODO race condition + ASSERT(pConsumer->consumerId == consumerId); + if (strcmp(topic->topicName, pReq->topic) == 0) { + pTopic = topic; + break; + } + } + if (pTopic == NULL) { + vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, + pTq->pVnode->vgId); + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + tmsgSendRsp(pMsg); + return 0; + } + + vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, + pTq->pVnode->vgId); + + rspV2.reqOffset = pReq->currentOffset; + rspV2.skipLogNum = 0; + + while (1) { + /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ + // TODO + consumerEpoch = atomic_load_32(&pConsumer->epoch); + if (consumerEpoch > reqEpoch) { + vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", + consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch); + break; + } + SWalReadHead* pHead; + if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { + // TODO: no more log, set timer to wait blocking time + // if data inserted during waiting, launch query and + // response to user + vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + pTq->pVnode->vgId, fetchOffset); + break; + } + vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + pTq->pVnode->vgId, fetchOffset, pHead->msgType); + /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ + /*pHead = pTopic->pReadhandle->pHead;*/ + if (pHead->msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + qTaskInfo_t task = pTopic->buffer.output[workerId].task; + ASSERT(task); + qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(false); + } + if (pDataBlock == NULL) { + /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ + break; + } + + taosArrayPush(pRes, pDataBlock); + } + + if (taosArrayGetSize(pRes) == 0) { + vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, + pReq->epoch, pTq->pVnode->vgId, fetchOffset); + fetchOffset++; + rspV2.skipLogNum++; + taosArrayDestroy(pRes); + continue; + } + rspV2.rspOffset = fetchOffset; + + int32_t blockSz = taosArrayGetSize(pRes); + int32_t tlen = 0; + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pBlock = taosArrayGet(pRes, i); + tlen += sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + } + + void* data = taosMemoryMalloc(tlen); + if (data == NULL) { + pMsg->code = -1; + taosMemoryFree(pHead); + } + + rspV2.blockData = data; + + void* dataBlockBuf = data; + int32_t pos; + for (int32_t i = 0; i < blockSz; i++) { + pos = 0; + SSDataBlock* pBlock = taosArrayGet(pRes, i); + blockCompressEncode(pBlock, dataBlockBuf, &pos, pBlock->info.numOfCols, false); + taosArrayPush(rspV2.blockPos, &rspV2.dataLen); + rspV2.dataLen += pos; + dataBlockBuf = POINTER_SHIFT(dataBlockBuf, pos); + } + + int32_t msgLen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2); + void* buf = rpcMallocCont(msgLen); + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; + + void* msgBodyBuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqPollRspV2(&msgBodyBuf, &rspV2); + + /*rsp.pBlockData = pRes;*/ + + /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", pTq->pVnode->vgId, fetchOffset, + pHead->msgType, consumerId, pReq->epoch); + tmsgSendRsp(pMsg); + taosMemoryFree(pHead); + return 0; + } else { + taosMemoryFree(pHead); + fetchOffset++; + rspV2.skipLogNum++; + } + } + + /*if (blockingTime != 0) {*/ + /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ + /*} else {*/ + + rspV2.rspOffset = fetchOffset - 1; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqPollRspV2(&abuf, &rspV2); + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + tmsgSendRsp(pMsg); + vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", pTq->pVnode->vgId, fetchOffset, consumerId, + pReq->epoch); + /*}*/ + + return 0; +} int32_t tqProcessRebReq(STQ* pTq, char* msg) { SMqMVRebReq req = {0}; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 626cb1b5f0..e897bc8892 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -92,7 +92,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, return false; } - pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockEstimateEncodeSize(pInput->pData); + pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pInput->pData); pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 45a9d31f4c..756893f217 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -195,7 +195,7 @@ void parseArgument(int32_t argc, char *argv[]) { } static int running = 1; -static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } +/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ // calc dir size (not include itself 4096Byte) int64_t getDirectorySize(char *dir) @@ -363,9 +363,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { - msg_process(tmqmessage); + /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); @@ -392,12 +392,12 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 3000); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); if (tmqmessage) { batchCnt++; - skipLogNum += tmqGetSkipLogNum(tmqmessage); + /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ if (0 != g_stConfInfo.showMsgFlag) { - msg_process(tmqmessage); + /*msg_process(tmqmessage);*/ } tmq_message_destroy(tmqmessage); } else { diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 236d1e2eed..273852b498 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -170,7 +170,7 @@ void parseInputString() { static int running = 1; -static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } +/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ int queryDB(TAOS *taos, char *command) { @@ -226,7 +226,7 @@ void loop_consume(tmq_t* tmq) { int32_t totalRows = 0; int32_t skipLogNum = 0; while (running) { - tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 8000); + TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, 8000); if (tmqMsg) { totalMsgs++; @@ -237,9 +237,9 @@ void loop_consume(tmq_t* tmq) { } #endif - skipLogNum += tmqGetSkipLogNum(tmqMsg); + /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/ if (0 != g_stConfInfo.showMsgFlag) { - msg_process(tmqMsg); + /*msg_process(tmqMsg);*/ } tmq_message_destroy(tmqMsg); } else { From f9d162bae2474137ed1bdf02a9159b75712efed2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:02:55 +0800 Subject: [PATCH 11/20] tsma use TDB --- source/dnode/vnode/CMakeLists.txt | 3 +- source/dnode/vnode/inc/vnode.h | 6 +- source/dnode/vnode/src/inc/tsdbSma.h | 68 ++++ source/dnode/vnode/src/inc/vnodeInt.h | 2 + source/dnode/vnode/src/meta/metaBDBImpl.c | 19 +- source/dnode/vnode/src/meta/metaTDBImpl.c | 356 +++++++++++++++++- source/dnode/vnode/src/tq/tq.c | 6 +- source/dnode/vnode/src/tsdb/tsdbSma.c | 218 +++++++++-- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 137 +++++++ source/dnode/vnode/src/vnd/vnodeWrite.c | 8 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 11 +- source/libs/tdb/src/db/tdbBtree.c | 25 +- .../script/tsim/sma/tsmaCreateInsertData.sim | 49 +++ 13 files changed, 829 insertions(+), 79 deletions(-) create mode 100644 source/dnode/vnode/src/inc/tsdbSma.h create mode 100644 source/dnode/vnode/src/tsdb/tsdbTDBImpl.c create mode 100644 tests/script/tsim/sma/tsmaCreateInsertData.sim diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bd2e4213d5..4b8aa5dfaf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources( # tsdb # "src/tsdb/tsdbBDBImpl.c" + "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbFile.c" @@ -40,7 +41,7 @@ target_sources( "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbScan.c" - # "src/tsdb/tsdbSma.c" + "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" # tq diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 113501a26b..aab835b958 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -357,7 +357,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid); +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode); STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid); SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup); int metaGetTbNum(SMeta *pMeta); @@ -369,8 +369,8 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid); -void metaCloseSmaCurosr(SMSmaCursor *pSmaCur); -const char *metaSmaCursorNext(SMSmaCursor *pSmaCur); +void metaCloseSmaCursor(SMSmaCursor *pSmaCur); +int64_t metaSmaCursorNext(SMSmaCursor *pSmaCur); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h new file mode 100644 index 0000000000..ce03fa7c67 --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_TSDB_SMA_H_ +#define _TD_VNODE_TSDB_SMA_H_ + +#include "tdbInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSmaKey SSmaKey; + +struct SSmaKey { + TSKEY skey; + int64_t groupId; +}; + + +typedef struct SDBFile SDBFile; + +struct SDBFile { + int32_t fid; + TDB *pDB; + char *path; +}; + +int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path); +int32_t tsdbCloseDBEnv(TENV *pEnv); +int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t tsdbCloseDBF(SDBFile *pDBF); +int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); +void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); + +void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); +#if 0 +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); +int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); +#endif + +// internal func +static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { + int32_t len = 0; + len += taosEncodeFixedI64(pData, tsKey); + len += taosEncodeFixedI64(pData, groupId); + return len; +} + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6e1f00f931..29a45723c4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -220,6 +220,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); #include "tq.h" +#include "tsdbSma.h" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index caa101b5d0..a8270f746d 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -667,7 +667,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { return pTbCfg; } -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { STSma * pCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; @@ -920,7 +920,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { return pCur; } -void metaCloseSmaCurosr(SMSmaCursor *pCur) { +void metaCloseSmaCursor(SMSmaCursor *pCur) { if (pCur) { if (pCur->pCur) { pCur->pCur->close(pCur->pCur); @@ -930,7 +930,8 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { } } -const char *metaSmaCursorNext(SMSmaCursor *pCur) { +int64_t metaSmaCursorNext(SMSmaCursor *pCur) { +#if 0 DBT skey = {0}; DBT pkey = {0}; DBT pval = {0}; @@ -946,6 +947,8 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { } else { return NULL; } +#endif + return 0; } STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { @@ -972,7 +975,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - metaCloseSmaCurosr(pCur); + metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; @@ -980,7 +983,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pval.data; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - metaCloseSmaCurosr(pCur); + metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; @@ -990,8 +993,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { break; } - metaCloseSmaCurosr(pCur); - + metaCloseSmaCursor(pCur); + return pSW; } @@ -1004,7 +1007,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { int ret; // TODO: lock? - ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); + ret = pDB->pSmaIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); if (ret != 0) { return NULL; } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index e3acba7eb6..6f218ad72b 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -22,6 +22,8 @@ typedef struct SPoolMem { struct SPoolMem *next; } SPoolMem; +#define META_TDB_SMA_TEST + static SPoolMem *openPool(); static void clearPool(SPoolMem *pPool); static void closePool(SPoolMem *pPool); @@ -38,6 +40,10 @@ struct SMetaDB { TDB * pNtbIdx; TDB * pCtbIdx; SPoolMem *pPool; +#ifdef META_TDB_SMA_TEST + TDB *pSmaDB; + TDB *pSmaIdx; +#endif }; typedef struct __attribute__((__packed__)) { @@ -55,6 +61,11 @@ typedef struct { tb_uid_t uid; } SCtbIdxKey; +typedef struct { + tb_uid_t uid; + int64_t smaUid; +} SSmaIdxKey; + static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); @@ -115,6 +126,17 @@ static inline int metaCtbIdxCmpr(const void *arg1, int len1, const void *arg2, i return metaUidCmpr(&pKey1->uid, sizeof(tb_uid_t), &pKey2->uid, sizeof(tb_uid_t)); } +static inline int metaSmaIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SSmaIdxKey *pKey1 = (SSmaIdxKey *)arg1; + SSmaIdxKey *pKey2 = (SSmaIdxKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + return metaUidCmpr(&pKey1->smaUid, sizeof(int64_t), &pKey2->smaUid, sizeof(int64_t)); +} + int metaOpenDB(SMeta *pMeta) { SMetaDB *pMetaDb; int ret; @@ -143,6 +165,15 @@ int metaOpenDB(SMeta *pMeta) { return -1; } +#ifdef META_TDB_SMA_TEST + ret = tdbDbOpen("sma.db", sizeof(int64_t), TDB_VARIANT_LEN, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } +#endif + // open schema DB ret = tdbDbOpen("schema.db", sizeof(SSchemaDbKey), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); @@ -180,6 +211,15 @@ int metaOpenDB(SMeta *pMeta) { return -1; } +#ifdef META_TDB_SMA_TEST + ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, metaSmaIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } +#endif + pMetaDb->pPool = openPool(); tdbTxnOpen(&pMetaDb->txn, 0, poolMalloc, poolFree, pMetaDb->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); tdbBegin(pMetaDb->pEnv, NULL); @@ -193,10 +233,16 @@ void metaCloseDB(SMeta *pMeta) { tdbCommit(pMeta->pDB->pEnv, &pMeta->pDB->txn); tdbTxnClose(&pMeta->pDB->txn); clearPool(pMeta->pDB->pPool); +#ifdef META_TDB_SMA_TEST + tdbDbClose(pMeta->pDB->pSmaIdx); +#endif tdbDbClose(pMeta->pDB->pCtbIdx); tdbDbClose(pMeta->pDB->pNtbIdx); tdbDbClose(pMeta->pDB->pStbIdx); tdbDbClose(pMeta->pDB->pNameIdx); +#ifdef META_TDB_SMA_TEST + tdbDbClose(pMeta->pDB->pSmaDB); +#endif tdbDbClose(pMeta->pDB->pSchemaDB); tdbDbClose(pMeta->pDB->pTbDB); taosMemoryFree(pMeta->pDB); @@ -491,7 +537,6 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { taosMemoryFree(tbCfg.name); taosMemoryFree(tbCfg.stbCfg.pTagSchema); continue; - ; } else if (tbCfg.type == META_CHILD_TABLE) { kvRowFree(tbCfg.ctbCfg.pTag); } @@ -566,51 +611,326 @@ int metaGetTbNum(SMeta *pMeta) { return 0; } +struct SMSmaCursor { + TDBC *pCur; + tb_uid_t uid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + STSmaWrapper *pSW = NULL; + + pSW = taosMemoryCalloc(1, sizeof(*pSW)); + if (pSW == NULL) { + return NULL; + } + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (pCur == NULL) { + taosMemoryFree(pSW); + return NULL; + } + + void *pBuf = NULL; + SSmaIdxKey *pSmaIdxKey = NULL; + + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + pSmaIdxKey = pCur->pKey; + ASSERT(pSmaIdxKey != NULL); + + void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false); + + if (pSmaVal == NULL) { + tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid); + continue; + } + + + ++pSW->number; + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); + if (tptr == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + pSW->tSma = tptr; + pBuf = pSmaVal; + if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + TDB_FREE(pSmaVal); + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pSW; + +#endif } int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { // TODO ASSERT(0); +#ifndef META_TDB_SMA_TEST + DBT key = {0}; + + key.data = (void *)indexName; + key.size = strlen(indexName); + + metaDBWLock(pMeta->pDB); + // TODO: No guarantee of consistence. + // Use transaction or DB->sync() for some guarantee. + pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0); + metaDBULock(pMeta->pDB); +#endif return 0; } int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // TODO - ASSERT(0); + // ASSERT(0); + +#ifdef META_TDB_SMA_TEST + int32_t ret = 0; + SMetaDB *pMetaDb = pMeta->pDB; + void *pBuf = NULL, *qBuf = NULL; + void *key = {0}, *val = {0}; + + // save sma info + int32_t len = tEncodeTSma(NULL, pSmaCfg); + pBuf = taosMemoryCalloc(1, len); + if (pBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + key = (void *)&pSmaCfg->indexUid; + qBuf = pBuf; + tEncodeTSma(&qBuf, pSmaCfg); + val = pBuf; + + int32_t kLen = sizeof(pSmaCfg->indexUid); + int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); + + ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + if (ret < 0) { + taosMemoryFreeClear(pBuf); + return -1; + } + + // add sma idx + SSmaIdxKey smaIdxKey; + smaIdxKey.uid = pSmaCfg->tableUid; + smaIdxKey.smaUid = pSmaCfg->indexUid; + key = &smaIdxKey; + kLen = sizeof(smaIdxKey); + val = NULL; + vLen = 0; + + ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + if (ret < 0) { + taosMemoryFreeClear(pBuf); + return -1; + } + + // release + taosMemoryFreeClear(pBuf); + + if (pMeta->pDB->pPool->size > 0) { + metaCommit(pMeta); + } + +#endif return 0; } -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + SMetaDB *pDB = pMeta->pDB; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + int ret = -1; + + // Set key + pKey = (void *)&indexUid; + kLen = sizeof(indexUid); + + // Query + ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen); + if (ret != 0 || !pVal) { + return NULL; + } + + if (!isDecode) { + // return raw value + return pVal; + } + + // Decode + STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); + if (pCfg == NULL) { + taosMemoryFree(pVal); + return NULL; + } + + void *pBuf = pVal; + if (tDecodeTSma(pBuf, pCfg) == NULL) { + tdDestroyTSma(pCfg); + taosMemoryFree(pCfg); + TDB_FREE(pVal); + return NULL; + } + + TDB_FREE(pVal); + return pCfg; +#endif } -const char *metaSmaCursorNext(SMSmaCursor *pCur) { +/** + * @brief + * + * @param pMeta + * @param uid 0 means iterate all uids. + * @return SMSmaCursor* + */ +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + SMSmaCursor *pCur = NULL; + SMetaDB *pDB = pMeta->pDB; + int ret; + + pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); + if (pCur == NULL) { + return NULL; + } + + pCur->uid = uid; + ret = tdbDbcOpen(pDB->pSmaIdx, &(pCur->pCur)); + if ((ret != 0) || (pCur->pCur == NULL)) { + taosMemoryFree(pCur); + return NULL; + } + + if (uid != 0) { + // TODO: move to the specific uid + } + + return pCur; +#endif } -void metaCloseSmaCurosr(SMSmaCursor *pCur) { +/** + * @brief + * + * @param pCur + * @return int64_t smaIndexUid + */ +int64_t metaSmaCursorNext(SMSmaCursor *pCur) { // TODO - ASSERT(0); + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + int ret; + void *pBuf; + SSmaIdxKey *smaIdxKey; + + ret = tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); + if (ret < 0) { + return 0; + } + smaIdxKey = pCur->pKey; + return smaIdxKey->smaUid; +#endif +} + +void metaCloseSmaCursor(SMSmaCursor *pCur) { + // TODO + // ASSERT(0); +#ifdef META_TDB_SMA_TEST + if (pCur) { + if (pCur->pCur) { + tdbDbcClose(pCur->pCur); + } + + taosMemoryFree(pCur); + } +#endif } SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { // TODO // ASSERT(0); // comment this line to pass CI - return NULL; -} + // return NULL: +#ifdef META_TDB_SMA_TEST + SArray *pUids = NULL; + SMetaDB *pDB = pMeta->pDB; + void *pKey; -SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + // TODO: lock? + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); + if (pCur == NULL) { + return NULL; + } + // TODO: lock? + + SSmaIdxKey *pSmaIdxKey = NULL; + tb_uid_t uid = 0; + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + ASSERT(pSmaIdxKey != NULL); + pSmaIdxKey = pCur->pKey; + + if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) { + continue; + } + uid = pSmaIdxKey->uid; + + if (!pUids) { + pUids = taosArrayInit(16, sizeof(tb_uid_t)); + if (!pUids) { + metaCloseSmaCursor(pCur); + return NULL; + } + } + + taosArrayPush(pUids, &uid); + + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pUids; +#endif } static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0d4acd4594..794427475e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,9 +82,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi memcpy(data, msg, msgLen); if (msgType == TDMT_VND_SUBMIT) { - // if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { - // return -1; - // } + if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { + return -1; + } } SRpcMsg req = { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 07b7d62165..bfdad836f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -38,6 +38,29 @@ typedef enum { SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +struct SSmaEnv { + TdThreadRwlock lock; + TXN txn; + SPoolMem *pPool; + SDiskID did; + TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; +}; + +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_DID(env) ((env)->did) +#define SMA_ENV_ENV(env) ((env)->dbEnv) +#define SMA_ENV_PATH(env) ((env)->path) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) + typedef struct { STsdb *pTsdb; SDBFile dFile; @@ -104,7 +127,8 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); @@ -117,9 +141,121 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); +// Pool Memory +static SPoolMem *openPool(); +static void clearPool(SPoolMem *pPool); +static void closePool(SPoolMem *pPool); +static void *poolMalloc(void *arg, size_t size); +static void poolFree(void *arg, void *ptr); + +static int tsdbSmaBeginCommit(SSmaEnv *pEnv); +static int tsdbSmaEndCommit(SSmaEnv *pEnv); + // implementation -static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); } -static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); } +static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { + return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); +} +static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { + return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); +} + +static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockRdlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockWrlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockUnlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + if (!pPool) return; + + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + tdbOsFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + if (pPool) { + clearPool(pPool); + tdbOsFree(pPool); + } +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + tdbOsFree(pMem); +} int32_t tsdbInitSma(STsdb *pTsdb) { // tSma @@ -213,7 +349,12 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) char aname[TSDB_FILENAME_LEN] = {0}; tfsAbsoluteName(pTsdb->pTfs, did, path, aname); - if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + tsdbFreeSmaEnv(pEnv); + return NULL; + } + + if ((pEnv->pPool = openPool()) == NULL) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -248,7 +389,8 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { taosMemoryFreeClear(pSmaEnv->pStat); taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); - tsdbCloseBDBEnv(pSmaEnv->dbEnv); + tsdbCloseDBEnv(pSmaEnv->dbEnv); + closePool(pSmaEnv->pPool); } } @@ -414,7 +556,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t } // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); + STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid, true); if (pSma == NULL) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); @@ -498,10 +640,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) { return TSDB_CODE_FAILED; } -#ifndef TSDB_SMA_TEST - TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; -#endif - // Firstly, assume that tSma can only be created on super table/normal table. // getActiveTimeWindow @@ -563,6 +701,10 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) { TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey); + + // TODO: release only when suid changes. + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); } } @@ -676,10 +818,12 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { * @param dataLen * @return int32_t */ -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; + // TODO: insert sma data blocks into B+Tree(TDB) - if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { + if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); return TSDB_CODE_FAILED; @@ -826,6 +970,30 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe return daysPerFile; } +static int tsdbSmaBeginCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + // start a new txn + tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { + tsdbWarn("tsdbSma tdb restart txn fail"); + return -1; + } + return 0; +} + +static int tsdbSmaEndCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + + // Commit current txn + if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { + tsdbWarn("tsdbSma tdb commit fail"); + return -1; + } + tdbTxnClose(pTxn); + clearPool(pEnv->pPool); + return 0; +} + /** * @brief Insert/Update Time-range-wise SMA data. * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. @@ -911,14 +1079,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char int64_t groupId = pDataBlock->info.groupId; for (int32_t j = 0; j < rows; ++j) { printf("|"); - TSKEY skey = 1649295200000; // TSKEY_INITIAL_VAL; // the start key of TS window by interval + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval void *pSmaKey = &smaKey; bool isStartKey = false; - { - // just for debugging - isStartKey = true; - tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); - } + int32_t tlen = 0; // reset the len pDataBuf = &dataBuf; // reset the buf for (int32_t k = 0; k < colNum; ++k) { @@ -929,7 +1093,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char if (!isStartKey) { isStartKey = true; skey = *(TSKEY *)var; - printf("==> skey = %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); } else { printf(" %" PRIi64 " |", *(int64_t *)var); @@ -1010,6 +1174,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char // TODO: tsdbStartTSmaCommit(); if (fid != tSmaH.dFile.fid) { if (tSmaH.dFile.fid != TSDB_IVLD_FID) { + tsdbSmaEndCommit(pEnv); tsdbCloseDBF(&tSmaH.dFile); } tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); @@ -1020,12 +1185,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } + tsdbSmaBeginCommit(pEnv); } - if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen) != 0) { + if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); + tsdbSmaEndCommit(pEnv); tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; @@ -1044,9 +1211,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char printf("\n"); } } - + tsdbSmaEndCommit(pEnv); // TODO: not commit for every insert tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_SUCCESS; } @@ -1370,8 +1538,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - void *result = NULL; - uint32_t valueSize = 0; + void *result = NULL; + int32_t valueSize = 0; if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); @@ -1422,7 +1590,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { return -1; } tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, - vCreateSmaReq.tSma.indexUid); + vCreateSmaReq.tSma.indexUid); // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; @@ -1464,7 +1632,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { return -1; } - tsdbTSmaSub(pTsdb, 1); + tsdbTSmaSub(pTsdb, 1); // TODO: return directly or go on follow steps? return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c new file mode 100644 index 0000000000..519ecd3fa0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "vnodeInt.h" + +int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { + int ret = 0; + + if (path == NULL) return -1; + + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + + if (ret != 0) { + tsdbError("Failed to create tsdb db env, ret = %d", ret); + return -1; + } + + return 0; +} + +int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } + +static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + const SSmaKey *pKey1 = (const SSmaKey *)arg1; + const SSmaKey *pKey2 = (const SSmaKey *)arg2; + + ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); + + if (pKey1->skey < pKey2->skey) { + return -1; + } else if (pKey1->skey > pKey2->skey) { + return 1; + } + if (pKey1->groupId < pKey2->groupId) { + return -1; + } else if (pKey1->groupId > pKey2->groupId) { + return 1; + } + + return 0; +} + +static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { + int ret; + FKeyComparator compFunc; + + // Create a database + compFunc = tsdbSmaKeyCmpr; + ret = tdbDbOpen(pFName, TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, ppDB); + + return 0; +} + +static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } + +int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { + // TEnv is shared by a group of SDBFile + if (!pEnv || !pDBF) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + // Open DBF + if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + tsdbCloseDBDb(pDBF->pDB); + return -1; + } + + return 0; +} + +int32_t tsdbCloseDBF(SDBFile *pDBF) { + int32_t ret = 0; + if (pDBF->pDB) { + ret = tsdbCloseDBDb(pDBF->pDB); + pDBF->pDB = NULL; + } + taosMemoryFreeClear(pDBF->path); + return ret; +} + +int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { + int32_t ret; + + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + if (ret < 0) { + tsdbError("Failed to create insert sma data into db, ret = %d", ret); + return -1; + } + + return 0; +} + +void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { + void *result; + void *pVal; + int ret; + + ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + + if (ret < 0) { + tsdbError("Failed to get sma data from db, ret = %d", ret); + return NULL; + } + + ASSERT(*valLen >= 0); + + result = taosMemoryMalloc(*valLen); + + if (result == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // TODO: lock? + // TODO: Would the key/value be destoryed during return the data? + // TODO: How about the key is updated while value length is changed? The original value buffer would be freed + // automatically? + memcpy(result, pVal, *valLen); + + return result; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index bf2260c51c..0449319dc2 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -19,7 +19,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO // blockDebugShowData(data); - // tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); + tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); } void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { @@ -232,9 +232,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? #endif - // if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // // TODO - // } + if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { + // TODO + } // } break; // case TDMT_VND_CANCEL_SMA: { // timeRangeSMA // } break; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 6a4adfe4f8..da874716f2 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -210,7 +210,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { // get value by indexName STSma *qSmaCfg = NULL; - qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1); + qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1, true); assert(qSmaCfg != NULL); printf("name1 = %s\n", qSmaCfg->indexName); printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt); @@ -221,7 +221,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tdDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); - qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2); + qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt); @@ -233,11 +233,12 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { taosMemoryFreeClear(qSmaCfg); // get index name by table uid +#if 0 SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); assert(pSmaCur != NULL); uint32_t indexCnt = 0; while (1) { - const char *indexName = metaSmaCursorNext(pSmaCur); + const char *indexName = (const char *)metaSmaCursorNext(pSmaCur); if (indexName == NULL) { break; } @@ -245,8 +246,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { ++indexCnt; } EXPECT_EQ(indexCnt, nCntTSma); - metaCloseSmaCurosr(pSmaCur); - + metaCloseSmaCursor(pSmaCur); +#endif // get wrapper by table uid STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid); assert(pSW != NULL); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9df749bec7..8a95635ddc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1196,21 +1196,22 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return -1; } - // TODO: vLen may be zero - pVal = TDB_REALLOC(*ppVal, cd.vLen); - if (pVal == NULL) { - TDB_FREE(pKey); - return -1; - } - *ppKey = pKey; - *ppVal = pVal; - *kLen = cd.kLen; - *vLen = cd.vLen; - memcpy(pKey, cd.pKey, cd.kLen); - memcpy(pVal, cd.pVal, cd.vLen); + + if (ppVal) { + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppVal = pVal; + *vLen = cd.vLen; + memcpy(pVal, cd.pVal, cd.vLen); + } ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { diff --git a/tests/script/tsim/sma/tsmaCreateInsertData.sim b/tests/script/tsim/sma/tsmaCreateInsertData.sim new file mode 100644 index 0000000000..87140e5d84 --- /dev/null +++ b/tests/script/tsim/sma/tsmaCreateInsertData.sim @@ -0,0 +1,49 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d1 +sql show databases +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d1 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) + +sql show tables +if $rows != 1 then + return -1 +endi + +print =============== insert data, mode1: one row one table in sql +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) + + +print =============== create sma index from super table +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(2m) +print $data00 $data01 $data02 $data03 +sleep 300 + +print =============== trigger stream to execute sma aggr task and insert sma data into sma store +sql insert into ct1 values(now+5s, 20, 20.0, 30.0) +#=================================================================== + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From d9e8a40f26e7702be9bf72345528a52a89fcf2aa Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:05:23 +0800 Subject: [PATCH 12/20] add tsma create/insert data case --- tests/script/jenkins/basic.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index c5c041be91..c14d4fe27b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -80,4 +80,7 @@ ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m +# --- sma +./test.sh -f tsim/sma/tsmaCreateInsertData.sim + #======================b1-end=============== From 79bc57977964e445688e23ebd3696f2acce40969 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 13 Apr 2022 16:07:38 +0800 Subject: [PATCH 13/20] test: fix tmq test case --- tests/test/c/tmqSim.c | 226 ++++++++++++++++++++---------------------- 1 file changed, 109 insertions(+), 117 deletions(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 2527f1d01a..702d88f258 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -13,51 +13,49 @@ * along with this program. If not, see . */ -// clang-format off - #include +#include #include +#include #include -#include #include #include +#include #include -#include -#include #include "taos.h" #include "taoserror.h" #include "tlog.h" -#define GREEN "\033[1;32m" -#define NC "\033[0m" +#define GREEN "\033[1;32m" +#define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) typedef struct { - // input from argvs - char dbName[32]; - char topicString[256]; - char keyString[1024]; - int32_t showMsgFlag; - int32_t consumeDelay; // unit s - int32_t consumeMsgCnt; + // input from argvs + char dbName[32]; + char topicString[256]; + char keyString[1024]; + int32_t showMsgFlag; + int32_t consumeDelay; // unit s + int32_t consumeMsgCnt; - // save result after parse agrvs - int32_t numOfTopic; - char topics[32][64]; - - int32_t numOfKey; - char key[32][64]; - char value[32][64]; + // save result after parse agrvs + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; } SConfInfo; static SConfInfo g_stConfInfo; -//char* g_pRowValue = NULL; -//TdFilePtr g_fp = NULL; +// char* g_pRowValue = NULL; +// TdFilePtr g_fp = NULL; static void printHelp() { char indent[10] = " "; @@ -80,13 +78,12 @@ static void printHelp() { exit(EXIT_SUCCESS); } -void parseArgument(int32_t argc, char *argv[]) { - +void parseArgument(int32_t argc, char* argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); - g_stConfInfo.showMsgFlag = 0; - g_stConfInfo.consumeDelay = 8000; + g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.consumeDelay = 8000; g_stConfInfo.consumeMsgCnt = 0; - + for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { printHelp(); @@ -107,7 +104,7 @@ void parseArgument(int32_t argc, char *argv[]) { g_stConfInfo.consumeMsgCnt = atol(argv[++i]); } else { printf("%s unknow para: %s %s", GREEN, argv[++i], NC); - exit(-1); + exit(-1); } } @@ -117,91 +114,87 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s topicString:%s %s", GREEN, g_stConfInfo.topicString, NC); pPrint("%s keyString:%s %s", GREEN, g_stConfInfo.keyString, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); -#endif +#endif } -void splitStr(char **arr, char *str, const char *del) { - char *s = strtok(str, del); - while(s != NULL) { +void splitStr(char** arr, char* str, const char* del) { + char* s = strtok(str, del); + while (s != NULL) { *arr++ = s; s = strtok(NULL, del); } } -void ltrim(char *str) -{ - if (str == NULL || *str == '\0') - { - return; - } - int len = 0; - char *p = str; - while (*p != '\0' && isspace(*p)) - { - ++p; ++len; - } - memmove(str, p, strlen(str) - len + 1); - //return str; +void ltrim(char* str) { + if (str == NULL || *str == '\0') { + return; + } + int len = 0; + char* p = str; + while (*p != '\0' && isspace(*p)) { + ++p; + ++len; + } + memmove(str, p, strlen(str) - len + 1); + // return str; } - void parseInputString() { - //printf("topicString: %s\n", g_stConfInfo.topicString); - //printf("keyString: %s\n\n", g_stConfInfo.keyString); + // printf("topicString: %s\n", g_stConfInfo.topicString); + // printf("keyString: %s\n\n", g_stConfInfo.keyString); - char *token; + char* token; const char delim[2] = ","; const char ch = ':'; token = strtok(g_stConfInfo.topicString, delim); - while(token != NULL) { - //printf("%s\n", token ); - strcpy(g_stConfInfo.topics[g_stConfInfo.numOfTopic], token); + while (token != NULL) { + // printf("%s\n", token ); + strcpy(g_stConfInfo.topics[g_stConfInfo.numOfTopic], token); ltrim(g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - //printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - g_stConfInfo.numOfTopic++; - + // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.numOfTopic++; + token = strtok(NULL, delim); } token = strtok(g_stConfInfo.keyString, delim); - while(token != NULL) { - //printf("%s\n", token ); - { - char* pstr = token; - ltrim(pstr); - char *ret = strchr(pstr, ch); - memcpy(g_stConfInfo.key[g_stConfInfo.numOfKey], pstr, ret-pstr); - strcpy(g_stConfInfo.value[g_stConfInfo.numOfKey], ret+1); - //printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], g_stConfInfo.value[g_stConfInfo.numOfKey]); - g_stConfInfo.numOfKey++; + while (token != NULL) { + // printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char* ret = strchr(pstr, ch); + memcpy(g_stConfInfo.key[g_stConfInfo.numOfKey], pstr, ret - pstr); + strcpy(g_stConfInfo.value[g_stConfInfo.numOfKey], ret + 1); + // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], + // g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.numOfKey++; } - + token = strtok(NULL, delim); } } - -static int running = 1; +static int running = 1; /*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ - -int queryDB(TAOS *taos, char *command) { - TAOS_RES *pRes = taos_query(taos, command); - int code = taos_errno(pRes); - //if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - if (code != 0) { - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; - } - taos_free_result(pRes); - return 0 ; +int queryDB(TAOS* taos, char* command) { + TAOS_RES* pRes = taos_query(taos, command); + int code = taos_errno(pRes); + // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + if (code != 0) { + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; + } + taos_free_result(pRes); + return 0; } tmq_t* build_consumer() { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -209,13 +202,13 @@ tmq_t* build_consumer() { TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); + taos_free_result(pRes); + exit(-1); } taos_free_result(pRes); tmq_conf_t* conf = tmq_conf_new(); - //tmq_conf_set(conf, "group.id", "tg2"); + // tmq_conf_set(conf, "group.id", "tg2"); for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) { tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]); } @@ -225,7 +218,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); - //tmq_list_append(topic_list, "test_stb_topic_1"); + // tmq_list_append(topic_list, "test_stb_topic_1"); for (int32_t i = 0; i < g_stConfInfo.numOfTopic; i++) { tmq_list_append(topic_list, g_stConfInfo.topics[i]); } @@ -241,19 +234,19 @@ void loop_consume(tmq_t* tmq) { while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, 8000); if (tmqMsg) { - totalMsgs++; + totalMsgs++; - #if 0 +#if 0 TAOS_ROW row; while (NULL != (row = tmq_get_row(tmqMsg))) { totalRows++; } - #endif - +#endif + /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/ - if (0 != g_stConfInfo.showMsgFlag) { + if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqMsg);*/ - } + } tmq_message_destroy(tmqMsg); } else { break; @@ -263,13 +256,12 @@ void loop_consume(tmq_t* tmq) { err = tmq_consumer_close(tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); + exit(-1); } - + printf("{consume success: %d, %d}", totalMsgs, totalRows); } - void parallel_consume(tmq_t* tmq) { tmq_resp_err_t err; @@ -277,26 +269,26 @@ void parallel_consume(tmq_t* tmq) { int32_t totalRows = 0; int32_t skipLogNum = 0; while (running) { - tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000); + TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000); if (tmqMsg) { - totalMsgs++; + totalMsgs++; - #if 0 +#if 0 TAOS_ROW row; while (NULL != (row = tmq_get_row(tmqMsg))) { totalRows++; } - #endif - - skipLogNum += tmqGetSkipLogNum(tmqMsg); - if (0 != g_stConfInfo.showMsgFlag) { - msg_process(tmqMsg); - } +#endif + + /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/ + if (0 != g_stConfInfo.showMsgFlag) { + /*msg_process(tmqMsg);*/ + } tmq_message_destroy(tmqMsg); - if (totalMsgs >= g_stConfInfo.consumeMsgCnt) { + if (totalMsgs >= g_stConfInfo.consumeMsgCnt) { break; - } + } } else { break; } @@ -305,22 +297,22 @@ void parallel_consume(tmq_t* tmq) { err = tmq_consumer_close(tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); + exit(-1); } - - printf("%d", totalMsgs); // output to sim for check result + + printf("%d", totalMsgs); // output to sim for check result } -int main(int32_t argc, char *argv[]) { +int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); parseInputString(); - - tmq_t* tmq = build_consumer(); + + tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); - if ((NULL == tmq) || (NULL == topic_list)){ + if ((NULL == tmq) || (NULL == topic_list)) { return -1; } - + tmq_resp_err_t err = tmq_subscribe(tmq, topic_list); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); From 7cf63f0990ef8e4d87ca7bc281133e1891cf6de0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:09:55 +0800 Subject: [PATCH 14/20] remove sleep --- tests/script/tsim/sma/tsmaCreateInsertData.sim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/tsim/sma/tsmaCreateInsertData.sim b/tests/script/tsim/sma/tsmaCreateInsertData.sim index 87140e5d84..9fc3700da1 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertData.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertData.sim @@ -39,7 +39,6 @@ sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s print =============== create sma index from super table sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(2m) print $data00 $data01 $data02 $data03 -sleep 300 print =============== trigger stream to execute sma aggr task and insert sma data into sma store sql insert into ct1 values(now+5s, 20, 20.0, 30.0) From 11e772d4d4074c9160db4f5089770a1664ffb2a0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 13 Apr 2022 16:21:54 +0800 Subject: [PATCH 15/20] fix(grant): taosd -k. --- include/os/osFile.h | 2 +- include/os/osSystem.h | 7 ++++ source/dnode/mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/exe/dndMain.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 7 ++++ .../dnode/mnode/impl/inc/mndGrant.h | 4 +-- source/dnode/mnode/impl/src/mndGrant.c | 6 ++-- source/dnode/mnode/impl/src/mnode.c | 2 ++ source/os/src/osFile.c | 2 +- source/os/src/osSystem.c | 32 +++++++++++++++++++ 10 files changed, 57 insertions(+), 9 deletions(-) rename include/common/tgrant.h => source/dnode/mnode/impl/inc/mndGrant.h (92%) diff --git a/include/os/osFile.h b/include/os/osFile.h index 89b58cdd65..36ca6fb8bb 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 15959a2d8c..33b0a46ee9 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,13 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +typedef struct TdCmd *TdCmdPtr; + +TdCmdPtr taosOpenCmd(const char *cmd); +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); +int32_t taosEOFCmd(TdCmdPtr pCmd); +int64_t taosCloseCmd(TdCmdPtr *ppCmd); + void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 297e3a08d7..8fb4b02479 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" ) target_link_libraries(taosd dnode) @@ -28,7 +29,6 @@ IF (TD_GRANT) ENDIF () IF (TD_USB_DONGLE) TARGET_LINK_LIBRARIES(taosd usb_dongle) -else() ENDIF () if(${BUILD_TEST}) diff --git a/source/dnode/mgmt/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c index 66ab2a6dcd..80c431fe36 100644 --- a/source/dnode/mgmt/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" #include "tconfig.h" -#include "tgrant.h" +#include "mndGrant.h" static struct { bool dumpConfig; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 341dbf6135..8cb5e2a528 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -9,6 +9,13 @@ target_link_libraries( mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(mnode grant) +ENDIF () +IF (TD_USB_DONGLE) + TARGET_LINK_LIBRARIES(mnode usb_dongle) +ENDIF () + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/include/common/tgrant.h b/source/dnode/mnode/impl/inc/mndGrant.h similarity index 92% rename from include/common/tgrant.h rename to source/dnode/mnode/impl/inc/mndGrant.h index 962af0ddc4..ad3dc7f79d 100644 --- a/include/common/tgrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -36,8 +36,8 @@ typedef enum { TSDB_GRANT_CPU_CORES, } EGrantType; -int32_t grantInit(); -void grantCleanUp(); +int32_t mndInitGrant(); +void mndCleanupGrant(); void grantParseParameter(); int32_t grantCheck(EGrantType grant); void grantReset(EGrantType grant, uint64_t value); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index cdf3271789..37f87142e6 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -17,11 +17,11 @@ #ifndef _GRANT #include "os.h" #include "taoserror.h" -#include "tgrant.h" +#include "mndGrant.h" #include "mndInt.h" -int32_t grantInit() { return TSDB_CODE_SUCCESS; } -void grantCleanUp() {} +int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } +void mndCleanupGrant() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } void grantReset(EGrantType grant, uint64_t value) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5a1780530c..cad6396338 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -40,6 +40,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndQuery.h" +#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -197,6 +198,7 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4bd6b9e5cd..4b55d4912c 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } -int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (pFile == NULL) { return -1; } diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index a36b3d41d0..665f6370e1 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,6 +29,8 @@ struct termios oldtio; #endif +typedef struct FILE TdCmd; + void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -178,3 +180,33 @@ void resetTerminalMode() { } #endif } + +TdCmdPtr taosOpenCmd(const char *cmd) { + if (cmd == NULL) return NULL; + return (TdCmdPtr)popen(cmd, "r"); +} + +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { + if (pCmd == NULL) { + return -1; + } + + size_t len = 0; + return getline(ptrBuf, &len, (FILE*)pCmd); +} + +int32_t taosEOFCmd(TdCmdPtr pCmd) { + if (pCmd == NULL) { + return 0; + } + return feof((FILE*)pCmd); +} + +int64_t taosCloseCmd(TdCmdPtr *ppCmd) { + if (ppCmd == NULL || *ppCmd == NULL) { + return 0; + } + pclose((FILE*)(*ppCmd)); + *ppCmd = NULL; + return 0; +} From 222987c002125434357038c16636366ab115e779 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 13 Apr 2022 16:39:36 +0800 Subject: [PATCH 16/20] Revert "fix(grant): taosd -k." This reverts commit 11e772d4d4074c9160db4f5089770a1664ffb2a0. --- .../inc/mndGrant.h => include/common/tgrant.h | 4 +-- include/os/osFile.h | 2 +- include/os/osSystem.h | 7 ---- source/dnode/mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/exe/dndMain.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 7 ---- source/dnode/mnode/impl/src/mndGrant.c | 6 ++-- source/dnode/mnode/impl/src/mnode.c | 2 -- source/os/src/osFile.c | 2 +- source/os/src/osSystem.c | 32 ------------------- 10 files changed, 9 insertions(+), 57 deletions(-) rename source/dnode/mnode/impl/inc/mndGrant.h => include/common/tgrant.h (92%) diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/include/common/tgrant.h similarity index 92% rename from source/dnode/mnode/impl/inc/mndGrant.h rename to include/common/tgrant.h index ad3dc7f79d..962af0ddc4 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/include/common/tgrant.h @@ -36,8 +36,8 @@ typedef enum { TSDB_GRANT_CPU_CORES, } EGrantType; -int32_t mndInitGrant(); -void mndCleanupGrant(); +int32_t grantInit(); +void grantCleanUp(); void grantParseParameter(); int32_t grantCheck(EGrantType grant); void grantReset(EGrantType grant, uint64_t value); diff --git a/include/os/osFile.h b/include/os/osFile.h index 36ca6fb8bb..89b58cdd65 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 33b0a46ee9..15959a2d8c 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,13 +29,6 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif -typedef struct TdCmd *TdCmdPtr; - -TdCmdPtr taosOpenCmd(const char *cmd); -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); -int32_t taosEOFCmd(TdCmdPtr pCmd); -int64_t taosCloseCmd(TdCmdPtr *ppCmd); - void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 8fb4b02479..297e3a08d7 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -20,7 +20,6 @@ add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" - PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" ) target_link_libraries(taosd dnode) @@ -29,6 +28,7 @@ IF (TD_GRANT) ENDIF () IF (TD_USB_DONGLE) TARGET_LINK_LIBRARIES(taosd usb_dongle) +else() ENDIF () if(${BUILD_TEST}) diff --git a/source/dnode/mgmt/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c index 80c431fe36..66ab2a6dcd 100644 --- a/source/dnode/mgmt/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" #include "tconfig.h" -#include "mndGrant.h" +#include "tgrant.h" static struct { bool dumpConfig; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 8cb5e2a528..341dbf6135 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -9,13 +9,6 @@ target_link_libraries( mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) -IF (TD_GRANT) - TARGET_LINK_LIBRARIES(mnode grant) -ENDIF () -IF (TD_USB_DONGLE) - TARGET_LINK_LIBRARIES(mnode usb_dongle) -ENDIF () - if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 37f87142e6..cdf3271789 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -17,11 +17,11 @@ #ifndef _GRANT #include "os.h" #include "taoserror.h" -#include "mndGrant.h" +#include "tgrant.h" #include "mndInt.h" -int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } -void mndCleanupGrant() {} +int32_t grantInit() { return TSDB_CODE_SUCCESS; } +void grantCleanUp() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } void grantReset(EGrantType grant, uint64_t value) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index cad6396338..5a1780530c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -40,7 +40,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndQuery.h" -#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -198,7 +197,6 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4b55d4912c..4bd6b9e5cd 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } -int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { if (pFile == NULL) { return -1; } diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 665f6370e1..a36b3d41d0 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,8 +29,6 @@ struct termios oldtio; #endif -typedef struct FILE TdCmd; - void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -180,33 +178,3 @@ void resetTerminalMode() { } #endif } - -TdCmdPtr taosOpenCmd(const char *cmd) { - if (cmd == NULL) return NULL; - return (TdCmdPtr)popen(cmd, "r"); -} - -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { - if (pCmd == NULL) { - return -1; - } - - size_t len = 0; - return getline(ptrBuf, &len, (FILE*)pCmd); -} - -int32_t taosEOFCmd(TdCmdPtr pCmd) { - if (pCmd == NULL) { - return 0; - } - return feof((FILE*)pCmd); -} - -int64_t taosCloseCmd(TdCmdPtr *ppCmd) { - if (ppCmd == NULL || *ppCmd == NULL) { - return 0; - } - pclose((FILE*)(*ppCmd)); - *ppCmd = NULL; - return 0; -} From 9a09f15de8bf754dcbf0c6ad291985bcbfc5a660 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:38:57 +0800 Subject: [PATCH 17/20] feat(query): add timediff function TD-14243 --- include/libs/scalar/scalar.h | 5 +- source/libs/function/src/builtins.c | 14 +++ source/libs/scalar/src/sclfunc.c | 136 ++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index caea52a5ef..10b4866a96 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -31,8 +31,8 @@ pNode will be freed in API; */ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -/* -pDst need to freed in caller +/* +pDst need to freed in caller */ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); @@ -77,6 +77,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 36de0d1149..27067fc966 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -423,6 +423,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = timeTruncateFunction, .finalizeFunc = NULL }, + { + .name = "timediff", + .type = FUNCTION_TYPE_TIMEDIFF, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = checkAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timeDiffFunction, + .finalizeFunc = NULL + }, { .name = "_rowts", .type = FUNCTION_TYPE_ROWTS, @@ -651,6 +661,10 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; break; } + case FUNCTION_TYPE_TIMEDIFF: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + break; + } case FUNCTION_TYPE_TBNAME: { // todo diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b53dc955de..76059cf2e1 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1104,6 +1104,142 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } +int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 && inputNum != 3) { + return TSDB_CODE_FAILED; + } + + int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); + int64_t timeUnit = -1, timeVal[2] = {0}; + if (inputNum == 3) { + if (GET_PARAM_TYPE(&pInput[2]) != TSDB_DATA_TYPE_BIGINT) { + return TSDB_CODE_FAILED; + } + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + } + + char *input[2]; + for (int32_t k = 0; k < 2; ++k) { + int32_t type = GET_PARAM_TYPE(&pInput[k]); + if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && + type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (IS_VAR_DATA_TYPE(type)) { + input[k] = pInput[k].columnData->pData + pInput[k].columnData->varmeta.offset[0]; + } else { + input[k] = pInput[k].columnData->pData; + } + } + + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + for (int32_t k = 0; k < 2; ++k) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + int32_t type = GET_PARAM_TYPE(&pInput[k]); + if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ + convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); + } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ + GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); + if (type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : + (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t timeValSec = timeVal[k] / factor; + if (timeValSec < 1000000000) { + timeVal[k] = timeValSec; + } + } + + char buf[20] = {0}; + NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal[k], sizeof(buf), buf); + int32_t tsDigits = (int32_t)strlen(buf); + if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal[k] = timeVal[k] * 1000000000; + } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal[k] = timeVal[k] * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal[k] = timeVal[k] * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal[k] = timeVal[k]; + } + } + + if (IS_VAR_DATA_TYPE(type)) { + input[k] += varDataTLen(input[k]); + } else { + input[k] += tDataTypes[type].bytes; + } + } + + int64_t result = (timeVal[0] >= timeVal[1]) ? (timeVal[0] - timeVal[1]) : + (timeVal[1] - timeVal[0]); + + if (timeUnit < 0) { // if no time unit given use db precision + switch(timePrec) { + case TSDB_TIME_PRECISION_MILLI: { + result = result / 1000000; + break; + } + case TSDB_TIME_PRECISION_MICRO: { + result = result / 1000; + break; + } + case TSDB_TIME_PRECISION_NANO: { + result = result / 1; + break; + } + } + } else { + int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : + (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + timeUnit = timeUnit * 1000 / factor; + switch(timeUnit) { + case 0: { /* 1u */ + result = result / 1000; + break; + } + case 1: { /* 1a */ + result = result / 1000000; + break; + } + case 1000: { /* 1s */ + result = result / 1000000000; + break; + } + case 60000: { /* 1m */ + result = result / 1000000000 / 60; + break; + } + case 3600000: { /* 1h */ + result = result / 1000000000 / 3600; + break; + } + case 86400000: { /* 1d */ + result = result / 1000000000 / 86400; + break; + } + case 604800000: { /* 1w */ + result = result / 1000000000 / 604800; + break; + } + default: { + break; + } + } + } + + colDataAppend(pOutput->columnData, i, (char *)&result, false); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } From e5b0b962d7e4f7a72a19375f5a79e23f6c3e94e4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 13 Apr 2022 17:00:41 +0800 Subject: [PATCH 18/20] test: adjust test case for show databases result on 3.0 (#11454) * [TD-13558]: taos shell refactor add taosTools as submodule * add tools/taos-tools * add more client interface for taosTools compile * update taos-tools * update taos-tools * refactor shell * [TD-13558]: taos shell test speed * [TD-13558]: taos -n startup works * taos -n rpc works * taos -n server works * cleanup code since no endPort in 3.0 * update taos-tools * [TD-13558]: taos -C works * improve taos shell -c WIP * update taos-tools * add demoapi.c * adjust show databases result for 3.0 --- tests/pytest/insert/binary.py | 41 +++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0cbb7876c6..567a2eaad5 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -13,6 +13,23 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getPath(self, tool="taos"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + return paths[0] + def run(self): tdSql.prepare() @@ -53,15 +70,31 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') - result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) - if "Query OK" not in result: tdLog.exit("err:insert '';") + config_dir = subprocess.check_output( + str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), + stderr=subprocess.STDOUT, + shell=True).decode('utf-8').replace( + '\n', + '') + + binPath = self.getPath("taos") + if (binPath == ""): + tdLog.exit("taos not found!") + else: + tdLog.info("taos found: %s" % binPath) + + result = ''.join( + os.popen( + r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % + (binPath, (config_dir))).readlines()) + if "Query OK" not in result: + tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') tdSql.query('show databases') tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) + tdSql.checkRows(1) # convert end def stop(self): From eabfc7d31cf38134d405bcc6c33adaffc50d91a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:38:57 +0800 Subject: [PATCH 19/20] feat(query): add timediff function TD-14243 --- source/libs/scalar/src/sclfunc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 76059cf2e1..db62a6b33d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1168,10 +1168,12 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } } - if (IS_VAR_DATA_TYPE(type)) { - input[k] += varDataTLen(input[k]); - } else { - input[k] += tDataTypes[type].bytes; + if (pInput[k].numOfRows != 1) { + if (IS_VAR_DATA_TYPE(type)) { + input[k] += varDataTLen(input[k]); + } else { + input[k] += tDataTypes[type].bytes; + } } } From 6eba8ad1ab461b80f8348c923f1966a3b8b7d4e2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 13 Apr 2022 17:50:33 +0800 Subject: [PATCH 20/20] test: add platform logic (#11459) * [TD-13558]: taos shell refactor add taosTools as submodule * add tools/taos-tools * add more client interface for taosTools compile * update taos-tools * update taos-tools * refactor shell * [TD-13558]: taos shell test speed * [TD-13558]: taos -n startup works * taos -n rpc works * taos -n server works * cleanup code since no endPort in 3.0 * update taos-tools * [TD-13558]: taos -C works * improve taos shell -c WIP * update taos-tools * add demoapi.c * adjust show databases result for 3.0 * test: add platform logic --- tests/pytest/insert/binary.py | 44 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 567a2eaad5..ffd1d6cb8c 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import platform import sys from util.log import * from util.cases import * @@ -70,32 +71,27 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output( - str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), - stderr=subprocess.STDOUT, - shell=True).decode('utf-8').replace( - '\n', - '') - binPath = self.getPath("taos") - if (binPath == ""): - tdLog.exit("taos not found!") - else: - tdLog.info("taos found: %s" % binPath) + if platform.system() == "Linux": + config_dir = subprocess.check_output( + str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), + stderr=subprocess.STDOUT, + shell=True).decode('utf-8').replace( + '\n', + '') - result = ''.join( - os.popen( - r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % - (binPath, (config_dir))).readlines()) - if "Query OK" not in result: - tdLog.exit("err:insert '';") - tdLog.info('drop database db') - tdSql.execute('drop database db') - tdLog.info('show databases') - tdSql.query('show databases') - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(1) -# convert end + binPath = self.getPath("taos") + if (binPath == ""): + tdLog.exit("taos not found!") + else: + tdLog.info("taos found: %s" % binPath) + + result = ''.join( + os.popen( + r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % + (binPath, (config_dir))).readlines()) + if "Query OK" not in result: + tdLog.exit("err:insert '';") def stop(self): tdSql.close()