diff --git a/include/common/ttime.h b/include/common/ttime.h index 306f54bedb..15450c31ca 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -64,6 +64,7 @@ char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); +int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal); void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 9d1b2c9ea0..148a9f77d6 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -85,8 +85,8 @@ typedef enum EFunctionType { // conversion function FUNCTION_TYPE_CAST = 2000, FUNCTION_TYPE_TO_ISO8601, + FUNCTION_TYPE_TO_UNIXTIMESTAMP, FUNCTION_TYPE_TO_JSON, - FUNCTION_TYPE_UNIXTIMESTAMP, // date and time function FUNCTION_TYPE_NOW = 2500, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index cd9c6f5bf2..226909a187 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -75,6 +75,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp /* Time related functions */ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/packaging/install.sh b/packaging/install.sh index 3aae074af5..49386ba05a 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -199,6 +199,8 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/* + ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index a65352f2b9..1baf393e9a 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -406,7 +406,31 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char default: { return -1; } - } + } +} + +int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { + int32_t charLen = varDataLen(inputData); + char *newColData; + if (type == TSDB_DATA_TYPE_BINARY) { + newColData = taosMemoryCalloc(1, charLen + 1); + memcpy(newColData, varDataVal(inputData), charLen); + taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + taosMemoryFree(newColData); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + newColData = taosMemoryCalloc(1, charLen / TSDB_NCHAR_SIZE + 1); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inputData), charLen, newColData); + if (len < 0){ + taosMemoryFree(newColData); + return TSDB_CODE_FAILED; + } + newColData[len] = 0; + taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + taosMemoryFree(newColData); + } else { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index ab5a02c438..624a2953c9 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -22,7 +22,7 @@ extern "C" { #include "functionMgt.h" -#define FUNCTION_NAME_MAX_LENGTH 16 +#define FUNCTION_NAME_MAX_LENGTH 32 #define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 55f38e9005..44888fa793 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -403,6 +403,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = toISO8601Function, .finalizeFunc = NULL }, + { + .name = "to_unixtimestamp", + .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = checkAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toUnixtimestampFunction, + .finalizeFunc = NULL + }, { .name = "_rowts", .type = FUNCTION_TYPE_ROWTS, @@ -622,6 +632,9 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_TO_ISO8601: { pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; } + case FUNCTION_TYPE_TO_UNIXTIMESTAMP: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + } case FUNCTION_TYPE_TBNAME: { // todo diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 58b62bb05e..3d59ffd93d 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -46,8 +46,9 @@ typedef struct SScalarCtx { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); -#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) -#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) +#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) +#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) +#define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam *param); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0ab0052624..e2d28fe454 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,6 +1,7 @@ #include "function.h" #include "scalar.h" #include "tdatablock.h" +#include "ttime.h" #include "sclInt.h" #include "sclvector.h" @@ -872,6 +873,36 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * return TSDB_CODE_SUCCESS; } +int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + int32_t timePrec = GET_PARAM_PRECISON(pInput); + if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (inputNum != 1) { + return TSDB_CODE_FAILED; + } + + char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + int64_t timeVal = 0; + convertStringToTimestamp(type, input, timePrec, &timeVal); + + colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + input += varDataTLen(input); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } diff --git a/tests/script/tsim/tmq/consumerMain.sim b/tests/script/tsim/tmq/consumerMain.sim deleted file mode 100644 index 51b90971fd..0000000000 --- a/tests/script/tsim/tmq/consumerMain.sim +++ /dev/null @@ -1,267 +0,0 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## -######## This test case include scene1 and scene3 -######## ######## ######## ######## ######## ######## ######## ######## ######## ######## - -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - -sql connect - -$loop_cnt = 0 -$vgroups = 1 -$dbNamme = d0 -loop_vgroups: -print =============== create database $dbNamme vgroups $vgroups -sql create database $dbNamme vgroups $vgroups -sql show databases -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 - -if $loop_cnt == 0 then - if $rows != 2 then - return -1 - endi - if $data02 != 1 then # vgroups - print vgroups: $data02 - return -1 - endi -else - if $rows != 3 then - return -1 - endi - if $data00 == d1 then - if $data02 != 4 then # vgroups - print vgroups: $data02 - return -1 - endi - else - if $data12 != 4 then # vgroups - print vgroups: $data12 - return -1 - endi - endi -endi - -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -$tbPrefix = ct -$tbNum = 2 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using stb tags( $i ) - $i = $i + 1 -endw - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb -print =============== will support: * from stb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 - -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb - -sql show tables -if $rows != 3 then - return -1 -endi - -print =============== run_back insert data - -if $loop_cnt == 0 then - run_back tsim/tmq/insertDataV1.sim -else - run_back tsim/tmq/insertDataV4.sim -endi - -sleep 1000 - -#$rowNum = 1000 -#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -# -#$i = 0 -#while $i < $tbNum -# $tb = $tbPrefix . $i -# -# $x = 0 -# while $x < $rowNum -# $c = $x / 10 -# $c = $c * 10 -# $c = $x - $c -# -# $binary = ' . binary -# $binary = $binary . $c -# $binary = $binary . ' -# -# sql insert into $tb values ($tstart , $c , $x , $binary ) -# sql insert into ntb values ($tstart , $c , $x , $binary ) -# $tstart = $tstart + 1 -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $tstart = 1640966400000 -#endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$consumeDelay = 5000 -$expectConsumeMsgCnt = 1000 -print expectConsumeMsgCnt: $expectConsumeMsgCnt, consumeDelay: $consumeDelay - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#print cmd result----> $system_content -##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content != $expect_result then -# return -1 -#endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -#if $system_content != @{consume success: 10000, 0}@ then -if $system_content >= $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -$expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCnt -$expect_result = $expect_result . @, @ -$expect_result = $expect_result . 0} -print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd result----> $system_content -if $system_content >= $expect_result then - return -1 -endi - -if $loop_cnt == 0 then - $loop_cnt = 1 - $vgroups = 4 - $dbNamme = d1 - goto loop_vgroups -endi - - -#system sh/exec.sh -n dnode1 -s stop -x SIGINT