From 5685346d7884dc980ebf0c74b20d06932022ac24 Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 24 Jun 2024 20:28:16 +0800 Subject: [PATCH 1/4] fix:[TD-30730] Modify precision rules for input parameters of function to_iso8601 and add test. --- source/libs/function/src/builtins.c | 9 --- source/libs/scalar/src/sclfunc.c | 78 ++++++++++++------------- tests/system-test/2-query/To_iso8601.py | 60 ++++++++++++++++++- 3 files changed, 95 insertions(+), 52 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 498b46dcfe..e3e84ac20b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2166,15 +2166,6 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) { - SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); - - if (!validateTimestampDigits(pValue)) { - pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY}; - return TSDB_CODE_SUCCESS; - } - } - // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 48bedde91a..6d87ae9baa 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1095,24 +1095,45 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * char fraction[20] = {0}; bool hasFraction = false; NUM_TO_STRING(type, input, sizeof(fraction), fraction); - int32_t tsDigits = (int32_t)strlen(fraction); + int32_t fractionLen; char buf[64] = {0}; int64_t timeVal; + char* format = NULL; + int64_t quot = 0; + long mod = 0; + GET_TYPED_DATA(timeVal, int64_t, type, input); - if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / ((int64_t)(1000 * 1000)); - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / ((int64_t)(1000 * 1000 * 1000)); - } else { + + switch (pInput->columnData[0].info.precision) { + case TSDB_TIME_PRECISION_MILLI: { + quot = timeVal / 1000; + fractionLen = 5; + format = ".%03" PRId64; + mod = timeVal % 1000; + break; + } + + case TSDB_TIME_PRECISION_MICRO: { + quot = timeVal / 1000000; + fractionLen = 8; + format = ".%06" PRId64; + mod = timeVal % 1000000; + break; + } + + case TSDB_TIME_PRECISION_NANO: { + quot = timeVal / 1000000000; + fractionLen = 11; + format = ".%09" PRId64; + mod = timeVal % 1000000000; + break; + } + + default: { colDataSetNULL(pOutput->columnData, i); continue; } - hasFraction = true; - memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); } // trans current timezone's unix ts to dest timezone @@ -1122,18 +1143,19 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * if (0 != offsetOfTimezone(tz, &offset)) { goto _end; } - timeVal -= offset + 3600 * ((int64_t)tsTimezone); + quot -= offset + 3600 * ((int64_t)tsTimezone); struct tm tmInfo; int32_t len = 0; - if (taosLocalTime((const time_t *)&timeVal, &tmInfo, buf) == NULL) { + if (taosLocalTime((const time_t *)", &tmInfo, buf) == NULL) { len = (int32_t)strlen(buf); goto _end; } - strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); - len = (int32_t)strlen(buf); + len = (int32_t)strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); + + len += snprintf(buf + len, fractionLen, format, mod); // add timezone string if (tzLen > 0) { @@ -1141,32 +1163,6 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * len += tzLen; } - if (hasFraction) { - int32_t fracLen = (int32_t)strlen(fraction) + 1; - - char *tzInfo; - if (buf[len - 1] == 'z' || buf[len - 1] == 'Z') { - tzInfo = &buf[len - 1]; - memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo)); - } else { - tzInfo = strchr(buf, '+'); - if (tzInfo) { - memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo)); - } else { - // search '-' backwards - tzInfo = strrchr(buf, '-'); - if (tzInfo) { - memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo)); - } - } - } - - char tmp[32] = {0}; - sprintf(tmp, ".%s", fraction); - memcpy(tzInfo, tmp, fracLen); - len += fracLen; - } - _end: memmove(buf + VARSTR_HEADER_SIZE, buf, len); varDataSetLen(buf, len); diff --git a/tests/system-test/2-query/To_iso8601.py b/tests/system-test/2-query/To_iso8601.py index 160473ffce..24865ada88 100644 --- a/tests/system-test/2-query/To_iso8601.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -19,6 +19,61 @@ class TDTestCase: self.dbname = 'db' self.stbname = f'{self.dbname}.stb' self.ntbname = f'{self.dbname}.ntb' + def check_timestamp_precision(self): + time_zone = time.strftime('%z') + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname} precision "us"') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create table if not exists {self.ntbname}(ts timestamp, c1 int, c2 timestamp)') + tdSql.execute(f'insert into {self.ntbname} values(now,1,today())') + ts_list = ['1', '11', '111', '1111', '11111', '111111', '1111111', '11111111', '111111111', '1111111111', + '11111111111','111111111111','1111111111111','11111111111111','111111111111111','1111111111111111', + '11111111111111111','111111111111111111','1111111111111111111'] + res_list_ms = ['1970-01-01T08:00:00.001+0800', '1970-01-01T08:00:00.011+0800', '1970-01-01T08:00:00.111+0800', + '1970-01-01T08:00:01.111+0800', '1970-01-01T08:00:11.111+0800', '1970-01-01T08:01:51.111+0800', + '1970-01-01T08:18:31.111+0800', '1970-01-01T11:05:11.111+0800', '1970-01-02T14:51:51.111+0800', + '1970-01-14T04:38:31.111+0800', '1970-05-09T22:25:11.111+0800', '1973-07-10T08:11:51.111+0800', + '2005-03-18T09:58:31.111+0800', '2322-02-06T03:45:11.111+0800', '5490-12-21T13:31:51.111+0800', + '37179-09-17T15:18:31.111+0800', '354067-02-04T09:05:11.111+0800', + '3522940-12-11T18:51:51.111+0800', '35211679-06-14T20:38:31.111+0800'] + res_list_us = ['1970-01-01T08:00:00.000001+0800', '1970-01-01T08:00:00.000011+0800', + '1970-01-01T08:00:00.000111+0800', '1970-01-01T08:00:00.001111+0800', + '1970-01-01T08:00:00.011111+0800', '1970-01-01T08:00:00.111111+0800', + '1970-01-01T08:00:01.111111+0800', '1970-01-01T08:00:11.111111+0800', + '1970-01-01T08:01:51.111111+0800', '1970-01-01T08:18:31.111111+0800', + '1970-01-01T11:05:11.111111+0800', '1970-01-02T14:51:51.111111+0800', + '1970-01-14T04:38:31.111111+0800', '1970-05-09T22:25:11.111111+0800', + '1973-07-10T08:11:51.111111+0800', '2005-03-18T09:58:31.111111+0800', + '2322-02-06T03:45:11.111111+0800', '5490-12-21T13:31:51.111111+0800', + '37179-09-17T15:18:31.111111+0800'] + res_list_ns = ['1970-01-01T08:00:00.000000001+0800', '1970-01-01T08:00:00.000000011+0800', + '1970-01-01T08:00:00.000000111+0800', '1970-01-01T08:00:00.000001111+0800', + '1970-01-01T08:00:00.000011111+0800', '1970-01-01T08:00:00.000111111+0800', + '1970-01-01T08:00:00.001111111+0800', '1970-01-01T08:00:00.011111111+0800', + '1970-01-01T08:00:00.111111111+0800', '1970-01-01T08:00:01.111111111+0800', + '1970-01-01T08:00:11.111111111+0800', '1970-01-01T08:01:51.111111111+0800', + '1970-01-01T08:18:31.111111111+0800', '1970-01-01T11:05:11.111111111+0800', + '1970-01-02T14:51:51.111111111+0800', '1970-01-14T04:38:31.111111111+0800', + '1970-05-09T22:25:11.111111111+0800', '1973-07-10T08:11:51.111111111+0800', + '2005-03-18T09:58:31.111111111+0800'] + # test to_iso8601's precision with default precision 'ms' + for i in range(len(ts_list)): + tdSql.query(f'select to_iso8601({ts_list[i]})') + tdSql.checkEqual(tdSql.queryResult[0][0],res_list_ms[i]) + # test to_iso8601's precision with table's precision 'us' + for i in range(len(ts_list)): + tdSql.query(f'select to_iso8601({ts_list[i]}) from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],res_list_us[i]) + + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname} precision "ns"') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create table if not exists {self.ntbname}(ts timestamp, c1 int, c2 timestamp)') + tdSql.execute(f'insert into {self.ntbname} values(now,1,today())') + # test to_iso8601's precision with table's precision 'ns' + for i in range(len(ts_list)): + tdSql.query(f'select to_iso8601({ts_list[i]}) from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],res_list_ns[i]) def check_customize_param_ms(self): time_zone = time.strftime('%z') tdSql.execute(f'drop database if exists {self.dbname}') @@ -65,7 +120,7 @@ class TDTestCase: tdSql.checkRows(1) for i in range(0,3): tdSql.query("select to_iso8601(1) from db.ntb") - tdSql.checkData(i,0,"1970-01-01T08:00:01+0800") + tdSql.checkData(i,0,"1970-01-01T08:00:00.001+0800") tdSql.checkRows(3) tdSql.query("select to_iso8601(ts) from db.ntb") tdSql.checkRows(3) @@ -97,7 +152,7 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select to_iso8601(1) from db.stb") for i in range(0,3): - tdSql.checkData(i,0,"1970-01-01T08:00:01+0800") + tdSql.checkData(i,0,"1970-01-01T08:00:00.001+0800") tdSql.checkRows(3) tdSql.query("select to_iso8601(ts) from db.stb") tdSql.checkRows(3) @@ -113,6 +168,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method self.check_base_function() self.check_customize_param_ms() + self.check_timestamp_precision() def stop(self): tdSql.close() From 4601583361d0b8c5c155299451f317fcc46d45d7 Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 25 Jun 2024 16:50:09 +0800 Subject: [PATCH 2/4] fix:[TD-30730] Modify precision rules for input parameters of function timetruncate and add test. --- source/libs/scalar/src/sclfunc.c | 191 +--------------------- tests/system-test/2-query/timetruncate.py | 36 +++- 2 files changed, 42 insertions(+), 185 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 6d87ae9baa..1a9d7f4900 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1343,9 +1343,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); - int64_t factor = TSDB_TICK_PER_SECOND(timePrec); - int64_t unit = timeUnit * 1000 / factor; - for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataSetNULL(pOutput->columnData, i); @@ -1355,201 +1352,27 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; } - // If converted value is less than 10digits in second, use value in second instead - int64_t timeValSec = timeVal / 1000000000; - if (timeValSec < 1000000000) { - timeVal = timeValSec; - } } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */ GET_TYPED_DATA(timeVal, int64_t, type, input); } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/ GET_TYPED_DATA(timeVal, int64_t, type, input); - int64_t timeValSec = timeVal / factor; - if (timeValSec < 1000000000) { - timeVal = timeValSec; - } } char buf[20] = {0}; NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal, sizeof(buf), buf); - int32_t tsDigits = (int32_t)strlen(buf); - switch (unit) { - case 0: { /* 1u or 1b */ - if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - if (timePrec == TSDB_TIME_PRECISION_NANO && timeUnit == 1) { - timeVal = timeVal * 1; - } else { - timeVal = timeVal / 1000 * 1000; - } - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor; - } else { - timeVal = timeVal * 1; - } - break; - } - case 1: { /* 1a */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal * 1; - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000 * 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000 * 1000000; - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor; - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - case 1000: { /* 1s */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000 * 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000000 * 1000000; - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000000 * 1000000000; - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor; - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - case 60000: { /* 1m */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000 / 60 * 60 * 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000000 / 60 * 60 * 1000000; - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000000 / 60 * 60 * 1000000000; - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor / factor / 60 * 60 * factor; - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - case 3600000: { /* 1h */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000 / 3600 * 3600 * 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000000 / 3600 * 3600 * 1000000; - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000000 / 3600 * 3600 * 1000000000; - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor / factor / 3600 * 3600 * factor; - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - case 86400000: { /* 1d */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)86400) * 1000); - } else { - timeVal = timeVal / 1000 / 86400 * 86400 * 1000; - } - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)86400) * 1000000); - } else { - timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; - } - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)86400) * 1000000000); - } else { - timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; - } - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - if (ignoreTz) { - timeVal = (timeVal - (timeVal + offsetFromTz(timezone, 1)) % (86400L)) * factor; - } else { - timeVal = timeVal * factor / factor / 86400 * 86400 * factor; - } - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - case 604800000: { /* 1w */ - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)604800) * 1000); - } else { - timeVal = timeVal / 1000 / 604800 * 604800 * 1000; - } - } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)604800) * 1000000); - } else { - timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000; - } - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)604800) * 1000000000); - } else { - timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000; - } - } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1)) % (((int64_t)604800L) * factor); - } else { - timeVal = timeVal * factor / factor / 604800 * 604800 * factor; - } - } else { - colDataSetNULL(pOutput->columnData, i); - continue; - } - break; - } - default: { - timeVal = timeVal * 1; - break; - } + // truncate the timestamp to time_unit precision + int64_t seconds = timeUnit / TSDB_TICK_PER_SECOND(timePrec); + if (ignoreTz && (seconds == 604800 || seconds == 86400)) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, TSDB_TICK_PER_SECOND(timePrec))) % (((int64_t)seconds) * TSDB_TICK_PER_SECOND(timePrec));; + } else { + timeVal = timeVal / timeUnit * timeUnit; } - - // truncate the timestamp to db precision - switch (timePrec) { - case TSDB_TIME_PRECISION_MILLI: { - if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000; - } - break; - } - case TSDB_TIME_PRECISION_MICRO: { - if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal * 1000; - } - break; - } - case TSDB_TIME_PRECISION_NANO: { - if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal * 1000; - } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal * 1000000; - } - break; - } - } - colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false); } diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 09bdfcef63..67a71cde17 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -22,6 +22,7 @@ class TDTestCase: '2020-4-1 00:00:00.001002', '2020-5-1 00:00:00.001002001' ] + self.unix_ts = ['1','1111','1111111','1111111111','1111111111111'] self.db_param_precision = ['ms','us','ns'] self.time_unit = ['1w','1d','1h','1m','1s','1a','1u','1b'] self.error_unit = ['2w','2d','2h','2m','2s','2a','2u','1c','#1'] @@ -134,7 +135,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 ) else: # assuming the client timezone is UTC+0800 - tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000))) + tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000000) % (86400 * 1000000000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': @@ -167,16 +168,49 @@ class TDTestCase: self.check_tb_type(unit,tb_type,ignore_tz) tdSql.checkRows(len(self.ts_str)) self.check_ms_timestamp(unit,date_time,ignore_tz) + for uts in self.unix_ts: + ans_time = [] + if tb_type.lower() == 'ntb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ntbname}') + elif tb_type.lower() == 'ctb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ctbname}') + elif tb_type.lower() == 'stb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.stbname}') + for i in range(len(self.ts_str)): + ans_time.append(int(uts)) + self.check_ms_timestamp(unit, ans_time, ignore_tz) elif precision.lower() == 'us': for ignore_tz in tz_options: self.check_tb_type(unit,tb_type,ignore_tz) tdSql.checkRows(len(self.ts_str)) self.check_us_timestamp(unit,date_time,ignore_tz) + for uts in self.unix_ts: + ans_time = [] + if tb_type.lower() == 'ntb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ntbname}') + elif tb_type.lower() == 'ctb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ctbname}') + elif tb_type.lower() == 'stb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.stbname}') + for i in range(len(self.ts_str)): + ans_time.append(int(uts)) + self.check_us_timestamp(unit, ans_time, ignore_tz) elif precision.lower() == 'ns': for ignore_tz in tz_options: self.check_tb_type(unit,tb_type, ignore_tz) tdSql.checkRows(len(self.ts_str)) self.check_ns_timestamp(unit,date_time,ignore_tz) + for uts in self.unix_ts: + ans_time = [] + if tb_type.lower() == 'ntb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ntbname}') + elif tb_type.lower() == 'ctb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.ctbname}') + elif tb_type.lower() == 'stb': + tdSql.query(f'select timetruncate({uts},{unit},{ignore_tz}) from {self.stbname}') + for i in range(len(self.ts_str)): + ans_time.append(int(uts)) + self.check_ns_timestamp(unit, ans_time, ignore_tz) for unit in self.error_unit: if tb_type.lower() == 'ntb': tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}') From 5dda4dba6f7a6a0f4708d82731c2abb2f6311ee4 Mon Sep 17 00:00:00 2001 From: sima Date: Wed, 26 Jun 2024 09:43:49 +0800 Subject: [PATCH 3/4] fix:[TD-30730] Change doc's description about functions. --- docs/en/12-taos-sql/10-function.md | 6 +++--- docs/zh/12-taos-sql/10-function.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 27dbbfcc08..4bd701f713 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -398,7 +398,7 @@ Conversion functions change the data type of a value. CAST(expr AS type_name) ``` -**Description**: Convert the input data `expr` into the type specified by `type_name`. This function can be used only in SELECT statements. +**Description**: Convert the input data `expr` into the type specified by `type_name`. **Return value type**: The type specified by parameter `type_name` @@ -435,8 +435,7 @@ TO_ISO8601(expr [, timezone]) **More explanations**: - You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]. For example, TO_ISO8601(1, "+00:00"). -- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp -- If the input is a column of TIMESTAMP type, the precision of the returned value is same as the precision set for the current data base in use +- The precision of the input timestamp will be recognized automatically according to the precision of the table used, milliseconds will be used if no table is specified. #### TO_JSON @@ -650,6 +649,7 @@ use_current_timezone: { - Time unit specified by `time_unit` can be: 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) - The precision of the returned timestamp is same as the precision set for the current data base in use +- The precision of the input timestamp will be recognized automatically according to the precision of the table used, milliseconds will be used if no table is specified. - If the input data is not formatted as a timestamp, the returned value is null. - When using 1d/1w as the time unit to truncate timestamp, you can specify whether to truncate based on the current time zone by setting the use_current_timezone parameter. Value 0 indicates truncation using the UTC time zone, value 1 indicates truncation using the current time zone. diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 6f4f9b3d84..26996a39fd 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -398,7 +398,7 @@ UPPER(expr) CAST(expr AS type_name) ``` -**功能说明**：数据类型转换函数，返回 expr 转换为 type_name 指定的类型后的结果。只适用于 select 子句中。 +**功能说明**：数据类型转换函数，返回 expr 转换为 type_name 指定的类型后的结果。 **返回结果类型**：CAST 中指定的类型（type_name)。 @@ -435,8 +435,7 @@ TO_ISO8601(expr [, timezone]) **使用说明**： - timezone 参数允许输入的时区格式为: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 -- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定; -- 如果输入是 TIMESTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. #### TO_JSON @@ -650,6 +649,7 @@ use_current_timezone: { - 支持的时间单位 time_unit 如下： 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. - 输入包含不符合时间日期格式的字符串则返回 NULL。 - 当使用 1d/1w 作为时间单位对时间戳进行截断时， 可通过设置 use_current_timezone 参数指定是否根据当前时区进行截断处理。 值 0 表示使用 UTC 时区进行截断，值 1 表示使用当前时区进行截断。 From 96e32227263f8f401f64972b2b2c12f4176fd412 Mon Sep 17 00:00:00 2001 From: sima Date: Wed, 26 Jun 2024 15:34:50 +0800 Subject: [PATCH 4/4] fix:[TD-30730] fix mergejoin operator's timetruncate caculation on timezone. --- source/libs/executor/src/hashjoinoperator.c | 2 +- source/libs/executor/src/mergejoinoperator.c | 2 +- source/libs/scalar/src/sclfunc.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 31d3385676..2fe2ccc56f 100755 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -116,7 +116,7 @@ int32_t hJoinLaunchPrimExpr(SSDataBlock* pBlock, SHJoinTableCtx* pTable, int32_t SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId); if (0 != pCtx->timezoneUnit) { for (int32_t i = startIdx; i <= endIdx; ++i) { - ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] / pCtx->truncateUnit * pCtx->truncateUnit - pCtx->timezoneUnit; + ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] - pCtx->timezoneUnit) % pCtx->truncateUnit; } } else { for (int32_t i = startIdx; i <= endIdx; ++i) { diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index faaebc1cd8..2e2101231b 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -948,7 +948,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) { SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId); if (0 != pCtx->timezoneUnit) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { - ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] / pCtx->truncateUnit * pCtx->truncateUnit - pCtx->timezoneUnit; + ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit; } } else { for (int32_t i = 0; i < pBlock->info.rows; ++i) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1a9d7f4900..282e935dd8 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1369,7 +1369,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara // truncate the timestamp to time_unit precision int64_t seconds = timeUnit / TSDB_TICK_PER_SECOND(timePrec); if (ignoreTz && (seconds == 604800 || seconds == 86400)) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, TSDB_TICK_PER_SECOND(timePrec))) % (((int64_t)seconds) * TSDB_TICK_PER_SECOND(timePrec));; + timeVal = timeVal - (timeVal + offsetFromTz(timezone, TSDB_TICK_PER_SECOND(timePrec))) % timeUnit; } else { timeVal = timeVal / timeUnit * timeUnit; }