From 4f629ec68bf46c38c6690a6ca680af8defb30b07 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 24 Jun 2022 16:53:56 +0800 Subject: [PATCH 1/5] enh(query): enhance cast function to support more types TD-15473 --- source/libs/function/src/builtins.c | 21 +++--- source/libs/scalar/src/sclfunc.c | 108 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f985cc324d..60107dee32 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1219,18 +1219,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } - if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } + //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} + //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7ea0457592..721e38980e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -730,6 +730,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp char *input = colDataGetData(pInput[0].columnData, i); switch(outputType) { + case TSDB_DATA_TYPE_TINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_INT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10); @@ -748,6 +802,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_UTINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10); From 4a7938e9fffe1c8466a7c84dd059ef0c9c2ee043 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:05:17 +0800 Subject: [PATCH 2/5] cast support nchar->binary, binary/nchar->timestamp --- source/libs/scalar/src/sclfunc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a3cd1aba6d..e8f48cd0fa 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -881,8 +881,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + //convert to 0 + *(int64_t *)output = 0; } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } @@ -897,8 +897,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); + memcpy(varDataVal(output), newBuf, len); + varDataSetLen(output, len); + taosMemoryFree(newBuf); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); From 3b9953f483a8ec3794144b8649c3ac63e7986765 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:15:17 +0800 Subject: [PATCH 3/5] cast function support ->float, ->double --- source/libs/scalar/src/sclfunc.c | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e8f48cd0fa..4caccfc7f3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -879,6 +879,42 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_FLOAT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(float *)output = taosStr2Float(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(float *)output = taosStr2Float(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(float *)output, float, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(double *)output = taosStr2Double(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(double *)output = taosStr2Double(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(double *)output, double, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From b139daf08bbcdd958e9e930413f1ea87fb691a41 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 13:32:14 +0800 Subject: [PATCH 4/5] add cast function support ->bool --- source/libs/scalar/src/sclfunc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4caccfc7f3..80bebafef2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -915,6 +915,24 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_BOOL: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(bool *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(bool *)output, bool, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From 52bfe2e74cac305bebb3adaf8a2a49dcedce1a6a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 14:19:17 +0800 Subject: [PATCH 5/5] modify test cases for cast.py --- tests/system-test/2-query/cast.py | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 387170991f..934bbbd7b4 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -630,27 +630,27 @@ class TDTestCase: ( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) ) tdLog.printNoPrefix("==========step40: error cast condition, should return error ") - tdSql.error("select cast(c1 as int) as b from ct4") - tdSql.error("select cast(c1 as bool) as b from ct4") - tdSql.error("select cast(c1 as tinyint) as b from ct4") - tdSql.error("select cast(c1 as smallint) as b from ct4") - tdSql.error("select cast(c1 as float) as b from ct4") - tdSql.error("select cast(c1 as double) as b from ct4") - tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") - tdSql.error("select cast(c1 as int unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int) as b from ct4") + #tdSql.error("select cast(c1 as bool) as b from ct4") + #tdSql.error("select cast(c1 as tinyint) as b from ct4") + #tdSql.error("select cast(c1 as smallint) as b from ct4") + #tdSql.error("select cast(c1 as float) as b from ct4") + #tdSql.error("select cast(c1 as double) as b from ct4") + #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int unsigned) as b from ct4") - tdSql.error("select cast(c2 as int) as b from ct4") - tdSql.error("select cast(c3 as bool) as b from ct4") - tdSql.error("select cast(c4 as tinyint) as b from ct4") - tdSql.error("select cast(c5 as smallint) as b from ct4") - tdSql.error("select cast(c6 as float) as b from ct4") - tdSql.error("select cast(c7 as double) as b from ct4") - tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c2 as int) as b from ct4") + #tdSql.error("select cast(c3 as bool) as b from ct4") + #tdSql.error("select cast(c4 as tinyint) as b from ct4") + #tdSql.error("select cast(c5 as smallint) as b from ct4") + #tdSql.error("select cast(c6 as float) as b from ct4") + #tdSql.error("select cast(c7 as double) as b from ct4") + #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c8 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + #tdSql.error("select cast(c8 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as binary(64) ) as b from ct4") pass def run(self):