xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(query): fix time function related bugs (#11604) 

* fix(query): fix timediff args cannot be integer values

TD-14798

* fix(query): fix timediff/timetruncate/to_unixtimestamp handling NULL value and empty string

TD-14802
TD-14803

* refactor(query): constant NULL value processing logic

TD-14802
TD-14803

* refactor(query): support timefunction arithmetic operation with boolean
constant
This commit is contained in:
Ganlin Zhao 2022-04-18 19:20:08 +08:00 committed by GitHub
parent 59008ec626
commit 00ae854c78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 72 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
source/common/src/tdatablock.c
View File

@ -1100,6 +1100,11 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
pColumn->nullbitmap = tmp;
memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
if (pColumn->info.type == TSDB_DATA_TYPE_NULL) {
return TSDB_CODE_SUCCESS;
}
assert(pColumn->info.bytes);
tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes);
if (tmp == NULL) {

12
source/common/src/ttime.c
View File

@ -415,7 +415,11 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
if (type == TSDB_DATA_TYPE_BINARY) {
newColData = taosMemoryCalloc(1, charLen + 1);
memcpy(newColData, varDataVal(inputData), charLen);
taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return ret;
}
taosMemoryFree(newColData);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
newColData = taosMemoryCalloc(1, charLen / TSDB_NCHAR_SIZE + 1);
@ -425,7 +429,11 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
return TSDB_CODE_FAILED;
}
newColData[len] = 0;
taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return ret;
}
taosMemoryFree(newColData);
} else {
return TSDB_CODE_FAILED;

15
source/libs/function/src/builtins.c
View File

@ -330,7 +330,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
}
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_VAR_DATA_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
if (!IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@ -358,7 +358,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
if ((!IS_VAR_DATA_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || !IS_INTEGER_TYPE(para2Type)) {
if ((!IS_VAR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || !IS_INTEGER_TYPE(para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@ -372,12 +372,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
if ((!IS_VAR_DATA_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) ||
(!IS_VAR_DATA_TYPE(para2Type) && TSDB_DATA_TYPE_TIMESTAMP != para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
for (int32_t i = 0; i < 2; ++i) {
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
if (!IS_VAR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
}
if (3 == paraNum) {
if (!IS_INTEGER_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);

6
source/libs/parser/src/parTranslater.c
View File

@ -492,8 +492,10 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BIGINT == rdt.type) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BIGINT == ldt.type)) {
if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) ||
(TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type) ) {
pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
} else {

5
source/libs/scalar/inc/sclvector.h
View File

@ -52,6 +52,9 @@ static FORCE_INLINE double getVectorDoubleValue_FLOAT(void *src, int32_t index)
static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index) {
return (double)*((double *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) {
return (double)*((bool *)src + index);
}
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL;
@ -77,6 +80,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
p = getVectorDoubleValue_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_BOOL) {
p = getVectorDoubleValue_BOOL;
} else {
assert(0);
}

27
source/libs/scalar/src/scalar.c
View File

@ -398,7 +398,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
SScalarParam *params = NULL;
int32_t rowNum = 0;
int32_t code = 0;
SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
@ -411,7 +411,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t paramNum = scalarGetOperatorParamNum(node->opType);
SScalarParam* pLeft = &params[0];
SScalarParam* pRight = paramNum > 1 ? &params[1] : NULL;
OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC);
_return:
@ -426,7 +426,7 @@ _return:
EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)*pNode;
SScalarParam output = {0};
ctx->code = sclExecFunction(node, ctx, &output);
if (ctx->code) {
return DEAL_RES_ERROR;
@ -440,16 +440,19 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
res->node.resType = node->node.resType;
int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
res->datum.p = output.columnData->pData;
output.columnData->pData = NULL;
if (colDataIsNull_s(output.columnData, 0)) {
res->node.resType.type = TSDB_DATA_TYPE_NULL;
} else {
memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes);
res->node.resType = node->node.resType;
int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) {
res->datum.p = output.columnData->pData;
output.columnData->pData = NULL;
} else {
memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes);
}
}
nodesDestroyNode(*pNode);
*pNode = (SNode*)res;
@ -469,7 +472,7 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) {
sclError("make value node failed");
sclFreeParam(&output);
sclFreeParam(&output);
ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}

51
source/libs/scalar/src/sclfunc.c
View File

@ -795,13 +795,6 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput);
if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP) {
return TSDB_CODE_FAILED;
}
if (inputNum != 1) {
return TSDB_CODE_FAILED;
}
char *input = pInput[0].columnData->pData;
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
@ -867,13 +860,6 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput);
int32_t timePrec = GET_PARAM_PRECISON(pInput);
if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
return TSDB_CODE_FAILED;
}
if (inputNum != 1) {
return TSDB_CODE_FAILED;
}
char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
@ -883,7 +869,11 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
}
int64_t timeVal = 0;
convertStringToTimestamp(type, input, timePrec, &timeVal);
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal);
if (ret != TSDB_CODE_SUCCESS) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
colDataAppend(pOutput->columnData, i, (char *)&timeVal, false);
input += varDataTLen(input);
@ -897,18 +887,6 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(&pInput[0]);
int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
if (inputNum != 2) {
return TSDB_CODE_FAILED;
}
if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP &&
type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
return TSDB_CODE_FAILED;
}
if (GET_PARAM_TYPE(&pInput[1]) != TSDB_DATA_TYPE_BIGINT) { //time_unit
return TSDB_CODE_FAILED;
}
int64_t timeUnit, timeVal = 0;
GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
@ -930,7 +908,11 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
}
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal);
int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal);
if (ret != TSDB_CODE_SUCCESS) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
//If converted value is less than 10digits in second, use value in second instead
int64_t timeValSec = timeVal / 1000000000;
if (timeValSec < 1000000000) {
@ -1096,16 +1078,9 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
}
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 2 && inputNum != 3) {
return TSDB_CODE_FAILED;
}
int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
int64_t timeUnit = -1, timeVal[2] = {0};
if (inputNum == 3) {
if (GET_PARAM_TYPE(&pInput[2]) != TSDB_DATA_TYPE_BIGINT) {
return TSDB_CODE_FAILED;
}
GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
}
@ -1133,7 +1108,11 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t type = GET_PARAM_TYPE(&pInput[k]);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]);
int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]);
if (ret != TSDB_CODE_SUCCESS) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
} else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/
GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]);
if (type == TSDB_DATA_TYPE_TIMESTAMP) {

12
source/libs/scalar/src/sclvector.c
View File

@ -57,6 +57,10 @@ int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) {
int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) {
return (int64_t)*((double *)src + index);
}
int64_t getVectorBigintValue_BOOL(void *src, int32_t index) {
return (int64_t)*((bool *)src + index);
}
_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
_getBigintValue_fn_t p = NULL;
if(srcType==TSDB_DATA_TYPE_TINYINT) {
@ -81,6 +85,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
p = getVectorBigintValue_DOUBLE;
}else if(srcType==TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorBigintValue_BIGINT;
}else if(srcType==TSDB_DATA_TYPE_BOOL) {
p = getVectorBigintValue_BOOL;
}else {
assert(0);
}
@ -620,8 +626,10 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert);
SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert);
if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) ||
(GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT)) { //timestamp plus duration
if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) ||
(GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) ||
(GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BOOL) ||
(GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { //timestamp plus duration
int64_t *output = (int64_t *)pOutputCol->pData;
_getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type);
_getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type);