xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:[TS-5761] error for in operator

This commit is contained in:
wangmm0220 2024-12-19 18:23:34 +08:00
parent ed523c9c13
commit fc14fe4455
7 changed files with 92 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/libs/function/function.h
View File

@ -288,7 +288,7 @@ struct SScalarParam {
bool colAlloced;
SColumnInfoData *columnData;
SHashObj *pHashFilter;
SHashObj *pHashFilterVar;
SHashObj *pHashFilterOthers;
int32_t hashValueType;
void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value
int32_t numOfRows;

31
source/libs/scalar/src/filter.c
View File

@ -1298,7 +1298,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree,
if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) {
SNodeListNode *listNode = (SNodeListNode *)node->pRight;
SListCell *cell = listNode->pNodeList->pHead;
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
if (out.columnData == NULL) {
@ -1308,8 +1307,9 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree,
out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; // reserved space for simple_copy
int32_t overflowCount = 0;
for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
SNode* nodeItem = NULL;
FOREACH(nodeItem, listNode->pNodeList) {
SValueNode *valueNode = (SValueNode *)nodeItem;
if (valueNode->node.resType.type != type) {
int32_t overflow = 0;
code = sclConvertValueToSclParam(valueNode, &out, &overflow);
@ -1319,7 +1319,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree,
}
if (overflow) {
cell = cell->pNext;
++overflowCount;
continue;
}
@ -1358,8 +1357,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree,
code = terrno;
break;
}
cell = cell->pNext;
}
if(overflowCount == listNode->pNodeList->length) {
ctx->ignore = true;
@ -4863,19 +4860,16 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
if (LIST_LENGTH(listNode->pNodeList) > 10 || OP_TYPE_NOT_IN == node->opType) {
stat->scalarMode = true;
}
int32_t type = -1;
int32_t type = refNode->node.resType.type;
exprNode = &listNode->node;
SListCell *cell = listNode->pNodeList->pHead;
for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
cell = cell->pNext;
int32_t tmp = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type);
if (tmp != 0 && tmp != refNode->node.resType.type){
SNode* nodeItem = NULL;
FOREACH(nodeItem, listNode->pNodeList) {
SValueNode *valueNode = (SValueNode *)nodeItem;
int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type);
if (tmp != 0){
stat->scalarMode = true;
if (IS_NUMERIC_TYPE(tmp) && tmp > type){
type = tmp;
}
}
}
if (IS_NUMERIC_TYPE(type)){
@ -5031,11 +5025,11 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) {
}
case OP_TYPE_IN: {
SNodeListNode *listNode = (SNodeListNode *)oper->valNode;
SListCell *cell = listNode->pNodeList->pHead;
SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType};
SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType};
for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
SNode* nodeItem = NULL;
FOREACH(nodeItem, listNode->pNodeList) {
SValueNode *valueNode = (SValueNode *)nodeItem;
SFltSclDatum valDatum;
FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode));
if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) {
@ -5045,7 +5039,6 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) {
minDatum.i = TMIN(minDatum.i, valDatum.i);
maxDatum.i = TMAX(maxDatum.i, valDatum.i);
}
cell = cell->pNext;
}
SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum};
SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum};

53
source/libs/scalar/src/scalar.c
View File

@ -116,7 +116,7 @@ _return:
SCL_RET(code);
}
// processType = 0 means all type. 1 means number, 2 means var
// processType = 0 means all type. 1 means number, 2 means var, 3 means float, 4 means var&integer
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType) {
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
if (NULL == pObj) {
@ -128,7 +128,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_
int32_t code = 0;
SNodeListNode *nodeList = (SNodeListNode *)pNode;
SListCell *cell = nodeList->pNodeList->pHead;
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
if (out.columnData == NULL) {
SCL_ERR_JRET(terrno);
@ -136,21 +135,14 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_
int32_t len = 0;
void *buf = NULL;
for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
if (IS_VAR_DATA_TYPE(valueNode->node.resType.type)){
if (processType == 1) {
cell = cell->pNext;
SNode* nodeItem = NULL;
FOREACH(nodeItem, nodeList->pNodeList) {
SValueNode *valueNode = (SValueNode *)nodeItem;
if ((IS_VAR_DATA_TYPE(valueNode->node.resType.type) && (processType == 1 || processType == 3)) ||
(IS_INTEGER_TYPE(valueNode->node.resType.type) && (processType == 2 || processType == 3)) ||
(IS_FLOAT_TYPE(valueNode->node.resType.type) && (processType == 2 || processType == 4))) {
continue;
}
} else{
if (processType == 2)
{
cell = cell->pNext;
continue;
}
}
if (valueNode->node.resType.type != type) {
out.columnData->info.type = type;
@ -172,7 +164,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_
}
if (overflow) {
cell = cell->pNext;
continue;
}
@ -198,7 +189,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_
}
colInfoDataCleanup(out.columnData, out.numOfRows);
cell = cell->pNext;
}
*data = pObj;
@ -245,9 +235,9 @@ void sclFreeParam(SScalarParam *param) {
param->pHashFilter = NULL;
}
if (param->pHashFilterVar != NULL) {
taosHashCleanup(param->pHashFilterVar);
param->pHashFilterVar = NULL;
if (param->pHashFilterOthers != NULL) {
taosHashCleanup(param->pHashFilterOthers);
param->pHashFilterOthers = NULL;
}
}
@ -388,15 +378,15 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
SCL_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t type = -1;
SListCell *cell = nodeList->pNodeList->pHead;
for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
cell = cell->pNext;
int32_t tmp = vectorGetConvertType(ctx->type.selfType, valueNode->node.resType.type);
if (tmp != 0 && IS_NUMERIC_TYPE(tmp) && tmp > type){
int32_t type = ctx->type.selfType;
SNode* nodeItem = NULL;
FOREACH(nodeItem, nodeList->pNodeList) {
SValueNode *valueNode = (SValueNode *)nodeItem;
int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type);
if (tmp != 0){
type = tmp;
}
}
if (IS_NUMERIC_TYPE(type)){
ctx->type.peerType = type;
@ -404,7 +394,10 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
type = ctx->type.peerType;
if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(type)){
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type, 1));
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilterVar, node, ctx->type.selfType, 2));
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilterOthers, node, ctx->type.selfType, 2));
} else if (IS_INTEGER_TYPE(ctx->type.selfType) && IS_FLOAT_TYPE(type)){
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type, 3));
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilterOthers, node, ctx->type.selfType, 4));
} else {
SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type, 0));
}
@ -414,8 +407,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
taosHashCleanup(param->pHashFilter);
param->pHashFilter = NULL;
taosHashCleanup(param->pHashFilterVar);
param->pHashFilterVar = NULL;
taosHashCleanup(param->pHashFilterOthers);
param->pHashFilterOthers = NULL;
sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
return terrno;
}

23
source/libs/scalar/src/sclvector.c
View File

@ -1011,12 +1011,12 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/*NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1,
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 5, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 5, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 5, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 6, 6, 6, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1,
@ -1030,7 +1030,8 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0};
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0
};
int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
@ -1071,7 +1072,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex,
int32_t numOfRows) {
if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterVar != NULL)){
if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterOthers != NULL)){
return TSDB_CODE_SUCCESS;
}
output->numOfRows = input->numOfRows;
@ -2008,7 +2009,7 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara
char *pLeftData = colDataGetData(pLeft->columnData, i);
bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter);
if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterVar) > 0){
if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterOthers) > 0){
do{
if (optr == OP_TYPE_IN && res){
break;
@ -2017,7 +2018,7 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara
break;
}
pLeftData = colDataGetData(pLeftVar->columnData, i);
res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterVar);
res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterOthers);
}while(0);
}
colDataSetInt8(pOut->columnData, i, (int8_t *)&res);
@ -2048,7 +2049,7 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
SCL_ERR_JRET(vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows));
param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft;
param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight;
if (pRight->pHashFilterVar != NULL){
if (pRight->pHashFilterOthers != NULL){
param3 = pLeft;
}
}

8
source/util/src/thashutil.c
View File

@ -152,8 +152,8 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) {
if (FLT_EQUAL(f, 0.0)) {
return 0;
}
if (fabs(f) < INT32_MAX) {
int32_t t = (int32_t)(floor(f));
if (fabs(f) < FLT_MAX / BASE - DLT) {
int32_t t = (int32_t)(round(BASE * (f + DLT)));
return (uint32_t)t;
} else {
return 0x7fc00000;
@ -168,8 +168,8 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) {
if (FLT_EQUAL(f, 0.0)) {
return 0;
}
if (fabs(f) < INT32_MAX) {
int32_t t = (int32_t)(floor(f));
if (fabs(f) < DBL_MAX / BASE - DLT) {
int32_t t = (int32_t)(round(BASE * (f + DLT)));
return (uint32_t)t;
} else {
return 0x7fc00000;

11
tests/system-test/2-query/ts-5761-scalemode.py
View File

@ -33,7 +33,18 @@ class TDTestCase:
tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')")
tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')")
tdSql.execute("CREATE TABLE stt( time TIMESTAMP, c1 BIGINT, c2 timestamp, c3 int, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))")
tdSql.execute("create table tt1 using stt tags('1', '1.7')")
# create index for all tags
tdSql.execute("INSERT INTO tt1 VALUES (1641024000000, 9223372036854775807, 1641024000000, 1, 1, 1, '1', '1.7')")
def check(self):
tdSql.query(f"SELECT * FROM tt1 WHERE c1 in (1.7, 9223372036854775803, '')")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM tt1 WHERE c1 = 9223372036854775803")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)")

11
tests/system-test/2-query/ts-5761.py
View File

@ -32,7 +32,18 @@ class TDTestCase:
tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')")
tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')")
tdSql.execute("CREATE TABLE stt( time TIMESTAMP, c1 BIGINT, c2 timestamp, c3 int, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))")
tdSql.execute("create table tt1 using stt tags('1', '1.7')")
# create index for all tags
tdSql.execute("INSERT INTO tt1 VALUES (1641024000000, 9223372036854775807, 1641024000000, 1, 1, 1, '1', '1.7')")
def check(self):
tdSql.query(f"SELECT * FROM tt1 WHERE c1 in (1.7, 9223372036854775803, '')")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM tt1 WHERE c1 = 9223372036854775803")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7")
tdSql.checkRows(0)
tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)")