xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(query): opt filter perf.

This commit is contained in:
Haojun Liao 2022-12-27 16:55:38 +08:00
parent 58b5d2e0aa
commit 1e69538c37
4 changed files with 56 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
source/libs/scalar/inc/sclInt.h
View File

@ -50,9 +50,6 @@ typedef struct SScalarCtx {
#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN)
#define SCL_DOWNGRADE_DATETYPE(_type) \
((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT)
#define SCL_NO_NEED_CONVERT_COMPARISION(_ltype, _rtype, _optr) \
(IS_NUMERIC_TYPE(_ltype) && IS_NUMERIC_TYPE(_rtype) && \
((_optr) >= OP_TYPE_GREATER_THAN && (_optr) <= OP_TYPE_NOT_EQUAL))
#define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__)

2
source/libs/scalar/src/filter.c
View File

@ -4082,7 +4082,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC
SArray *pList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pList, &pSrc);
int32_t code = scalarCalculate(info->sclCtx.node, pList, &output);
code = scalarCalculate(info->sclCtx.node, pList, &output);
taosArrayDestroy(pList);
FLT_ERR_RET(code);

67
source/libs/scalar/src/sclvector.c
View File

@ -37,6 +37,11 @@
#define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) {
return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) &&
(optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL);
}
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) {
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
@ -338,6 +343,7 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v);
}
// todo remove this malloc
static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
int32_t len = 0;
int32_t inputLen = varDataLen(buf);
@ -399,6 +405,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
pCtx->pOut->numOfRows = pCtx->pIn->numOfRows;
char* tmp = NULL;
for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) {
if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) {
colDataAppendNULL(pCtx->pOut->columnData, i);
@ -421,12 +429,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
continue;
}
}
int32_t bufSize = pCtx->pIn->columnData->info.bytes;
char *tmp = taosMemoryMalloc(varDataTLen(data));
if (!tmp) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
if (tmp == NULL) {
tmp = taosMemoryMalloc(bufSize);
if (tmp == NULL) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (vton) {
memcpy(tmp, data, varDataTLen(data));
} else {
@ -434,6 +446,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
} else if (TSDB_DATA_TYPE_NCHAR == convertType) {
// we need to convert it to native char string, and then perform the string to numeric data
ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@ -448,9 +461,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
(*func)(tmp, pCtx->pOut, i, overflow);
taosMemoryFreeClear(tmp);
}
if (tmp != NULL) {
taosMemoryFreeClear(tmp);
}
return TSDB_CODE_SUCCESS;
}
@ -925,25 +940,43 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
return TSDB_CODE_SUCCESS;
}
int8_t type = 0;
int32_t code = 0;
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param2 = NULL, *paramOut2 = NULL;
int32_t code = 0;
if (leftType < rightType) {
// always convert least data
if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) &&
leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
int8_t type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
if (pLeft->numOfRows > pRight->numOfRows) {
type = leftType;
} else {
type = rightType;
}
} else {
// we only define half value in the convert-matrix, so make sure param1 always less equal than param2
if (leftType < rightType) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
}
}
if (type != GET_PARAM_TYPE(param1)) {
@ -1683,7 +1716,7 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *
SScalarParam *param1 = NULL;
SScalarParam *param2 = NULL;
if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
param1 = pLeft;
param2 = pRight;
} else {

10
source/util/src/tcompare.c
View File

@ -1071,7 +1071,7 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
TdUcs4 c, c1;
TdUcs4 matchOne = L'_'; // "_"
TdUcs4 matchAll = L'%'; // "%"
@ -1079,10 +1079,10 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
int32_t i = 0;
int32_t j = 0;
while ((c = patterStr[i++]) != 0) {
while ((c = pattern[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
while ((c = pattern[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
}
@ -1100,7 +1100,7 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size,
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
int32_t ret = WCSPatternMatch(&pattern[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
@ -1198,7 +1198,7 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) {
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));