xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix[query]: enable filter on nchar type.

This commit is contained in:
Haojun Liao 2022-04-11 17:03:41 +08:00
parent 6b681d79d4
commit 8240efda4c
3 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
source/libs/scalar/src/filter.c
View File

@ -1748,7 +1748,6 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
SFilterField* fi = right;
SValueNode* var = (SValueNode *)fi->desc;
if (var == NULL) {
assert(fi->data != NULL);
continue;
@ -1767,13 +1766,18 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
}
SDataType *dType = &var->node.resType;
size_t bytes = 0;
if (type == TSDB_DATA_TYPE_BINARY) {
size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE;
fi->data = taosMemoryCalloc(1, len + 1 + VARSTR_HEADER_SIZE);
bytes = len + 1 + VARSTR_HEADER_SIZE;
fi->data = taosMemoryCalloc(1, bytes);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE;
fi->data = taosMemoryCalloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE;
bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
fi->data = taosMemoryCalloc(1, bytes);
} else if (type != TSDB_DATA_TYPE_JSON){
if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
/*
@ -1797,8 +1801,11 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
} else {
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
out.columnData->info.type = type;
out.columnData->info.bytes = tDataTypes[type].bytes;
ASSERT(!IS_VAR_DATA_TYPE(type));
if (IS_VAR_DATA_TYPE(type)) {
out.columnData->info.bytes = bytes;
} else {
out.columnData->info.bytes = tDataTypes[type].bytes;
}
// todo refactor the convert
int32_t code = doConvertDataType(var, &out);
@ -2985,13 +2992,13 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)) {
if (colData == NULL || colDataIsNull_s((SColumnInfoData *)info->cunits[uidx].colData, i)) {
(*p)[i] = 0;
all = false;
continue;
}
// match/nmatch for nchar type need convert from ucs4 to mbs
// match/nmatch for nchar type need convert from ucs4 to mbs
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){
char *newColData = taosMemoryCalloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
@ -3222,19 +3229,13 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
info->unitFlags = taosMemoryMalloc(info->unitNum * sizeof(*info->unitFlags));
filterDumpInfoToString(info, "Final", 0);
return code;
_return:
qInfo("init from node failed, code:%d", code);
return code;
}
bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) {
if (FILTER_EMPTY_RES(info)) {
return false;

1
source/libs/scalar/src/scalar.c
View File

@ -46,7 +46,6 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) {
colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
colInfoDataEnsureCapacity(out->columnData, 1);
int32_t code = vectorConvertImpl(&in, out);
sclFreeParam(&in);

48
source/libs/scalar/src/sclvector.c
View File

@ -177,10 +177,25 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v);
}
static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
int32_t len = 0;
int32_t inputLen = varDataLen(buf);
char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
/*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
varDataSetLen(t, len);
colDataAppend(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t);
}
//TODO opt performance
int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
int32_t bufSize = pIn->columnData->info.bytes;
char *tmp = taosMemoryMalloc(bufSize);
bool vton = false;
_bufConverteFunc func = NULL;
if (TSDB_DATA_TYPE_BOOL == outType) {
func = varToBool;
@ -190,6 +205,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
func = varToUnsigned;
} else if (IS_FLOAT_TYPE(outType)) {
func = varToFloat;
} else if (outType == TSDB_DATA_TYPE_NCHAR) {
func = varToNchar;
vton = true;
} else {
sclError("invalid convert outType:%d", outType);
return TSDB_CODE_QRY_APP_ERROR;
@ -197,26 +215,30 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
pOut->numOfRows = pIn->numOfRows;
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
if (colDataIsNull(pIn->columnData, pIn->numOfRows, i, NULL)) {
if (colDataIsNull_s(pIn->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
char* data = colDataGetData(pIn->columnData, i);
if (TSDB_DATA_TYPE_BINARY == inType) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
if (vton) {
memcpy(tmp, data, varDataTLen(data));
} else {
ASSERT (varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(data), varDataLen(data), tmp);
if (len < 0){
sclError("castConvert taosUcs4ToMbs error 1");
taosMemoryFreeClear(tmp);
return TSDB_CODE_QRY_APP_ERROR;
if (TSDB_DATA_TYPE_VARCHAR == inType) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
} else {
ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
taosMemoryFreeClear(tmp);
return TSDB_CODE_QRY_APP_ERROR;
}
tmp[len] = 0;
}
tmp[len] = 0;
}
(*func)(tmp, pOut, i);