xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: add unit test for json compare

This commit is contained in:
wangmm0220 2022-04-20 16:52:32 +08:00
parent 2133a2de1d
commit a016f1157a
3 changed files with 358 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
include/common/tdatablock.h
View File

@ -57,11 +57,30 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
#define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) #define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1)
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \
: colDataGetNumData(p1_, r_))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
if(colDataIsNull_var(pColumnInfoData, row)){
return true;
}
char *data = colDataGetVarData(pColumnInfoData, row);
return (*data == TSDB_DATA_TYPE_NULL);
}
if (!pColumnInfoData->hasNull) { if (!pColumnInfoData->hasNull) {
return false; return false;
} }
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
return colDataIsNull_var(pColumnInfoData, row); return colDataIsNull_var(pColumnInfoData, row);
} else { } else {
if (pColumnInfoData->nullbitmap == NULL) { if (pColumnInfoData->nullbitmap == NULL) {
@ -99,16 +118,6 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
} }
} }
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \
: colDataGetNumData(p1_, r_))
static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
// There is a placehold for each NULL value of binary or nchar type. // There is a placehold for each NULL value of binary or nchar type.
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@ -271,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
#endif #endif
#endif /*_TD_COMMON_EP_H_*/ #endif /*_TD_COMMON_EP_H_*/

335
source/libs/scalar/src/sclvector.c
View File

@ -28,6 +28,75 @@
#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_INT: {
GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UINT: {
GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
GET_TYPED_DATA(*((float *)outData), float, inType, inData);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(*((double *)outData), double, inType, inData);
break;
}
default:{
ASSERT(0);
}
}
}
void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
char *tmp = taosMemoryMalloc(varDataTLen(inData));
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
}
tmp[len] = 0;
ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
double value = strtod(tmp, NULL);
*((double *)outData) = value;
taosMemoryFreeClear(tmp);
}
typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@ -64,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) {
return (int64_t)*((bool *)src + index); return (int64_t)*((bool *)src + index);
} }
int64_t getVectorBigintValue_JSON(void *src, int32_t index){
ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0;
if (*data == TSDB_DATA_TYPE_NULL){
return 0;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return (int64_t)out;
}
_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
_getBigintValue_fn_t p = NULL; _getBigintValue_fn_t p = NULL;
if(srcType==TSDB_DATA_TYPE_TINYINT) { if(srcType==TSDB_DATA_TYPE_TINYINT) {
@ -90,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
p = getVectorBigintValue_BIGINT; p = getVectorBigintValue_BIGINT;
}else if(srcType==TSDB_DATA_TYPE_BOOL) { }else if(srcType==TSDB_DATA_TYPE_BOOL) {
p = getVectorBigintValue_BOOL; p = getVectorBigintValue_BOOL;
}else if(srcType==TSDB_DATA_TYPE_JSON) {
p = getVectorBigintValue_JSON;
}else { }else {
assert(0); assert(0);
} }
@ -200,75 +285,6 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn
taosMemoryFree(t); taosMemoryFree(t);
} }
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_INT: {
GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UINT: {
GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
GET_TYPED_DATA(*((float *)outData), float, inType, inData);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(*((double *)outData), double, inType, inData);
break;
}
default:{
ASSERT(0);
}
}
}
void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
char *tmp = taosMemoryMalloc(varDataTLen(inData));
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
}
tmp[len] = 0;
ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
double value = strtod(tmp, NULL);
*((double *)outData) = value;
taosMemoryFreeClear(tmp);
}
//TODO opt performance, tmp is not needed. //TODO opt performance, tmp is not needed.
int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
ASSERT(!IS_VAR_DATA_TYPE(outType)); ASSERT(!IS_VAR_DATA_TYPE(outType));
@ -345,12 +361,13 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
} }
double getVectorDoubleValue_JSON(void *src, int32_t index){ double getVectorDoubleValue_JSON(void *src, int32_t index){
ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
char *data = colDataGetVarData((SColumnInfoData*)src, index); char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0; double out = 0;
if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY if (*data == TSDB_DATA_TYPE_NULL){
return out;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} { } else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} }
return out; return out;
@ -569,24 +586,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
} }
int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */ /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 0, 0, 0, /*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 0, 7, 0, 0,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 0, 0, 0, /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 0, 0, 0, /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 0, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 0, 0, 0, /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 0, /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
}; };
@ -677,7 +694,7 @@ enum {
static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
SColumnInfoData* pCol = pParam->columnData; SColumnInfoData* pCol = pParam->columnData;
if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) { if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) {
pDest->numOfRows = pParam->numOfRows; pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@ -718,10 +735,6 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+ getVectorDoubleValueFnRight(RIGHT_COL, 0); + getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -735,12 +748,12 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -771,8 +784,11 @@ char *getJsonValue(char *json, char *key){ //todo
for (int i = 0; i < cols; ++i) { for (int i = 0; i < cols; ++i) {
SColIdx *pColIdx = kvRowColIdxAt(json, i); SColIdx *pColIdx = kvRowColIdxAt(json, i);
char *data = kvRowColVal(json, pColIdx); char *data = kvRowColVal(json, pColIdx);
if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ if(i == 0){
return NULL; if(*data == TSDB_DATA_TYPE_NULL) {
return NULL;
}
continue;
} }
if(memcmp(key, data, varDataTLen(data)) == 0){ if(memcmp(key, data, varDataTLen(data)) == 0){
return data + varDataTLen(data); return data + varDataTLen(data);
@ -798,7 +814,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
} }
char *pLeftData = colDataGetVarData(pLeft->columnData, i); char *pLeftData = colDataGetVarData(pLeft->columnData, i);
char *value = getJsonValue(pLeftData, pRightData); char *value = getJsonValue(pLeftData, pRightData);
if (!value || *value == TSDB_DATA_TYPE_NULL) { if (!value) {
colDataSetNull_var(pOutputCol, i); colDataSetNull_var(pOutputCol, i);
pOutputCol->hasNull = true; pOutputCol->hasNull = true;
continue; continue;
@ -835,14 +851,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
} }
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -861,15 +869,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -899,10 +898,6 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
*output = (getVectorDoubleValueFnLeft(LEFT_COL, i) *output = (getVectorDoubleValueFnLeft(LEFT_COL, i)
- getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor; - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor;
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -916,12 +911,12 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor; *output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor;
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -951,15 +946,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
} }
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -978,15 +964,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -1016,10 +993,6 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, 0); * getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -1047,15 +1020,6 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, i); * getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -1090,15 +1054,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/getVectorDoubleValueFnRight(RIGHT_COL, i); /getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
@ -1111,10 +1066,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
*output = getVectorDoubleValueFnLeft(LEFT_COL, 0) *output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
/ getVectorDoubleValueFnRight(RIGHT_COL, i); / getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = pRightCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows));
}
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
@ -1128,10 +1079,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/ getVectorDoubleValueFnRight(RIGHT_COL, 0); / getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
}
} }
} }
@ -1240,11 +1187,6 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
*output = - getVectorDoubleValueFnLeft(LEFT_COL, i); *output = - getVectorDoubleValueFnLeft(LEFT_COL, i);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
}
doReleaseVec(pLeftCol, leftConvert); doReleaseVec(pLeftCol, leftConvert);
} }
@ -1315,10 +1257,6 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0); *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -1345,15 +1283,6 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i); *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -1381,10 +1310,6 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx; *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx;
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -1411,14 +1336,6 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i); *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -1439,6 +1356,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pRight->pHashFilter != NULL) { if (pRight->pHashFilter != NULL) {
for (; i >= 0 && i < pLeft->numOfRows; i += step) { for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
@ -1452,6 +1370,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step) { for (; i < pRight->numOfRows && i >= 0; i += step) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore continue; // TODO set null or ignore
} }
@ -1463,6 +1382,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
bool isJsonnull = false; bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){ if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore continue; // TODO set null or ignore
} }
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
@ -1472,6 +1392,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
ASSERT(pLeft->pHashFilter == NULL); ASSERT(pLeft->pHashFilter == NULL);
for (; i >= 0 && i < pLeft->numOfRows; i += step) { for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
@ -1482,6 +1403,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
bool isJsonnull = false; bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){ if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore continue; // TODO set null or ignore
} }
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
@ -1490,6 +1412,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
for (; i >= 0 && i < pRight->numOfRows; i += step) { for (; i >= 0 && i < pRight->numOfRows; i += step) {
if (colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
@ -1500,6 +1423,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
bool isJsonnull = false; bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){ if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore continue; // TODO set null or ignore
} }
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
@ -1589,10 +1513,6 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
char *data = colDataGetVarData(pLeft->columnData, i);
v = (*data == TSDB_DATA_TYPE_NULL)? 1:0;
}
colDataAppendInt8(pOut->columnData, i, &v); colDataAppendInt8(pOut->columnData, i, &v);
} }
pOut->numOfRows = pLeft->numOfRows; pOut->numOfRows = pLeft->numOfRows;
@ -1601,10 +1521,6 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1;
if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
char *data = colDataGetVarData(pLeft->columnData, i);
v = (*data == TSDB_DATA_TYPE_NULL)? 0:1;
}
colDataAppendInt8(pOut->columnData, i, &v); colDataAppendInt8(pOut->columnData, i, &v);
} }
pOut->numOfRows = pLeft->numOfRows; pOut->numOfRows = pLeft->numOfRows;
@ -1671,3 +1587,4 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return NULL; return NULL;
} }
} }

235
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -38,6 +38,7 @@
#include "scalar.h" #include "scalar.h"
#include "nodes.h" #include "nodes.h"
#include "tlog.h" #include "tlog.h"
#include "parUtil.h"
#define _DEBUG_PRINT_ 0 #define _DEBUG_PRINT_ 0
@ -911,17 +912,32 @@ TEST(constantTest, greater_and_lower) {
} }
void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
char key[32] = {0}; char keyVar[32] = {0};
memcpy(varDataVal(key), key, strlen(key)); memcpy(varDataVal(keyVar), key, strlen(key));
varDataLen(key) = strlen(key); varDataLen(keyVar) = strlen(key);
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; SNode *pLeft = NULL, *pRight = NULL;
scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar);
scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json); scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
} }
void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
int32_t resType = TSDB_DATA_TYPE_NULL;
if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
resType = TSDB_DATA_TYPE_DOUBLE;
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
resType = TSDB_DATA_TYPE_BIGINT;
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL ||
opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL ||
opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
opType == OP_TYPE_NMATCH){
resType = TSDB_DATA_TYPE_BOOL;
}
SNode *right = NULL; SNode *right = NULL;
scltMakeValueNode(&right, rightType, rightData); scltMakeValueNode(&right, rightType, rightData);
scltMakeOpNode(opNode, opType, resType, *opNode, right); scltMakeOpNode(opNode, opType, resType, *opNode, right);
@ -932,15 +948,15 @@ void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorT
scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
} }
void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
SArray *blockList = taosArrayInit(2, POINTER_BYTES); SArray *blockList = taosArrayInit(2, POINTER_BYTES);
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SNode *opNode = NULL; SNode *opNode = NULL;
makeJsonArrow(&src, &opNode, row, key); makeJsonArrow(&src, &opNode, json, (char*)key);
taosArrayPush(blockList, &src); taosArrayPush(blockList, &src);
makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value); makeOperator(&opNode, blockList, opType, rightType, rightData);
int32_t code = scalarCalculate(opNode, blockList, NULL); int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -948,39 +964,209 @@ void makeCalculate(void *json, void *key, void *value, double exceptValue, EOper
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
ASSERT_EQ(res->info.rows, 1); ASSERT_EQ(res->info.rows, 1);
SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue); if(colDataIsNull_f(column->nullbitmap, 0)){
ASSERT_EQ(DBL_MAX, exceptValue);
printf("result:NULL\n");
}else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
double tmp = *((double *)colDataGetData(column, 0));
ASSERT_TRUE(tmp == exceptValue);
printf("result:%lf\n", tmp);
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0)));
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
printf("result:%d\n", *((bool *)colDataGetData(column, 0)));
}
taosArrayDestroyEx(blockList, scltFreeDataBlock); taosArrayDestroyEx(blockList, scltFreeDataBlock);
nodesDestroyNode(opNode); nodesDestroyNode(opNode);
} }
TEST(columnTest, int_value_add_json_column) { TEST(columnTest, json_column_arith_op) {
scltInitLogFile(); scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
char *key = "k1";
SKVRowBuilder kvRowBuilder; SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder); tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
int32_t leftv = 1; const int32_t len = 8;
makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD); EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
leftv = 8; printf("--------------------json int---------------------\n");
makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB); char *key = "k1";
double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
leftv = 2; printf("--------------------json string---------------------\n");
makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI);
leftv = 2; key = "k2";
makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV); double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
leftv = 3; printf("---------------------json null--------------------\n");
makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE);
makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS); key = "k3";
double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
printf("---------------------json bool--------------------\n");
key = "k4";
double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
printf("----------------------json double-------------------\n");
key = "k5";
double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
}
void *prepareNchar(char* rightData){
int32_t len = 0;
int32_t inputLen = strlen(rightData);
char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len);
varDataSetLen(t, len);
return t;
}
TEST(columnTest, json_column_logic_op) {
scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
const int32_t len = 9;
const int32_t len1 = 4;
EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH};
int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
printf("--------------------json int---------------------\n");
char *key = "k1";
bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json string---------------------\n");
key = "k2";
bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json null---------------------\n");
key = "k3";
double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json bool---------------------\n");
key = "k4";
bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k5";
bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k6";
bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
taosMemoryFree(rightData);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
taosMemoryFree(rightData);
}
} }
TEST(columnTest, smallint_value_add_int_column) { TEST(columnTest, smallint_value_add_int_column) {
@ -3171,3 +3357,4 @@ int main(int argc, char** argv) {
} }
#pragma GCC diagnostic pop #pragma GCC diagnostic pop