xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/vnode_refact1

This commit is contained in:
Hongze Cheng 2022-04-20 12:29:57 +00:00
parent fbddf859c8 3ecf6df9e3
commit c8acaa77bb
36 changed files with 1302 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
include/common/tdatablock.h
View File

@ -54,12 +54,34 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
} while (0) } while (0)
#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
#define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1)
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \
: colDataGetNumData(p1_, r_))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
if(colDataIsNull_var(pColumnInfoData, row)){
return true;
}
char *data = colDataGetVarData(pColumnInfoData, row);
return (*data == TSDB_DATA_TYPE_NULL);
}
if (!pColumnInfoData->hasNull) { if (!pColumnInfoData->hasNull) {
return false; return false;
} }
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.offset[row] == -1; if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
return colDataIsNull_var(pColumnInfoData, row);
} else { } else {
if (pColumnInfoData->nullbitmap == NULL) { if (pColumnInfoData->nullbitmap == NULL) {
return false; return false;
@ -86,7 +108,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
} }
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.offset[row] == -1; return colDataIsNull_var(pColumnInfoData, row);
} else { } else {
if (pColumnInfoData->nullbitmap == NULL) { if (pColumnInfoData->nullbitmap == NULL) {
return false; return false;
@ -96,17 +118,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
} }
} }
#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
// SColumnInfoData, rowNumber
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
: ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
// There is a placehold for each NULL value of binary or nchar type. // There is a placehold for each NULL value of binary or nchar type.
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type.
} else { } else {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
} }
@ -117,7 +132,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
for (int32_t i = start; i < start + nRows; ++i) { for (int32_t i = start; i < start + nRows; ++i) {
pColumnInfoData->varmeta.offset[i] = -1; // it is a null value of VAR type. colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type.
} }
} else { } else {
for (int32_t i = start; i < start + nRows; ++i) { for (int32_t i = start; i < start + nRows; ++i) {
@ -265,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
#endif #endif
#endif /*_TD_COMMON_EP_H_*/ #endif /*_TD_COMMON_EP_H_*/

5
include/common/tdataformat.h
View File

@ -502,7 +502,7 @@ typedef struct {
#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
#define kvRowLen(r) (*(TDRowLenT *)(r)) #define kvRowLen(r) (*(uint16_t *)(r))
#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
#define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetLen(r, len) kvRowLen(r) = (len)
#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
@ -608,7 +608,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
if (pBuilder->nCols >= pBuilder->tCols) { if (pBuilder->nCols >= pBuilder->tCols) {
pBuilder->tCols *= 2; pBuilder->tCols *= 2;
SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
pBuilder->nCols++; pBuilder->nCols++;
int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
if (tlen > pBuilder->alloc - pBuilder->size) { if (tlen > pBuilder->alloc - pBuilder->size) {
while (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) {
pBuilder->alloc *= 2; pBuilder->alloc *= 2;

4
include/common/ttypes.h
View File

@ -49,7 +49,7 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor; } tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypes[15]; extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
bool isValidDataType(int32_t type); bool isValidDataType(int32_t type);

1
include/libs/scalar/scalar.h
View File

@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
/* Time related functions */ /* Time related functions */
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

1
include/util/taoserror.h
View File

@ -609,6 +609,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INTER_SLIDING_UNIT TAOS_DEF_ERROR_CODE(0, 0x2630) #define TSDB_CODE_PAR_INTER_SLIDING_UNIT TAOS_DEF_ERROR_CODE(0, 0x2630)
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x2631) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x2631)
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632)
#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633)
//planner //planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

2
include/util/tcompare.h
View File

@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);

2
include/util/tdef.h
View File

@ -273,6 +273,8 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_AUTH_LEN 16 #define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32 #define TSDB_PASSWORD_LEN 32
#define TSDB_USET_PASSWORD_LEN 129 #define TSDB_USET_PASSWORD_LEN 129

1
include/util/tjson.h
View File

@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson);
char* tjsonToUnformattedString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson);
SJson* tjsonParse(const char* pStr); SJson* tjsonParse(const char* pStr);
bool tjsonValidateJson(const char* pJson);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/util/tutil.h
View File

@ -26,8 +26,6 @@ extern "C" {
#endif #endif
int32_t strdequote(char *src); int32_t strdequote(char *src);
int32_t strndequote(char *dst, const char *z, int32_t len);
int32_t strRmquote(char *z, int32_t len);
size_t strtrim(char *src); size_t strtrim(char *src);
char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
char **strsplit(char *src, const char *delim, int32_t *num); char **strsplit(char *src, const char *delim, int32_t *num);

70
source/client/src/clientImpl.c
View File

@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData; pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} }
if (type == TSDB_DATA_TYPE_JSON) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
int32_t len = 0;
SResultColumn* pCol = &pResultInfo->pCol[i];
for (int32_t j = 0; j < numOfRows; ++j) {
if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData;
int32_t jsonInnerType = *pStart;
char *jsonInnerData = pStart + CHAR_BYTES;
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
if(jsonInnerType == TSDB_DATA_TYPE_NULL){
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_JSON){
int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
length = 0;
}
varDataSetLen(dst, length);
}else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES);
if (length <= 0) {
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
length = 0;
}
varDataSetLen(dst, length + CHAR_BYTES*2);
*(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
}else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){
double jsonVd = *(double*)(jsonInnerData);
sprintf(varDataVal(dst), "%.9lf", jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){
int64_t jsonVd = *(int64_t*)(jsonInnerData);
sprintf(varDataVal(dst), "%" PRId64, jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst)));
}else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){
sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false");
varDataSetLen(dst, strlen(varDataVal(dst)));
}else {
ASSERT(0);
}
if(len + varDataTLen(dst) > colLength[i]){
p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
}
p = pResultInfo->convertBuf[i] + len;
memcpy(p, dst, varDataTLen(dst));
pCol->offset[j] = len;
len += varDataTLen(dst);
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

21
source/common/src/tdatablock.c
View File

@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
int32_t type = pColumnInfoData->info.type; int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = varDataTLen(pData);
if(type == TSDB_DATA_TYPE_JSON) {
if(*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0;
}else if(*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData+CHAR_BYTES);
}else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES;
}else if(*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES;
}
dataLen += CHAR_BYTES;
}
SVarColAttr* pAttr = &pColumnInfoData->varmeta; SVarColAttr* pAttr = &pColumnInfoData->varmeta;
if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) { if (pAttr->allocLen < pAttr->length + dataLen) {
uint32_t newSize = pAttr->allocLen; uint32_t newSize = pAttr->allocLen;
if (newSize == 0) { if (newSize == 0) {
newSize = 8; newSize = 8;
} }
while (newSize < pAttr->length + varDataTLen(pData)) { while (newSize < pAttr->length + dataLen) {
newSize = newSize * 1.5; newSize = newSize * 1.5;
} }
@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
uint32_t len = pColumnInfoData->varmeta.length; uint32_t len = pColumnInfoData->varmeta.length;
pColumnInfoData->varmeta.offset[currentRow] = len; pColumnInfoData->varmeta.offset[currentRow] = len;
memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); memcpy(pColumnInfoData->pData + len, pData, dataLen);
pColumnInfoData->varmeta.length += varDataTLen(pData); pColumnInfoData->varmeta.length += dataLen;
} else { } else {
memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
} }

3
source/common/src/ttypes.c
View File

@ -379,7 +379,7 @@ static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *
*maxIndex = 0; *maxIndex = 0;
} }
tDataTypeDescriptor tDataTypes[15] = { tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@ -402,6 +402,7 @@ tDataTypeDescriptor tDataTypes[15] = {
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
getStatics_u64}, getStatics_u64},
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
}; };
char tTokenTypeSwitcher[13] = { char tTokenTypeSwitcher[13] = {

2
source/common/src/tvariant.c
View File

@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
} }
case TSDB_DATA_TYPE_BINARY: { case TSDB_DATA_TYPE_BINARY: {
pVar->pz = strndup(z, n); pVar->pz = strndup(z, n);
pVar->nLen = strRmquote(pVar->pz, n); //pVar->nLen = strRmquote(pVar->pz, n);
break; break;
} }
case TSDB_DATA_TYPE_TIMESTAMP: { case TSDB_DATA_TYPE_TIMESTAMP: {

2
source/libs/function/src/builtins.c
View File

@ -939,7 +939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateToJson, .translateFunc = translateToJson,
.getEnvFunc = NULL, .getEnvFunc = NULL,
.initFunc = NULL, .initFunc = NULL,
.sprocessFunc = NULL, .sprocessFunc = toJsonFunction,
.finalizeFunc = NULL .finalizeFunc = NULL
} }
}; };

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1009,6 +1009,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
case OP_TYPE_NOT_LIKE: case OP_TYPE_NOT_LIKE:
case OP_TYPE_MATCH: case OP_TYPE_MATCH:
case OP_TYPE_NMATCH: case OP_TYPE_NMATCH:
case OP_TYPE_JSON_CONTAINS:
case OP_TYPE_IS_NULL: case OP_TYPE_IS_NULL:
case OP_TYPE_IS_NOT_NULL: case OP_TYPE_IS_NOT_NULL:
case OP_TYPE_IS_TRUE: case OP_TYPE_IS_TRUE:
@ -1027,7 +1028,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
bool nodesIsJsonOp(const SOperatorNode* pOp) { bool nodesIsJsonOp(const SOperatorNode* pOp) {
switch (pOp->opType) { switch (pOp->opType) {
case OP_TYPE_JSON_GET_VALUE: case OP_TYPE_JSON_GET_VALUE:
case OP_TYPE_JSON_CONTAINS:
return true; return true;
default: default:
break; break;

1
source/libs/parser/inc/parUtil.h
View File

@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta);
int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta);
int32_t getNumOfTags(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta);
STableComInfo getTableInfo(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta);
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);

1
source/libs/parser/src/parAstCreater.c
View File

@ -48,6 +48,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
} }
static void trimEscape(SToken* pName) { static void trimEscape(SToken* pName) {
// todo need to deal with `ioo``ii` -> ioo`ii
if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) { if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) {
pName->z += 1; pName->z += 1;
pName->n -= 2; pName->n -= 2;

53
source/libs/parser/src/parInsert.c
View File

@ -397,35 +397,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
} }
if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
}
// Remove quotation marks // Remove quotation marks
if (TK_NK_STRING == pToken->type) { if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
} }
// delete escape character: \\, \', \" int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
char delim = pToken->z[0];
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < pToken->n - 1; ++k) {
if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
tmpTokenBuf[j] = pToken->z[k + 1];
cnt++;
j++;
k++;
continue;
}
tmpTokenBuf[j] = pToken->z[k];
j++;
}
tmpTokenBuf[j] = 0;
pToken->z = tmpTokenBuf; pToken->z = tmpTokenBuf;
pToken->n -= 2 + cnt; pToken->n = len;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -603,6 +583,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
return func(pMsgBuf, pToken->z, pToken->n, param); return func(pMsgBuf, pToken->z, pToken->n, param);
} }
case TSDB_DATA_TYPE_JSON: {
if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
}
return func(pMsgBuf, pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_TIMESTAMP: { case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal; int64_t tmpVal;
if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@ -723,8 +710,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
} }
memset(&pColList->boundColumns[pColList->numOfBound], 0, if(pColList->numOfCols > pColList->numOfBound){
sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); memset(&pColList->boundColumns[pColList->numOfBound], 0,
sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -741,9 +730,18 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi
int8_t type = pa->schema->type; int8_t type = pa->schema->type;
int16_t colId = pa->schema->colId; int16_t colId = pa->schema->colId;
if(TSDB_DATA_TYPE_JSON == type){
return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
}
if (value == NULL) { // it is a null data
// tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx);
return TSDB_CODE_SUCCESS;
}
if (TSDB_DATA_TYPE_BINARY == type) { if (TSDB_DATA_TYPE_BINARY == type) {
STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
tdAddColToKVRow(pa->builder, colId, type, pa->buf); tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
} else if (TSDB_DATA_TYPE_NCHAR == type) { } else if (TSDB_DATA_TYPE_NCHAR == type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0; int32_t output = 0;
@ -751,13 +749,12 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi
char buf[512] = {0}; char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno)); snprintf(buf, tListLen(buf), "%s", strerror(errno));
return buildSyntaxErrMsg(pMsgBuf, buf, value); return buildSyntaxErrMsg(pMsgBuf, buf, value);
;
} }
varDataSetLen(pa->buf, output); varDataSetLen(pa->buf, output);
tdAddColToKVRow(pa->builder, colId, type, pa->buf); tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
} else { } else {
tdAddColToKVRow(pa->builder, colId, type, value); tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

46
source/libs/parser/src/parTranslater.c
View File

@ -543,14 +543,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
} }
} else if (nodesIsComparisonOp(pOp)) { } else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) { TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else { } else if (nodesIsJsonOp(pOp)){
// todo json operator if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -1813,6 +1817,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
SNode* pNode;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
if (NULL == pFuncs) { if (NULL == pFuncs) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1848,6 +1863,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY); code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkTableTags(pCxt, pStmt);
}
return code; return code;
} }
@ -3249,17 +3267,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
SKVRowBuilder* pBuilder) { SKVRowBuilder* pBuilder) {
if(pSchema->type == TSDB_DATA_TYPE_JSON){
if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
}
return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
}
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode; return pCxt->errCode;
} }
SVariant var;
valueNodeToVariant(pVal, &var); if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
char tagVal[TSDB_MAX_TAGS_LEN] = {0}; // todo
int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true); }else{
if (TSDB_CODE_SUCCESS == code) { tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal);
} }
return code;
return TSDB_CODE_SUCCESS;
} }
static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,

167
source/libs/parser/src/parUtil.c
View File

@ -14,6 +14,7 @@
*/ */
#include "parUtil.h" #include "parUtil.h"
#include "cJSON.h"
static char* getSyntaxErrFormat(int32_t errCode) { static char* getSyntaxErrFormat(int32_t errCode) {
switch (errCode) { switch (errCode) {
@ -115,6 +116,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "sliding value no larger than the interval value"; return "sliding value no larger than the interval value";
case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL: case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL:
return "sliding value can not less than 1% of interval value"; return "sliding value can not less than 1% of interval value";
case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG:
return "Only one tag if there is a json tag";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:
@ -215,24 +218,178 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
} }
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
// delete escape character: \\, \', \" if (len <=0 || dlen <= 0) return 0;
char delim = src[0]; char delim = src[0];
int32_t cnt = 0;
int32_t j = 0; int32_t j = 0;
for (uint32_t k = 1; k < len - 1; ++k) { for (uint32_t k = 1; k < len - 1; ++k) {
if (j >= dlen) { if (j >= dlen) {
break; dst[j - 1] = '\0';
return j;
} }
if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { if (src[k] == delim && src[k + 1] == delim) { // deal with "", ''
dst[j] = src[k + 1]; dst[j] = src[k + 1];
cnt++;
j++; j++;
k++; k++;
continue; continue;
} }
if (src[k] == '\\') { // deal with escape character
if(src[k+1] == 'n'){
dst[j] = '\n';
}else if(src[k+1] == 'r'){
dst[j] = '\r';
}else if(src[k+1] == 't'){
dst[j] = '\t';
}else if(src[k+1] == '\\'){
dst[j] = '\\';
}else if(src[k+1] == '\''){
dst[j] = '\'';
}else if(src[k+1] == '"'){
dst[j] = '"';
}else if(src[k+1] == '%' || src[k+1] == '_'){
dst[j++] = src[k];
dst[j] = src[k+1];
}else{
dst[j] = src[k+1];
}
j++;
k++;
continue;
}
dst[j] = src[k]; dst[j] = src[k];
j++; j++;
} }
dst[j] = '\0'; dst[j] = '\0';
return j; return j;
} }
static bool isValidateTag(char *input) {
if (!input) return false;
for (size_t i = 0; i < strlen(input); ++i) {
if (isprint(input[i]) == 0) return false;
}
return true;
}
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
// set json NULL data
uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
int jsonIndex = startColId + 1;
if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
return TSDB_CODE_SUCCESS;
}
// set json real data
cJSON *root = cJSON_Parse(json);
if (root == NULL){
return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
}
int size = cJSON_GetArraySize(root);
if(!cJSON_IsObject(root)){
return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
}
int retCode = 0;
char *tagKV = NULL;
SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i);
if (!item) {
qError("json inner error:%d", i);
retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
goto end;
}
char *jsonKey = item->string;
if(!isValidateTag(jsonKey)){
retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
goto end;
}
// if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
// tscError("json key too long error");
// retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
// goto end;
// }
size_t keyLen = strlen(jsonKey);
if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
continue;
}
// key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
if(!tagKV) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKV), jsonKey, keyLen);
varDataSetLen(tagKV, keyLen);
if(taosHashGetSize(keyHash) == 0){
uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type
}
taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless
if(item->type == cJSON_String){ // add json value format: type|data
char *jsonValue = item->valuestring;
int32_t valLen = (int32_t)strlen(jsonValue);
int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
char *tmp = taosMemoryRealloc(tagKV, totalLen);
if(!tmp) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
tagKV = tmp;
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
(int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
goto end;
}
varDataSetLen(valueData, valLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
}else if(item->type == cJSON_Number){
if(!isfinite(item->valuedouble)){
qError("json value is invalidate");
retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
goto end;
}
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
}else if(item->type == cJSON_True || item->type == cJSON_False){
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_BOOL;
*valueData = (char)(item->valueint);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
}else if(item->type == cJSON_NULL){
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
*valueType = TSDB_DATA_TYPE_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
}
else{
retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
goto end;
}
}
if(taosHashGetSize(keyHash) == 0){ // set json NULL true
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
}
end:
taosMemoryFree(tagKV);
taosHashCleanup(keyHash);
cJSON_Delete(root);
return retCode;
}

2
source/libs/scalar/inc/filterInt.h
View File

@ -308,7 +308,6 @@ struct SFilterInfo {
#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@ -321,7 +320,6 @@ struct SFilterInfo {
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)

4
source/libs/scalar/inc/sclvector.h
View File

@ -56,6 +56,8 @@ static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) {
return (double)*((bool *)src + index); return (double)*((bool *)src + index);
} }
double getVectorDoubleValue_JSON(void *src, int32_t index);
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL; _getDoubleValue_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) { if (srcType == TSDB_DATA_TYPE_TINYINT) {
@ -80,6 +82,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
p = getVectorDoubleValue_DOUBLE; p = getVectorDoubleValue_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorDoubleValue_BIGINT; p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_JSON) {
p = getVectorDoubleValue_JSON;
} else if (srcType == TSDB_DATA_TYPE_BOOL) { } else if (srcType == TSDB_DATA_TYPE_BOOL) {
p = getVectorDoubleValue_BOOL; p = getVectorDoubleValue_BOOL;
} else { } else {

100
source/libs/scalar/src/filter.c
View File

@ -57,28 +57,24 @@ OptrStr gOptrStr[] = {
{OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"},
// json operator // json operator
{OP_TYPE_JSON_GET_VALUE, "json get"}, {OP_TYPE_JSON_GET_VALUE, "->"},
{OP_TYPE_JSON_CONTAINS, "json contains"} {OP_TYPE_JSON_CONTAINS, "json contains"}
}; };
bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(maxv, minr); int32_t result = cfunc(maxv, minr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result >= 0; return result >= 0;
} }
bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(maxv, minr); int32_t result = cfunc(maxv, minr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result > 0; return result > 0;
} }
bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(minv, maxr); int32_t result = cfunc(minv, maxr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result <= 0; return result <= 0;
} }
bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
int32_t result = cfunc(minv, maxr); int32_t result = cfunc(minv, maxr);
//if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result < 0; return result < 0;
} }
bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@ -170,7 +166,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch,
compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
}; };
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@ -221,7 +217,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
assert(0); assert(0);
} }
} }
if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
return 28;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break;
@ -1049,6 +1050,8 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
cell = cell->pNext; cell = cell->pNext;
} }
colDataDestroy(out.columnData);
taosMemoryFree(out.columnData);
} else { } else {
filterAddFieldFromNode(info, node->pRight, &right); filterAddFieldFromNode(info, node->pRight, &right);
@ -1778,9 +1781,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
fi->data = taosMemoryCalloc(1, bytes); fi->data = taosMemoryCalloc(1, bytes);
} else if (type != TSDB_DATA_TYPE_JSON){ } else{
if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
/* /*
fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);
for (int32_t a = 0; a < dType->bytes; ++a) { for (int32_t a = 0; a < dType->bytes; ++a) {
int64_t *v = taosArrayGet(var->arr, a); int64_t *v = taosArrayGet(var->arr, a);
@ -1791,31 +1794,29 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
} else { } else {
fi->data = taosMemoryCalloc(1, sizeof(int64_t)); fi->data = taosMemoryCalloc(1, sizeof(int64_t));
} }
} else{ // type == TSDB_DATA_TYPE_JSON
// fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
} }
if(type != TSDB_DATA_TYPE_JSON) { if (dType->type == type) {
if (dType->type == type) { assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); } else {
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
out.columnData->info.type = type;
if (IS_VAR_DATA_TYPE(type)) {
out.columnData->info.bytes = bytes;
} else { } else {
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.bytes = tDataTypes[type].bytes;
out.columnData->info.type = type;
if (IS_VAR_DATA_TYPE(type)) {
out.columnData->info.bytes = bytes;
} else {
out.columnData->info.bytes = tDataTypes[type].bytes;
}
// todo refactor the convert
int32_t code = doConvertDataType(var, &out);
if (code != TSDB_CODE_SUCCESS) {
qError("convert value to type[%d] failed", type);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
} }
// todo refactor the convert
int32_t code = doConvertDataType(var, &out);
if (code != TSDB_CODE_SUCCESS) {
qError("convert value to type[%d] failed", type);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
colDataDestroy(out.columnData);
taosMemoryFree(out.columnData);
} }
// match/nmatch for nchar type need convert from ucs4 to mbs // match/nmatch for nchar type need convert from ucs4 to mbs
@ -2556,11 +2557,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
if (unit->right.type == FLD_TYPE_VALUE) { if (unit->right.type == FLD_TYPE_VALUE) {
if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
}else{
info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
}
} else { } else {
info->cunits[i].valData = NULL; info->cunits[i].valData = NULL;
} }
@ -2886,18 +2883,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0]; uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
if (!colData){ // for json->'key' is null
(*p)[i] = 1;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
}else{
(*p)[i] = 0;
}
}else{
(*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
}
if ((*p)[i] == 0) { if ((*p)[i] == 0) {
all = false; all = false;
} }
@ -2921,19 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
uint32_t uidx = info->groups[0].unitIdxs[0]; uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
if (!colData) { // for json->'key' is not null
(*p)[i] = 0;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
}else{ // for json->'key' is not null
(*p)[i] = 1;
}
}else {
(*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
}
if ((*p)[i] == 0) { if ((*p)[i] == 0) {
all = false; all = false;
} }
@ -3566,6 +3541,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if (OP_TYPE_JSON_CONTAINS == node->opType) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
SNode *t = node->pLeft; SNode *t = node->pLeft;
node->pLeft = node->pRight; node->pLeft = node->pRight;

36
source/libs/scalar/src/scalar.c
View File

@ -132,6 +132,7 @@ void sclFreeRes(SHashObj *res) {
void sclFreeParam(SScalarParam *param) { void sclFreeParam(SScalarParam *param) {
if (param->columnData != NULL) { if (param->columnData != NULL) {
colDataDestroy(param->columnData); colDataDestroy(param->columnData);
taosMemoryFree(param->columnData);
} }
if (param->pHashFilter != NULL) { if (param->pHashFilter != NULL) {
@ -605,37 +606,48 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
STargetNode *target = (STargetNode *)pNode; STargetNode *target = (STargetNode *)pNode;
if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); int32_t index = -1;
for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i);
if (pb->info.blockId == target->dataBlockId) {
index = i;
break;
}
}
if (index == -1) {
sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
// if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
if (NULL == res) { if (NULL == res) {
sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
ctx->code = TSDB_CODE_QRY_APP_ERROR; ctx->code = TSDB_CODE_QRY_APP_ERROR;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
for (int32_t i = 0; i < res->numOfRows; ++i) { colDataAssign(col, res->columnData, res->numOfRows);
if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) { block->info.rows = res->numOfRows;
colDataAppend(col, i, NULL, true);
} else {
char *p = colDataGetData(res->columnData, i);
colDataAppend(col, i, p, false);
}
}
sclFreeParam(res); sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);

54
source/libs/scalar/src/sclfunc.c
View File

@ -4,6 +4,7 @@
#include "ttime.h" #include "ttime.h"
#include "sclInt.h" #include "sclInt.h"
#include "sclvector.h" #include "sclvector.h"
#include "tjson.h"
typedef float (*_float_fn)(float); typedef float (*_float_fn)(float);
typedef double (*_double_fn)(double); typedef double (*_double_fn)(double);
@ -872,6 +873,59 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput);
if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
return TSDB_CODE_FAILED;
}
if (inputNum != 1) {
return TSDB_CODE_FAILED;
}
char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1);
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
if(type == TSDB_DATA_TYPE_NCHAR){
if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp);
if (len < 0) {
colDataAppendNULL(pOutput->columnData, i);
continue;
}
tmp[len] = 0;
}else{
if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
memcpy(tmp, varDataVal(input), varDataLen(input));
tmp[varDataTLen(input)] = 0;
}
if(!tjsonValidateJson(tmp)){
colDataAppendNULL(pOutput->columnData, i);
continue;
}
colDataAppend(pOutput->columnData, i, input, false);
input += varDataTLen(input);
}
taosMemoryFree(tmp);
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(&pInput[0]); int32_t type = GET_PARAM_TYPE(&pInput[0]);
int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);

527
source/libs/scalar/src/sclvector.c
View File

@ -25,6 +25,78 @@
#include "tdatablock.h" #include "tdatablock.h"
#include "ttypes.h" #include "ttypes.h"
#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_INT: {
GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UINT: {
GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
GET_TYPED_DATA(*((float *)outData), float, inType, inData);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(*((double *)outData), double, inType, inData);
break;
}
default:{
ASSERT(0);
}
}
}
void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
char *tmp = taosMemoryMalloc(varDataTLen(inData));
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
}
tmp[len] = 0;
ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
double value = strtod(tmp, NULL);
*((double *)outData) = value;
taosMemoryFreeClear(tmp);
}
typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@ -61,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) {
return (int64_t)*((bool *)src + index); return (int64_t)*((bool *)src + index);
} }
int64_t getVectorBigintValue_JSON(void *src, int32_t index){
ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0;
if (*data == TSDB_DATA_TYPE_NULL){
return 0;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return (int64_t)out;
}
_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
_getBigintValue_fn_t p = NULL; _getBigintValue_fn_t p = NULL;
if(srcType==TSDB_DATA_TYPE_TINYINT) { if(srcType==TSDB_DATA_TYPE_TINYINT) {
@ -87,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
p = getVectorBigintValue_BIGINT; p = getVectorBigintValue_BIGINT;
}else if(srcType==TSDB_DATA_TYPE_BOOL) { }else if(srcType==TSDB_DATA_TYPE_BOOL) {
p = getVectorBigintValue_BOOL; p = getVectorBigintValue_BOOL;
}else if(srcType==TSDB_DATA_TYPE_JSON) {
p = getVectorBigintValue_JSON;
}else { }else {
assert(0); assert(0);
} }
@ -213,7 +301,8 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
func = varToUnsigned; func = varToUnsigned;
} else if (IS_FLOAT_TYPE(outType)) { } else if (IS_FLOAT_TYPE(outType)) {
func = varToFloat; func = varToFloat;
} else if (outType == TSDB_DATA_TYPE_NCHAR) { } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar
ASSERT(inType == TSDB_DATA_TYPE_VARCHAR);
func = varToNchar; func = varToNchar;
vton = true; vton = true;
} else { } else {
@ -228,14 +317,28 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
continue; continue;
} }
char* data = colDataGetData(pIn->columnData, i); char* data = colDataGetVarData(pIn->columnData, i);
int32_t convertType = inType;
if(inType == TSDB_DATA_TYPE_JSON){
if(*data == TSDB_DATA_TYPE_NULL) {
colDataAppendNULL(pOut->columnData, i);
continue;
}
else if(*data == TSDB_DATA_TYPE_NCHAR) {
data += CHAR_BYTES;
convertType = TSDB_DATA_TYPE_NCHAR;
} else {
convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType);
continue;
}
}
if (vton) { if (vton) {
memcpy(tmp, data, varDataTLen(data)); memcpy(tmp, data, varDataTLen(data));
} else { } else {
if (TSDB_DATA_TYPE_VARCHAR == inType) { if (TSDB_DATA_TYPE_VARCHAR == convertType) {
memcpy(tmp, varDataVal(data), varDataLen(data)); memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0; tmp[varDataLen(data)] = 0;
} else { } else if (TSDB_DATA_TYPE_NCHAR == convertType){
ASSERT(varDataLen(data) <= bufSize); ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@ -256,6 +359,66 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
double getVectorDoubleValue_JSON(void *src, int32_t index){
char *data = colDataGetVarData((SColumnInfoData*)src, index);
double out = 0;
if (*data == TSDB_DATA_TYPE_NULL){
return out;
} else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} else {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return out;
}
void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
if(optr == OP_TYPE_JSON_CONTAINS) {
return;
}
if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
return;
}
if(typeLeft == TSDB_DATA_TYPE_JSON){
typeLeft = **pLeftData;
(*pLeftData) ++;
}
if(typeRight == TSDB_DATA_TYPE_JSON){
typeRight = **pRightData;
(*pRightData) ++;
}
if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
*isNull = true;
return;
}
int8_t type = vectorGetConvertType(typeLeft, typeRight);
if(type == 0) {
*fp = filterGetCompFunc(typeLeft, optr);
return;
}
*fp = filterGetCompFunc(type, optr);
if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
} else if(typeLeft != type) {
convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
}
if(typeRight == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
} else if(typeRight != type) {
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
}
// TODO opt performance // TODO opt performance
int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
SColumnInfoData* pInputCol = pIn->columnData; SColumnInfoData* pInputCol = pIn->columnData;
@ -422,24 +585,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
} }
int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */ /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 0, 0, 0, /*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 0, 7, 0, 0,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 0, 0, 0, /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 0, 0, 0, /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 0, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 0, 0, 0, /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 0, /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
}; };
@ -530,7 +693,7 @@ enum {
static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
SColumnInfoData* pCol = pParam->columnData; SColumnInfoData* pCol = pParam->columnData;
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) {
pDest->numOfRows = pParam->numOfRows; pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@ -564,11 +727,12 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0); if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+ getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
} }
} }
@ -583,12 +747,12 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -614,6 +778,50 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
} }
} }
char *getJsonValue(char *json, char *key){ //todo
int16_t cols = kvRowNCols(json);
for (int i = 0; i < cols; ++i) {
SColIdx *pColIdx = kvRowColIdxAt(json, i);
char *data = kvRowColVal(json, pColIdx);
if(i == 0){
if(*data == TSDB_DATA_TYPE_NULL) {
return NULL;
}
continue;
}
if(memcmp(key, data, varDataTLen(data)) == 0){
return data + varDataTLen(data);
}
}
return NULL;
}
void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData;
int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
char *pRightData = colDataGetVarData(pRight->columnData, 0);
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_var(pLeft->columnData, i)) {
colDataSetNull_var(pOutputCol, i);
pOutputCol->hasNull = true;
continue;
}
char *pLeftData = colDataGetVarData(pLeft->columnData, i);
char *value = getJsonValue(pLeftData, pRightData);
if (!value) {
colDataSetNull_var(pOutputCol, i);
pOutputCol->hasNull = true;
continue;
}
colDataAppend(pOutputCol, i, value, false);
}
}
void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData; SColumnInfoData *pOutputCol = pOut->columnData;
@ -636,17 +844,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -659,17 +862,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -692,11 +890,12 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor; if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); *output = (getVectorDoubleValueFnLeft(LEFT_COL, i)
- getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor;
} }
} }
} }
@ -711,12 +910,12 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor; *output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor;
} }
pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) {
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
}
} }
} }
@ -740,17 +939,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
}
}
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -763,17 +957,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -796,11 +985,12 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0); if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
} }
} }
@ -822,17 +1012,13 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
* getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -860,39 +1046,37 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
} }
*output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else { } else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pRightCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pRightCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows)); *output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
/ getVectorDoubleValueFnRight(RIGHT_COL, i);
} }
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else { } else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0); if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
/ getVectorDoubleValueFnRight(RIGHT_COL, 0);
} }
} }
} }
@ -920,13 +1104,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) { if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
} }
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -935,17 +1119,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
*output = lx - ((int64_t)(lx / rx)) * rx; *output = lx - ((int64_t)(lx / rx)) * rx;
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0);
if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else { } else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
if (colDataIsNull_f(pRightCol->nullbitmap, i)) { if (colDataIsNull_s(pRightCol, i)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
} }
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -955,17 +1139,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
} }
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0);
if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else { } else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { if (colDataIsNull_s(pLeftCol, i)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
} }
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
if (isnan(lx) || isinf(lx)) { if (isnan(lx) || isinf(lx)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -995,12 +1179,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
*output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = pLeftCol->hasNull; }
if (pOutputCol->hasNull) { *output = - getVectorDoubleValueFnLeft(LEFT_COL, i);
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
} }
doReleaseVec(pLeftCol, leftConvert); doReleaseVec(pLeftCol, leftConvert);
@ -1063,15 +1246,15 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0); if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0);
} }
} }
} }
@ -1093,17 +1276,12 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData; int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j];
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
@ -1120,16 +1298,16 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
int64_t *output = (int64_t *)pOutputCol->pData; int64_t *output = (int64_t *)pOutputCol->pData;
if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0); int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0);
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | rx; if (colDataIsNull_s(pLeftCol, i)) {
} colDataAppendNULL(pOutputCol, i);
pOutputCol->hasNull = pLeftCol->hasNull; continue; // TODO set null or ignore
if (pOutputCol->hasNull) { }
memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx;
} }
} }
} }
@ -1151,15 +1329,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
int64_t *output = (int64_t *)pOutputCol->pData; int64_t *output = (int64_t *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | getVectorBigintValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
} colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
if (pOutputCol->hasNull) {
int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
for (int32_t j = 0; j < numOfBitLen; ++j) {
pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
} }
*output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
@ -1181,6 +1355,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pRight->pHashFilter != NULL) { if (pRight->pHashFilter != NULL) {
for (; i >= 0 && i < pLeft->numOfRows; i += step) { for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
@ -1194,35 +1369,62 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step) { for (; i < pRight->numOfRows && i >= 0; i += step) {
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore continue; // TODO set null or ignore
} }
char *pLeftData = colDataGetData(pLeft->columnData, i); char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, i); char *pRightData = colDataGetData(pRight->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
char *pRightData = colDataGetData(pRight->columnData, 0);
ASSERT(pLeft->pHashFilter == NULL); ASSERT(pLeft->pHashFilter == NULL);
for (; i >= 0 && i < pLeft->numOfRows; i += step) { for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
char *pLeftData = colDataGetData(pLeft->columnData, i); char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, 0);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
char *pLeftData = colDataGetData(pLeft->columnData, 0);
for (; i >= 0 && i < pRight->numOfRows; i += step) { for (; i >= 0 && i < pRight->numOfRows; i += step) {
if (colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOut->columnData, i);
continue; continue;
} }
char *pLeftData = colDataGetData(pLeft->columnData, 0);
char *pRightData = colDataGetData(pLeft->columnData, i); char *pRightData = colDataGetData(pLeft->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
bool isJsonnull = false;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
if(isJsonnull){
colDataAppendNULL(pOut->columnData, i);
continue; // TODO set null or ignore
}
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
@ -1303,6 +1505,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
} }
void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
}
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
@ -1371,8 +1577,13 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorBitOr; return vectorBitOr;
case OP_TYPE_IS_TRUE: case OP_TYPE_IS_TRUE:
return vectorIsTrue; return vectorIsTrue;
case OP_TYPE_JSON_GET_VALUE:
return vectorJsonArrow;
case OP_TYPE_JSON_CONTAINS:
return vectorJsonContains;
default: default:
assert(0); assert(0);
return NULL; return NULL;
} }
} }

3
source/libs/scalar/test/scalar/CMakeLists.txt
View File

@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES( TARGET_LINK_LIBRARIES(
scalarTest scalarTest
PUBLIC os util common gtest qcom function nodes scalar PUBLIC os util common gtest qcom function nodes scalar parser
) )
TARGET_INCLUDE_DIRECTORIES( TARGET_INCLUDE_DIRECTORIES(
scalarTest scalarTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
) )

263
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -38,6 +38,7 @@
#include "scalar.h" #include "scalar.h"
#include "nodes.h" #include "nodes.h"
#include "tlog.h" #include "tlog.h"
#include "parUtil.h"
#define _DEBUG_PRINT_ 0 #define _DEBUG_PRINT_ 0
@ -92,6 +93,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
blockDataEnsureCapacity(res, rows); blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1; *dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
*slotId = 0; *slotId = 0;
} else { } else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
@ -909,6 +911,264 @@ TEST(constantTest, greater_and_lower) {
nodesDestroyNode(res); nodesDestroyNode(res);
} }
void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
char keyVar[32] = {0};
memcpy(varDataVal(keyVar), key, strlen(key));
varDataLen(keyVar) = strlen(key);
SNode *pLeft = NULL, *pRight = NULL;
scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar);
scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
}
void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
int32_t resType = TSDB_DATA_TYPE_NULL;
if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
resType = TSDB_DATA_TYPE_DOUBLE;
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
resType = TSDB_DATA_TYPE_BIGINT;
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL ||
opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL ||
opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
opType == OP_TYPE_NMATCH){
resType = TSDB_DATA_TYPE_BOOL;
}
SNode *right = NULL;
scltMakeValueNode(&right, rightType, rightData);
scltMakeOpNode(opNode, opType, resType, *opNode, right);
SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo);
scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
}
void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
SArray *blockList = taosArrayInit(2, POINTER_BYTES);
SSDataBlock *src = NULL;
SNode *opNode = NULL;
makeJsonArrow(&src, &opNode, json, (char*)key);
taosArrayPush(blockList, &src);
makeOperator(&opNode, blockList, opType, rightType, rightData);
int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0);
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
ASSERT_EQ(res->info.rows, 1);
SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
if(colDataIsNull_f(column->nullbitmap, 0)){
ASSERT_EQ(DBL_MAX, exceptValue);
printf("result:NULL\n");
}else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
double tmp = *((double *)colDataGetData(column, 0));
ASSERT_TRUE(tmp == exceptValue);
printf("result:%lf\n", tmp);
}else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0)));
}else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
printf("result:%d\n", *((bool *)colDataGetData(column, 0)));
}
taosArrayDestroyEx(blockList, scltFreeDataBlock);
nodesDestroyNode(opNode);
}
TEST(columnTest, json_column_arith_op) {
scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
const int32_t len = 8;
EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
printf("--------------------json int---------------------\n");
char *key = "k1";
double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
printf("--------------------json string---------------------\n");
key = "k2";
double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
printf("---------------------json null--------------------\n");
key = "k3";
double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
printf("---------------------json bool--------------------\n");
key = "k4";
double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
printf("----------------------json double-------------------\n");
key = "k5";
double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
}
void *prepareNchar(char* rightData){
int32_t len = 0;
int32_t inputLen = strlen(rightData);
char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len);
varDataSetLen(t, len);
return t;
}
TEST(columnTest, json_column_logic_op) {
scltInitLogFile();
char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
const int32_t len = 9;
const int32_t len1 = 4;
EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH};
int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
printf("--------------------json int---------------------\n");
char *key = "k1";
bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json string---------------------\n");
key = "k2";
bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json null---------------------\n");
key = "k3";
double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json bool---------------------\n");
key = "k4";
bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k5";
bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
taosMemoryFree(rightData);
}
printf("--------------------json double---------------------\n");
key = "k6";
bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
taosMemoryFree(rightData);
}
printf("---------------------json not exist--------------------\n");
key = "k10";
double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
for(int i = 0; i < len; i++){
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
}
for(int i = len; i < len + len1; i++){
void* rightData = prepareNchar(inputNchar[i-len]);
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
taosMemoryFree(rightData);
}
}
TEST(columnTest, smallint_value_add_int_column) { TEST(columnTest, smallint_value_add_int_column) {
scltInitLogFile(); scltInitLogFile();
@ -928,7 +1188,7 @@ TEST(columnTest, smallint_value_add_int_column) {
int16_t dataBlockId = 0, slotId = 0; int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
int32_t code = scalarCalculate(opNode, blockList, NULL); int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -3097,3 +3357,4 @@ int main(int argc, char** argv) {
} }
#pragma GCC diagnostic pop #pragma GCC diagnostic pop

5
source/util/src/tcompare.c
View File

@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
return compareLenPrefixedWStr(pRight, pLeft); return compareLenPrefixedWStr(pRight, pLeft);
} }
int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
if(pLeft) return 0;
return 1;
}
/* /*
* Compare two strings * Compare two strings
* TSDB_MATCH: Match * TSDB_MATCH: Match

34
source/util/src/tjson.c
View File

@ -276,3 +276,37 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void
} }
SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); }
bool tjsonValidateJson(const char *jIn) {
if (!jIn){
return false;
}
// set json real data
cJSON *root = cJSON_Parse(jIn);
if (root == NULL){
return false;
}
if(!cJSON_IsObject(root)){
return false;
}
int size = cJSON_GetArraySize(root);
for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i);
if (!item) {
return false;
}
char* jsonKey = item->string;
if (!jsonKey) return false;
for (size_t j = 0; j < strlen(jsonKey); ++i) {
if (isprint(jsonKey[i]) == 0) return false;
}
if (item->type == cJSON_Object || item->type == cJSON_Array) {
return false;
}
}
return true;
}

59
source/util/src/tutil.c
View File

@ -47,65 +47,6 @@ int32_t strdequote(char *z) {
return j + 1; // only one quote, do nothing return j + 1; // only one quote, do nothing
} }
int32_t strRmquote(char *z, int32_t len) {
// delete escape character: \\, \', \"
char delim = z[0];
if (delim != '\'' && delim != '\"') {
return len;
}
int32_t cnt = 0;
int32_t j = 0;
for (uint32_t k = 1; k < len - 1; ++k) {
if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) {
if (z[k] == '\\' && z[k + 1] == '_') {
// match '_' self
} else {
z[j] = z[k + 1];
cnt++;
j++;
k++;
continue;
}
}
z[j] = z[k];
j++;
}
z[j] = 0;
return len - 2 - cnt;
}
int32_t strndequote(char *dst, const char *z, int32_t len) {
assert(dst != NULL);
if (z == NULL || len == 0) {
return 0;
}
int32_t quote = z[0];
int32_t i = 1, j = 0;
while (z[i] != 0) {
if (z[i] == quote) {
if (z[i + 1] == quote) {
dst[j++] = (char)quote;
i++;
} else {
dst[j++] = 0;
return (j - 1);
}
} else {
dst[j++] = z[i];
}
i++;
}
return j + 1; // only one quote, do nothing
}
size_t strtrim(char *z) { size_t strtrim(char *z) {
int32_t i = 0; int32_t i = 0;
int32_t j = 0; int32_t j = 0;

3
tests/script/jenkins/basic.txt
View File

@ -93,4 +93,7 @@
# --- sma # --- sma
./test.sh -f tsim/sma/tsmaCreateInsertData.sim ./test.sh -f tsim/sma/tsmaCreateInsertData.sim
# --- valgrind
./test.sh -f tsim/valgrind/checkError.sim -v
#======================b1-end=============== #======================b1-end===============

38
tests/script/sh/checkValgrind.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
set +e
#set -x
NODE_NAME=
while getopts "n:" arg
do
case $arg in
n)
NODE_NAME=$OPTARG
;;
?)
echo "unkown argument"
;;
esac
done
SCRIPT_DIR=`dirname $0`
cd $SCRIPT_DIR/../
SCRIPT_DIR=`pwd`
IN_TDINTERNAL="community"
if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
cd ../../..
else
cd ../../
fi
TAOS_DIR=`pwd`
LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log
#CFG_DIR=$TAOS_DIR/sim/$NODE_NAME/cfg
#echo ---- $LOG_DIR
errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"`
echo $errors

83
tests/script/tsim/valgrind/checkError.sim Normal file
View File

@ -0,0 +1,83 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
#system sh/deploy.sh -n dnode2 -i 2
#system sh/deploy.sh -n dnode3 -i 3
#system sh/deploy.sh -n dnode4 -i 4
#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
#system sh/exec.sh -n dnode2 -s start
#system sh/exec.sh -n dnode3 -s start
#system sh/exec.sh -n dnode4 -s start
sleep 2000
#$loop_cnt = 0
#check_dnode_ready:
# $loop_cnt = $loop_cnt + 1
# sleep 200
# if $loop_cnt == 10 then
# print ====> dnode not ready!
# return -1
# endi
#sql show dnodes
#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
#if $data[0][0] != 1 then
# return -1
#endi
#if $data[0][4] != ready then
# goto check_dnode_ready
#endi
#
##sql connect
#sql create dnode $hostname port 7200
#sql create dnode $hostname port 7300
#sql create dnode $hostname port 7400
#
#$loop_cnt = 0
#check_dnode_ready_1:
#$loop_cnt = $loop_cnt + 1
#sleep 200
#if $loop_cnt == 10 then
# print ====> dnodes not ready!
# return -1
#endi
#sql show dnodes
#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
#if $data[0][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[1][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[2][4] != ready then
# goto check_dnode_ready_1
#endi
#if $data[3][4] != ready then
# goto check_dnode_ready_1
#endi
#=========== please add any actions above =================
print ====> stop all dondes to output valgrind log file
system sh/exec.sh -n dnode1 -s stop -x SIGINT
print ====> start to check if there are ERRORS in vagrind log file for each dnode
# -n : dnode[x] be check
system_content sh/checkValgrind.sh -n dnode1
print cmd return result----> [ $system_content ]
if $system_content == @ERROR SUMMARY: 0 errors@ then
return 0
endi
$null=
if $system_content == $null then
return 0
endi
return -1

6
tests/tsim/src/simExe.c
View File

@ -441,6 +441,12 @@ void simStoreSystemContentResult(SScript *script, char *filename) {
// if ((fd = fopen(filename, "r")) != NULL) { // if ((fd = fopen(filename, "r")) != NULL) {
if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) {
taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1);
int32_t len = strlen(script->system_ret_content);
for (int32_t i = 0; i < len; ++i) {
if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') {
script->system_ret_content[i] = 0;
}
}
taosCloseFile(&pFile); taosCloseFile(&pFile);
char rmCmd[MAX_FILE_NAME_LEN] = {0}; char rmCmd[MAX_FILE_NAME_LEN] = {0};
sprintf(rmCmd, "rm -f %s", filename); sprintf(rmCmd, "rm -f %s", filename);

74
tools/shell/src/shellEngine.c
View File

@ -224,63 +224,27 @@ int32_t shellRunCommand(TAOS *con, char *command) {
} }
} }
bool esc = false; char quote = 0, *cmd = command;
char quote = 0, *cmd = command, *p = command;
for (char c = *command++; c != 0; c = *command++) { for (char c = *command++; c != 0; c = *command++) {
if (esc) { if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
switch (c) { command ++;
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'G':
*p++ = '\\';
break;
case '\'':
case '"':
if (quote) {
*p++ = '\\';
}
break;
}
*p++ = c;
esc = false;
continue; continue;
} }
if (c == '\\') {
if (quote != 0 && (*command == '_' || *command == '\\')) {
// DO nothing
} else {
esc = true;
continue;
}
}
if (quote == c) { if (quote == c) {
quote = 0; quote = 0;
} else if (quote == 0 && (c == '\'' || c == '"')) { } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
quote = c; quote = c;
} } else if (c == ';' && quote == 0) {
c = *command;
*p++ = c; *command = 0;
if (c == ';' && quote == 0) {
c = *p;
*p = 0;
if (shellRunSingleCommand(con, cmd) < 0) { if (shellRunSingleCommand(con, cmd) < 0) {
return -1; return -1;
} }
*p = c; *command = c;
p = cmd; cmd = command;
} }
} }
*p = 0;
return shellRunSingleCommand(con, cmd); return shellRunSingleCommand(con, cmd);
} }
@ -574,19 +538,23 @@ static void shellPrintNChar(const char *str, int length, int width) {
while (pos < length) { while (pos < length) {
TdWchar wc; TdWchar wc;
int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
if (bytes == 0) { if (bytes <= 0) {
break; break;
} }
pos += bytes; if (pos + bytes > length) {
if (pos > length) {
break; break;
} }
int w = 0;
#ifdef WINDOWS #ifdef WINDOWS
int w = bytes; w = bytes;
#else #else
int w = taosWcharWidth(wc); if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
w = bytes;
}else{
w = taosWcharWidth(wc);
}
#endif #endif
pos += bytes;
if (w <= 0) { if (w <= 0) {
continue; continue;
} }
@ -680,6 +648,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
shellPrintNChar(val, length, width); shellPrintNChar(val, length, width);
break; break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
@ -792,7 +761,8 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
return TMAX(field->bytes, width); return TMAX(field->bytes, width);
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:{
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) { if (bytes > tsMaxBinaryDisplayWidth) {
return TMAX(tsMaxBinaryDisplayWidth, width); return TMAX(tsMaxBinaryDisplayWidth, width);