xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: add compare logic for json value

This commit is contained in:
wangmm0220 2022-04-16 21:24:07 +08:00
parent 73c34da280
commit b7bd654c77
11 changed files with 358 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/ttypes.h
View File

@ -49,7 +49,7 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))

2
include/util/tcompare.h
View File

@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);

1
include/util/tdef.h
View File

@ -274,7 +274,6 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_JSON_PLACEHOLDER 0x7F #define TSDB_JSON_PLACEHOLDER 0x7F
#define TSDB_JSON_null 0x00 #define TSDB_JSON_null 0x00
#define TSDB_JSON_KEY_NULL 0x00
#define TSDB_JSON_NOT_NULL 0x01 #define TSDB_JSON_NOT_NULL 0x01
#define TSDB_JSON_NULL 0x00 #define TSDB_JSON_NULL 0x00

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1006,6 +1006,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
case OP_TYPE_NOT_LIKE: case OP_TYPE_NOT_LIKE:
case OP_TYPE_MATCH: case OP_TYPE_MATCH:
case OP_TYPE_NMATCH: case OP_TYPE_NMATCH:
case OP_TYPE_JSON_CONTAINS:
case OP_TYPE_IS_NULL: case OP_TYPE_IS_NULL:
case OP_TYPE_IS_NOT_NULL: case OP_TYPE_IS_NOT_NULL:
case OP_TYPE_IS_TRUE: case OP_TYPE_IS_TRUE:
@ -1024,7 +1025,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
bool nodesIsJsonOp(const SOperatorNode* pOp) { bool nodesIsJsonOp(const SOperatorNode* pOp) {
switch (pOp->opType) { switch (pOp->opType) {
case OP_TYPE_JSON_GET_VALUE: case OP_TYPE_JSON_GET_VALUE:
case OP_TYPE_JSON_CONTAINS:
return true; return true;
default: default:
break; break;

38
source/libs/parser/src/parTranslater.c
View File

@ -469,14 +469,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
} else if (nodesIsComparisonOp(pOp)) { } else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) { TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
} }
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else { } else if (nodesIsJsonOp(pOp)){
// todo json operator if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -2814,25 +2818,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
SKVRowBuilder* pBuilder) { SKVRowBuilder* pBuilder) {
if(pSchema->type == TSDB_DATA_TYPE_JSON){
if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
}
return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
}
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode; return pCxt->errCode;
} }
SVariant var;
valueNodeToVariant(pVal, &var); if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
if(pSchema->type == TSDB_DATA_TYPE_JSON){ // todo
if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ }else{
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz); tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
}
return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
} }
char tagVal[TSDB_MAX_TAGS_LEN] = {0}; return TSDB_CODE_SUCCESS;
int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
if (TSDB_CODE_SUCCESS == code) {
tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
}
return code;
} }
static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,

76
source/libs/parser/src/parUtil.c
View File

@ -252,12 +252,10 @@ static bool isValidateTag(char *input) {
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
// set json NULL data // set json NULL data
uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
uint8_t jsonNULL = TSDB_JSON_NULL; uint8_t jsonNULL = TSDB_JSON_NULL;
int jsonIndex = startColId + 1; int jsonIndex = startColId + 1;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type
if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -273,6 +271,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
} }
int retCode = 0; int retCode = 0;
char *tagKV = NULL;
SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
for(int i = 0; i < size; i++) { for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i); cJSON* item = cJSON_GetArrayItem(root, i);
@ -296,66 +295,64 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){ if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
continue; continue;
} }
// key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
if(!tagKV) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKV), jsonKey, keyLen);
varDataSetLen(tagKV, keyLen);
if(taosHashGetSize(keyHash) == 0){ if(taosHashGetSize(keyHash) == 0){
uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type
} }
// json key encode by binary
void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
if(!tagKey) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKey), jsonKey, keyLen);
taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless
varDataSetLen(tagKey, keyLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey)); // add json key
taosMemoryFree(tagKey);
if(item->type == cJSON_String){ // add json value format: type|data if(item->type == cJSON_String){ // add json value format: type|data
char *jsonValue = item->valuestring; char *jsonValue = item->valuestring;
int32_t valLen = (int32_t)strlen(jsonValue); int32_t valLen = (int32_t)strlen(jsonValue);
char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1); int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
if(!tagVal) { char *tmp = taosMemoryRealloc(tagKV, totalLen);
if(!tmp) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end; goto end;
} ; }
tagVal[0] = TSDB_DATA_TYPE_NCHAR; tagKV = tmp;
char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES); char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData), char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
(int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
taosMemoryFree(tagVal);
goto end; goto end;
} }
varDataSetLen(tagData, valLen); varDataSetLen(valueData, valLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData)); tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
taosMemoryFree(tagVal);
}else if(item->type == cJSON_Number){ }else if(item->type == cJSON_Number){
if(!isfinite(item->valuedouble)){ if(!isfinite(item->valuedouble)){
qError("json value is invalidate"); qError("json value is invalidate");
retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
goto end; goto end;
} }
char tagVal[LONG_BYTES + CHAR_BYTES] = {0}; char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
: TSDB_DATA_TYPE_DOUBLE; *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble; else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint; tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
}else if(item->type == cJSON_True || item->type == cJSON_False){ }else if(item->type == cJSON_True || item->type == cJSON_False){
char tagVal[CHAR_BYTES + CHAR_BYTES] = {0}; char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
tagVal[0] = TSDB_DATA_TYPE_BOOL; char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
tagVal[1] = (char)(item->valueint); *valueType = TSDB_DATA_TYPE_BOOL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES); *valueData = (char)(item->valueint);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
}else if(item->type == cJSON_NULL){ }else if(item->type == cJSON_NULL){
char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL}; char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES); *valueType = TSDB_DATA_TYPE_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
} }
else{ else{
retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
@ -364,10 +361,11 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
} }
if(taosHashGetSize(keyHash) == 0){ // set json NULL true if(taosHashGetSize(keyHash) == 0){ // set json NULL true
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
} }
end: end:
taosMemoryFree(tagKV);
taosHashCleanup(keyHash); taosHashCleanup(keyHash);
cJSON_Delete(root); cJSON_Delete(root);
return retCode; return retCode;

4
source/libs/scalar/inc/sclvector.h
View File

@ -53,6 +53,8 @@ static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index)
return (double)*((double *)src + index); return (double)*((double *)src + index);
} }
double getVectorDoubleValue_JSON(void *src, int32_t index);
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL; _getDoubleValue_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) { if (srcType == TSDB_DATA_TYPE_TINYINT) {
@ -77,6 +79,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
p = getVectorDoubleValue_DOUBLE; p = getVectorDoubleValue_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorDoubleValue_BIGINT; p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_JSON) {
p = getVectorDoubleValue_JSON;
} else { } else {
assert(0); assert(0);
} }

9
source/libs/scalar/src/filter.c
View File

@ -170,7 +170,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch,
compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
}; };
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@ -221,7 +221,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
assert(0); assert(0);
} }
} }
if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
return 28;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break;

77
source/libs/scalar/src/scalar.c
View File

@ -600,44 +600,39 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { //EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
STargetNode *target = (STargetNode *)pNode; // STargetNode *target = (STargetNode *)pNode;
//
if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { // if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); // sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; // ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; // return DEAL_RES_ERROR;
} // }
//
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); // SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { // if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); // sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; // ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; // return DEAL_RES_ERROR;
} // }
//
SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); // // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); //
if (NULL == res) { // SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); // if (NULL == res) {
ctx->code = TSDB_CODE_QRY_APP_ERROR; // sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
return DEAL_RES_ERROR; // ctx->code = TSDB_CODE_QRY_APP_ERROR;
} // return DEAL_RES_ERROR;
// }
for (int32_t i = 0; i < res->numOfRows; ++i) { //
if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) { // colDataAssign(col, res->columnData, res->numOfRows);
colDataAppend(col, i, NULL, true); // block->info.rows = res->numOfRows;
} else { //
char *p = colDataGetData(res->columnData, i); // sclFreeParam(res);
colDataAppend(col, i, p, false); // taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
} // return DEAL_RES_CONTINUE;
} //}
sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
return DEAL_RES_CONTINUE;
}
EDealRes sclCalcWalker(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@ -657,9 +652,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
return sclWalkOperator(pNode, ctx); return sclWalkOperator(pNode, ctx);
} }
if (QUERY_NODE_TARGET == nodeType(pNode)) { // if (QUERY_NODE_TARGET == nodeType(pNode)) {
return sclWalkTarget(pNode, ctx); // return sclWalkTarget(pNode, ctx);
} // }
sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = TSDB_CODE_QRY_INVALID_INPUT;

283
source/libs/scalar/src/sclvector.c
View File

@ -177,24 +177,24 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v);
} }
static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { //static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
int32_t len = 0; // int32_t len = 0;
int32_t inputLen = varDataLen(buf); // int32_t inputLen = varDataLen(buf);
//
char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); // char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
/*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); // /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
varDataSetLen(t, len); // varDataSetLen(t, len);
//
colDataAppend(pOut->columnData, rowIndex, t, false); // colDataAppend(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t); // taosMemoryFree(t);
} //}
//TODO opt performance, tmp is not needed. //TODO opt performance, tmp is not needed.
int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
int32_t bufSize = pIn->columnData->info.bytes; int32_t bufSize = pIn->columnData->info.bytes;
char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
bool vton = false; // bool vton = false;
_bufConverteFunc func = NULL; _bufConverteFunc func = NULL;
if (TSDB_DATA_TYPE_BOOL == outType) { if (TSDB_DATA_TYPE_BOOL == outType) {
@ -205,9 +205,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
func = varToUnsigned; func = varToUnsigned;
} else if (IS_FLOAT_TYPE(outType)) { } else if (IS_FLOAT_TYPE(outType)) {
func = varToFloat; func = varToFloat;
} else if (outType == TSDB_DATA_TYPE_NCHAR) { // } else if (outType == TSDB_DATA_TYPE_NCHAR) { // can not be nchar or binary
func = varToNchar; // func = varToNchar;
vton = true; // vton = true;
} else { } else {
sclError("invalid convert outType:%d", outType); sclError("invalid convert outType:%d", outType);
return TSDB_CODE_QRY_APP_ERROR; return TSDB_CODE_QRY_APP_ERROR;
@ -221,9 +221,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
} }
char* data = colDataGetData(pIn->columnData, i); char* data = colDataGetData(pIn->columnData, i);
if (vton) { // if (vton) {
memcpy(tmp, data, varDataTLen(data)); // memcpy(tmp, data, varDataTLen(data));
} else { // } else {
if (TSDB_DATA_TYPE_VARCHAR == inType) { if (TSDB_DATA_TYPE_VARCHAR == inType) {
memcpy(tmp, varDataVal(data), varDataLen(data)); memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0; tmp[varDataLen(data)] = 0;
@ -239,7 +239,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
tmp[len] = 0; tmp[len] = 0;
} }
} // }
(*func)(tmp, pOut, i); (*func)(tmp, pOut, i);
} }
@ -248,6 +248,135 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_INT: {
GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UTINYINT: {
GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UINT: {
GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
GET_TYPED_DATA(*((float *)outData), float, inType, inData);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(*((double *)outData), double, inType, inData);
break;
}
default:{
ASSERT(0);
}
}
}
void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
char *tmp = taosMemoryMalloc(varDataTLen(inData));
if (inType == TSDB_DATA_TYPE_NCHAR) {
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
}
tmp[len] = 0;
} else {
memcpy(tmp, varDataVal(inData), varDataLen(inData));
tmp[varDataLen(inData)] = 0;
}
ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
double value = strtod(tmp, NULL);
*((double *)outData) = value;
taosMemoryFreeClear(tmp);
}
double getVectorDoubleValue_JSON(void *src, int32_t index){
ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index));
char *data = colDataGetData((SColumnInfoData*)src, index);
double out = 0;
if(*data == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
} {
convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
}
return out;
}
void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){
if(optr == OP_TYPE_JSON_CONTAINS) {
return;
}
if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
return;
}
if(typeLeft == TSDB_DATA_TYPE_JSON){
typeLeft = **pLeftData;
(*pLeftData) ++;
}
if(typeRight == TSDB_DATA_TYPE_JSON){
typeRight = **pRightData;
(*pRightData) ++;
}
int8_t type = vectorGetConvertType(typeLeft, typeRight);
if(type == 0) {
*fp = filterGetCompFunc(typeLeft, optr);
return;
}
*fp = filterGetCompFunc(type, optr);
if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
} else if(typeLeft != type) {
convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
*pLeftData = pLeftOut;
}
if(IS_VAR_DATA_TYPE(typeRight)) {
convertStringToDouble(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
} else if(typeRight != type) {
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
}
// TODO opt performance // TODO opt performance
int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
SColumnInfoData* pInputCol = pIn->columnData; SColumnInfoData* pInputCol = pIn->columnData;
@ -522,7 +651,7 @@ enum {
static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
SColumnInfoData* pCol = pParam->columnData; SColumnInfoData* pCol = pParam->columnData;
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) {
pDest->numOfRows = pParam->numOfRows; pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@ -556,7 +685,8 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0); *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+ getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull; pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) { if (pOutputCol->hasNull) {
@ -587,6 +717,36 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
} }
} }
char *getJsonValue(char *json, char *key){ //todo
return NULL;
}
void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData;
int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
char *pRightData = colDataGetData(pRight->columnData, 0);
for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (pLeft->columnData->varmeta.offset[i] == -1) {
pOutputCol->varmeta.offset[i] = -1;
pOutputCol->hasNull = true;
continue;
}
char *pLeftData = colDataGetData(pLeft->columnData, i);
char *value = getJsonValue(pLeftData, pRightData);
if (!value || *value == TSDB_DATA_TYPE_NULL) {
pOutputCol->varmeta.offset[i] = -1;
pOutputCol->hasNull = true;
continue;
}
colDataAppend(pOutputCol, i, pLeftData, false);
}
}
void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
SColumnInfoData *pOutputCol = pOut->columnData; SColumnInfoData *pOutputCol = pOut->columnData;
@ -605,7 +765,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+ getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@ -637,7 +802,8 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor; *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
- getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor;
} }
pOutputCol->hasNull = pLeftCol->hasNull; pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) { if (pOutputCol->hasNull) {
@ -664,7 +830,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
- getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@ -696,7 +867,8 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
colDataAppendNNULL(pOutputCol, 0, numOfRows); colDataAppendNNULL(pOutputCol, 0, numOfRows);
} else { } else {
for (; i >= 0 && i < numOfRows; i += step, output += 1) { for (; i >= 0 && i < numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0); *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
* getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull; pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) { if (pOutputCol->hasNull) {
@ -722,7 +894,12 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
* getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@ -760,7 +937,12 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
/getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
} }
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@ -776,7 +958,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else { } else {
for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i); *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0)
/ getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
} }
pOutputCol->hasNull = pRightCol->hasNull; pOutputCol->hasNull = pRightCol->hasNull;
if (pOutputCol->hasNull) { if (pOutputCol->hasNull) {
@ -788,7 +971,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else { } else {
for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0); *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
/ getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
} }
pOutputCol->hasNull = pLeftCol->hasNull; pOutputCol->hasNull = pLeftCol->hasNull;
if (pOutputCol->hasNull) { if (pOutputCol->hasNull) {
@ -825,8 +1009,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
continue; continue;
} }
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -835,7 +1019,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
*output = lx - ((int64_t)(lx / rx)) * rx; *output = lx - ((int64_t)(lx / rx)) * rx;
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0);
if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
} else { } else {
@ -845,7 +1029,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
continue; continue;
} }
double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -855,7 +1039,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
} }
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value
colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
} else { } else {
@ -865,7 +1049,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
continue; continue;
} }
double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
if (isnan(lx) || isinf(lx)) { if (isnan(lx) || isinf(lx)) {
colDataAppendNULL(pOutputCol, i); colDataAppendNULL(pOutputCol, i);
continue; continue;
@ -895,7 +1079,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
double *output = (double *)pOutputCol->pData; double *output = (double *)pOutputCol->pData;
for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
*output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i); if (colDataIsNull_s(pLeft->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
} }
pOutputCol->hasNull = pLeftCol->hasNull; pOutputCol->hasNull = pLeftCol->hasNull;
@ -1099,30 +1287,39 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
char *pLeftData = colDataGetData(pLeft->columnData, i); char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, i); char *pRightData = colDataGetData(pRight->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
} else if (pRight->numOfRows == 1) { } else if (pRight->numOfRows == 1) {
char *pRightData = colDataGetData(pRight->columnData, 0);
ASSERT(pLeft->pHashFilter == NULL); ASSERT(pLeft->pHashFilter == NULL);
for (; i >= 0 && i < pLeft->numOfRows; i += step) { for (; i >= 0 && i < pLeft->numOfRows; i += step) {
if (colDataIsNull_s(pLeft->columnData, i)) { if (colDataIsNull_s(pLeft->columnData, i)) {
continue; continue;
} }
char *pLeftData = colDataGetData(pLeft->columnData, i); char *pLeftData = colDataGetData(pLeft->columnData, i);
char *pRightData = colDataGetData(pRight->columnData, 0);
int64_t leftOut = 0;
int64_t rightOut = 0;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
} else if (pLeft->numOfRows == 1) { } else if (pLeft->numOfRows == 1) {
char *pLeftData = colDataGetData(pLeft->columnData, 0);
for (; i >= 0 && i < pRight->numOfRows; i += step) { for (; i >= 0 && i < pRight->numOfRows; i += step) {
if (colDataIsNull_s(pRight->columnData, i)) { if (colDataIsNull_s(pRight->columnData, i)) {
continue; continue;
} }
char *pLeftData = colDataGetData(pLeft->columnData, 0);
char *pRightData = colDataGetData(pLeft->columnData, i); char *pRightData = colDataGetData(pLeft->columnData, i);
int64_t leftOut = 0;
int64_t rightOut = 0;
convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
bool res = filterDoCompare(fp, optr, pLeftData, pRightData); bool res = filterDoCompare(fp, optr, pLeftData, pRightData);
colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
} }
@ -1203,6 +1400,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
} }
void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
}
void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
for(int32_t i = 0; i < pLeft->numOfRows; ++i) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
@ -1271,6 +1472,10 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
return vectorBitOr; return vectorBitOr;
case OP_TYPE_IS_TRUE: case OP_TYPE_IS_TRUE:
return vectorIsTrue; return vectorIsTrue;
case OP_TYPE_JSON_GET_VALUE:
return vectorJsonArrow;
case OP_TYPE_JSON_CONTAINS:
return vectorJsonContains;
default: default:
assert(0); assert(0);
return NULL; return NULL;

5
source/util/src/tcompare.c
View File

@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
return compareLenPrefixedWStr(pRight, pLeft); return compareLenPrefixedWStr(pRight, pLeft);
} }
int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
if(pLeft) return 0;
return 1;
}
/* /*
* Compare two strings * Compare two strings
* TSDB_MATCH: Match * TSDB_MATCH: Match