xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: add test casese for json compare logic

This commit is contained in:
wangmm0220 2022-04-19 17:52:04 +08:00
parent b96082fe1b
commit 412e2cbb0d
5 changed files with 177 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/util/tdef.h
View File

@ -272,11 +272,6 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_JSON_PLACEHOLDER 0x7F
#define TSDB_JSON_null 0x00
#define TSDB_JSON_NOT_NULL 0x01
#define TSDB_JSON_NULL 0x00
#define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32

4
source/libs/parser/src/parUtil.c
View File

@ -276,7 +276,7 @@ static bool isValidateTag(char *input) {
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
// set json NULL data
uint8_t jsonNULL = TSDB_JSON_NULL;
uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
int jsonIndex = startColId + 1;
if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
@ -328,7 +328,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
strncpy(varDataVal(tagKV), jsonKey, keyLen);
varDataSetLen(tagKV, keyLen);
if(taosHashGetSize(keyHash) == 0){
uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type
}
taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless

88
source/libs/scalar/src/scalar.c
View File

@ -604,39 +604,55 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE;
}
//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
// STargetNode *target = (STargetNode *)pNode;
//
// if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
// sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
// ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
// return DEAL_RES_ERROR;
// }
//
// SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
// if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
// sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
// ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
// return DEAL_RES_ERROR;
// }
//
// // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
//
// SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
// if (NULL == res) {
// sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
// ctx->code = TSDB_CODE_QRY_APP_ERROR;
// return DEAL_RES_ERROR;
// }
//
// colDataAssign(col, res->columnData, res->numOfRows);
// block->info.rows = res->numOfRows;
//
// sclFreeParam(res);
// taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
// return DEAL_RES_CONTINUE;
//}
EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
STargetNode *target = (STargetNode *)pNode;
if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
int32_t index = -1;
for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i);
if (pb->info.blockId == target->dataBlockId) {
index = i;
break;
}
}
if (index == -1) {
sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
// if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
if (NULL == res) {
sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
ctx->code = TSDB_CODE_QRY_APP_ERROR;
return DEAL_RES_ERROR;
}
colDataAssign(col, res->columnData, res->numOfRows);
block->info.rows = res->numOfRows;
sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
return DEAL_RES_CONTINUE;
}
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@ -656,9 +672,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
return sclWalkOperator(pNode, ctx);
}
// if (QUERY_NODE_TARGET == nodeType(pNode)) {
// return sclWalkTarget(pNode, ctx);
// }
if (QUERY_NODE_TARGET == nodeType(pNode)) {
return sclWalkTarget(pNode, ctx);
}
sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;

23
source/libs/scalar/src/sclvector.c
View File

@ -767,6 +767,17 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
}
char *getJsonValue(char *json, char *key){ //todo
int16_t cols = kvRowNCols(json);
for (int i = 0; i < cols; ++i) {
SColIdx *pColIdx = kvRowColIdxAt(json, i);
char *data = kvRowColVal(json, pColIdx);
if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
return NULL;
}
if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){
return data + varDataTLen(data);
}
}
return NULL;
}
@ -844,7 +855,11 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@ -957,7 +972,11 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
if (pLeft->numOfRows == pRight->numOfRows) {
for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
*output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
*output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
}
pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);

103
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -92,6 +92,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
*slotId = 0;
} else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
@ -124,6 +125,56 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
*pNode = (SNode *)vnode;
}
void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
SColumnNode *rnode = (SColumnNode *)node;
rnode->node.resType.type = dataType;
rnode->node.resType.bytes = dataBytes;
rnode->dataBlockId = 0;
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}};
idata.info.type = TSDB_DATA_TYPE_NULL;
idata.info.bytes = 10;
idata.info.colId = i + 1;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
}
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 3;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
blockDataEnsureCapacity(res, rowNum);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) {
uint32_t len = pColumnInfoData->varmeta.length;
pColumnInfoData->varmeta.offset[i] = len;
memcpy(pColumnInfoData->pData + len, value, varDataLen(value));
pColumnInfoData->varmeta.length += varDataLen(value);
}
rnode->slotId = 2;
rnode->colId = 3;
*block = res;
}
*pNode = (SNode *)rnode;
}
void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@ -909,6 +960,56 @@ TEST(constantTest, greater_and_lower) {
nodesDestroyNode(res);
}
TEST(columnTest, smallint_value_add_json_column) {
scltInitLogFile();
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int32_t leftv = 1;
char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}";
char key[10] = {0};
memcpy(varDataVal(key), "k1", 2);
varDataLen(key) = 2;
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
double eRes[5] = {1.0, -4, -3, 24, 101};
SSDataBlock *src = NULL;
int32_t rowNum = 1;
scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row);
scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
SNode *right = NULL;
scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv);
scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right);
SArray *blockList = taosArrayInit(2, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0);
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
ASSERT_EQ(res->info.rows, rowNum);
SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]);
}
taosArrayDestroyEx(blockList, scltFreeDataBlock);
nodesDestroyNode(opNode);
}
TEST(columnTest, smallint_value_add_int_column) {
scltInitLogFile();
@ -928,7 +1029,7 @@ TEST(columnTest, smallint_value_add_int_column) {
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
int32_t code = scalarCalculate(opNode, blockList, NULL);
ASSERT_EQ(code, 0);