From 2133a2de1d5ab0c761bf9fb9b85c0f5c139beba8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Apr 2022 20:30:06 +0800 Subject: [PATCH] refactor: add test casese for json compare logic --- include/common/tdataformat.h | 2 +- source/common/src/tdatablock.c | 21 ++- source/libs/scalar/src/sclvector.c | 4 +- source/libs/scalar/test/scalar/CMakeLists.txt | 3 +- .../libs/scalar/test/scalar/scalarTests.cpp | 143 +++++++----------- 5 files changed, 80 insertions(+), 93 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2dd887ae74..a0f4351bbf 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -502,7 +502,7 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowLen(r) (*(uint16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 14b7fe5d0e..5bdf7ddb85 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if(type == TSDB_DATA_TYPE_JSON) { + if(*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + }else if(*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData+CHAR_BYTES); + }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + }else if(*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } SVarColAttr* pAttr = &pColumnInfoData->varmeta; - if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) { + if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; if (newSize == 0) { newSize = 8; } - while (newSize < pAttr->length + varDataTLen(pData)) { + while (newSize < pAttr->length + dataLen) { newSize = newSize * 1.5; } @@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); - pColumnInfoData->varmeta.length += varDataTLen(pData); + memcpy(pColumnInfoData->pData + len, pData, dataLen); + pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 7ad200befc..c2eafedf9b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -774,7 +774,7 @@ char *getJsonValue(char *json, char *key){ //todo if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ return NULL; } - if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){ + if(memcmp(key, data, varDataTLen(data)) == 0){ return data + varDataTLen(data); } } @@ -803,7 +803,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pOutputCol->hasNull = true; continue; } - colDataAppend(pOutputCol, i, pLeftData, false); + colDataAppend(pOutputCol, i, value, false); } } diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 6cbac1e25d..03418d17ac 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( scalarTest - PUBLIC os util common gtest qcom function nodes scalar + PUBLIC os util common gtest qcom function nodes scalar parser ) TARGET_INCLUDE_DIRECTORIES( scalarTest PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/" + PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 975effc63a..8049e80590 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -125,56 +125,6 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); - SColumnNode *rnode = (SColumnNode *)node; - rnode->node.resType.type = dataType; - rnode->node.resType.bytes = dataBytes; - rnode->dataBlockId = 0; - - if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - } - - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - - blockDataEnsureCapacity(res, rowNum); - - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - for (int32_t i = 0; i < rowNum; ++i) { - uint32_t len = pColumnInfoData->varmeta.length; - pColumnInfoData->varmeta.offset[i] = len; - - memcpy(pColumnInfoData->pData + len, value, varDataLen(value)); - pColumnInfoData->varmeta.length += varDataLen(value); - } - - rnode->slotId = 2; - rnode->colId = 3; - - *block = res; - } - - *pNode = (SNode *)rnode; -} void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); @@ -960,54 +910,77 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } -TEST(columnTest, smallint_value_add_json_column) { - scltInitLogFile(); +void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ + char key[32] = {0}; + memcpy(varDataVal(key), key, strlen(key)); + varDataLen(key) = strlen(key); SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; - int32_t leftv = 1; - char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}"; - char key[10] = {0}; - memcpy(varDataVal(key), "k1", 2); - varDataLen(key) = 2; + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); + scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json); + scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); +} + +void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){ + SNode *right = NULL; + scltMakeValueNode(&right, rightType, rightData); + scltMakeOpNode(opNode, opType, resType, *opNode, right); + + SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo); + scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); +} + +void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){ + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + SSDataBlock *src = NULL; + SNode *opNode = NULL; + + makeJsonArrow(&src, &opNode, row, key); + taosArrayPush(blockList, &src); + + makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value); + + int32_t code = scalarCalculate(opNode, blockList, NULL); + ASSERT_EQ(code, 0); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, 1); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue); + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); +} + +TEST(columnTest, int_value_add_json_column) { + scltInitLogFile(); + char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; + char *key = "k1"; SKVRowBuilder kvRowBuilder; tdInitKVRowBuilder(&kvRowBuilder); parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - double eRes[5] = {1.0, -4, -3, 24, 101}; + int32_t leftv = 1; + makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD); - SSDataBlock *src = NULL; - int32_t rowNum = 1; + leftv = 8; + makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB); - scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); - scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row); - scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); + leftv = 2; + makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI); - SNode *right = NULL; - scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv); - scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right); + leftv = 2; + makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV); - SArray *blockList = taosArrayInit(2, POINTER_BYTES); - taosArrayPush(blockList, &src); - SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - int16_t dataBlockId = 0, slotId = 0; - scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); - scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + leftv = 3; + makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE); - int32_t code = scalarCalculate(opNode, blockList, NULL); - ASSERT_EQ(code, 0); - - SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); - ASSERT_EQ(res->info.rows, rowNum); - SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); - for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); - } - - taosArrayDestroyEx(blockList, scltFreeDataBlock); - nodesDestroyNode(opNode); + makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS); } TEST(columnTest, smallint_value_add_int_column) {