From 5a383152c4d3ab5e4c1e90c4bda35f60c319f801 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Feb 2022 19:45:05 +0800 Subject: [PATCH 01/12] feature/qnode --- include/common/tmsgdef.h | 2 +- include/util/taoserror.h | 5 + include/util/tdef.h | 15 ++ source/common/src/tmsg.c | 4 + source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mnode/impl/inc/mndInfoSchema.h | 45 ++++ source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 54 +++-- source/dnode/mnode/impl/src/mndInfoSchema.c | 247 ++++++++++++++++++++ source/dnode/mnode/impl/src/mndStb.c | 22 +- source/dnode/mnode/impl/src/mnode.c | 16 +- source/dnode/mnode/impl/test/stb/stb.cpp | 2 +- source/libs/catalog/inc/catalogInt.h | 19 +- source/libs/catalog/src/catalog.c | 98 ++++---- source/libs/catalog/test/catalogTests.cpp | 1 + source/libs/qcom/src/querymsg.c | 4 +- source/util/src/terror.c | 4 + 17 files changed, 450 insertions(+), 91 deletions(-) create mode 100644 source/dnode/mnode/impl/inc/mndInfoSchema.h create mode 100644 source/dnode/mnode/impl/src/mndInfoSchema.c diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a3e44968f7..b5f9b9937f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -127,7 +127,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STB_META, "mnode-stb-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2f7d406569..41cd89ecb5 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -237,6 +237,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +// mnode-infoSchema +#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) + // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C1) @@ -267,6 +270,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) + + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) diff --git a/include/util/tdef.h b/include/util/tdef.h index d0f2b77f1f..ae775d5f53 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -96,6 +96,21 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" +#define TSDB_INFORMATION_SCHEMA_DB "information_schema" +#define TSDB_INS_TABLE_DNODES "dnodes" +#define TSDB_INS_TABLE_MNODES "mnodes" +#define TSDB_INS_TABLE_MODULES "modules" +#define TSDB_INS_TABLE_QNODES "qnodes" +#define TSDB_INS_TABLE_USER_DATABASE "user_database" +#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" +#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" +#define TSDB_INS_TABLE_USER_STABLES "user_stables" +#define TSDB_INS_TABLE_USER_STREAMS "user_streams" +#define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" +#define TSDB_INS_TABLE_USER_USERS "user_users" +#define TSDB_INS_TABLE_VGROUPS "vgroups" + #define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8101c18ebf..022b8bda90 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1523,6 +1523,10 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + if (pRsp->vgNum <= 0) { + return 0; + } + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); if (pRsp->pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 0aae145d2f..ee933b222d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -95,7 +95,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STB_META)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; diff --git a/source/dnode/mnode/impl/inc/mndInfoSchema.h b/source/dnode/mnode/impl/inc/mndInfoSchema.h new file mode 100644 index 0000000000..7db4125402 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndInfoSchema.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_INFO_SCHEMA_H_ +#define _TD_MND_INFO_SCHEMA_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SInfosTableSchema { + char *name; + int32_t type; + int32_t bytes; +} SInfosTableSchema; + +typedef struct SInfosTableMeta { + char *name; + const SInfosTableSchema *schema; + int32_t colNum; +} SInfosTableMeta; + +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); +int32_t mndInitInfos(SMnode *pMnode); +void mndCleanupInfos(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_INFO_SCHEMA_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 5c32da966b..04742143ca 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -89,6 +89,7 @@ typedef struct SMnode { SProfileMgmt profileMgmt; STelemMgmt telemMgmt; SSyncMgmt syncMgmt; + SHashObj *infosMeta; MndMsgFp msgFp[TDMT_MAX]; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c8a4ce586..923f838d34 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -937,36 +937,40 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { goto USE_DB_OVER; } - pDb = mndAcquireDb(pMnode, usedbReq.db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto USE_DB_OVER; - } + if (0 == strcmp(usedbReq.db, TSDB_INFORMATION_SCHEMA_DB)) { + memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + } else { + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto USE_DB_OVER; - } + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } - if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; - } + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; - } + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } - if (usedbReq.vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - } + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c new file mode 100644 index 0000000000..b9ae707c7b --- /dev/null +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndInfoSchema.h" + +static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vnodes", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cores", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "offline_reason", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, + {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; + +static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, + {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, + {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, + {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, + {TSDB_INS_TABLE_USER_DATABASE, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, + {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, + {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, + {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)}, + {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, + {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, + }; + + +int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { + SSchema *schema = calloc(colNum, sizeof(SSchema)); + if (NULL == schema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + + for (int32_t i = 0; i < colNum; ++i) { + strcpy(schema->name, pSrc->name); + + schema->type = pSrc->type; + schema->colId = i + 1; + schema->bytes = pSrc->bytes; + + pSrc++; + } + + *pDst = schema; + + return TSDB_CODE_SUCCESS; +} + +int32_t mndInsInitMeta(SHashObj *hash) { + STableMetaRsp meta = {0}; + + strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); + meta.tableType = TSDB_NORMAL_TABLE; + meta.sversion = 1; + meta.tversion = 1; + + for (int32_t i = 0; i < tListLen(infosMeta); ++i) { + strcpy(meta.tbName, infosMeta[i].name); + meta.numOfColumns = infosMeta[i].colNum; + + if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) { + return -1; + } + + if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { + if (NULL == pMnode->infosMeta) { + terrno = TSDB_CODE_MND_NOT_READY; + return -1; + } + + STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); + if (NULL == meta) { + mError("invalid information schema table name:%s", tbName); + terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + return -1; + } + + *pRsp = *meta; + + pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pRsp->pSchemas = NULL; + return -1; + } + + memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); + + return 0; +} + +int32_t mndInitInfos(SMnode *pMnode) { + pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pMnode->infosMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return mndInsInitMeta(pMnode->infosMeta); +} + +void mndCleanupInfos(SMnode *pMnode) { + if (NULL == pMnode->infosMeta) { + return; + } + + void *pIter = taosHashIterate(pMnode->infosMeta, NULL); + while (pIter) { + STableMetaRsp *meta = (STableMetaRsp *)pIter; + + tfree(meta->pSchemas); + + pIter = taosHashIterate(pMnode->infosMeta, pIter); + } + + taosHashCleanup(pMnode->infosMeta); + pMnode->infosMeta = NULL; +} + + + diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 274ae10045..d8091dfa76 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" #include "tname.h" #define TSDB_STB_VER_NUMBER 1 @@ -38,7 +39,7 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq); +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -58,7 +59,7 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STB_META, mndProcessStbMetaReq); + mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -1310,7 +1311,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; int32_t code = -1; STableInfoReq infoReq = {0}; @@ -1321,9 +1322,16 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { goto RETRIEVE_META_OVER; } - mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + mDebug("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } + } else { + mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } } int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); @@ -1553,4 +1561,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 299e66a7c0..70b7229851 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -36,6 +36,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -221,6 +222,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { @@ -521,9 +523,17 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } } + +// Note: uid 0 is reserved uint64_t mndGenerateUid(char *name, int32_t len) { - int64_t us = taosGetTimestampUs(); int32_t hashval = MurmurHash3_32(name, len); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + + do { + int64_t us = taosGetTimestampUs(); + uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; + uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + if (uuid) { + return uuid; + } + } while (true); } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c551097e72..f45c0795cd 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -339,7 +339,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { void* pReq = rpcMallocCont(contLen); tSerializeSTableInfoReq(pReq, contLen, &infoReq); - SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5857437f9d..72e17ba363 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -189,11 +189,20 @@ typedef struct SCtgAction { #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) -#define CTG_IS_STABLE(isSTable) (1 == (isSTable)) -#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable)) -#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0) -#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) -#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_FLAG_STB 0x1 +#define CTG_FLAG_NOT_STB 0x2 +#define CTG_FLAG_UNKNOWN_STB 0x4 +#define CTG_FLAG_INF_DB 0x8 + +#define CTG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) +#define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) +#define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) +#define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) +#define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) +#define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) + +#define CTG_IS_INF_DBNAME(_sname) ((_sname)->dbname[0] == 'i' && 0 == strcmp((_sname)->dbname, TSDB_INFORMATION_SCHEMA_DB)) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d62c189d33..0a3ed0e489 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -629,7 +629,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -637,21 +637,21 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { ctgError("Build mnode stablemeta msg failed, code:%x", code); CTG_ERR_RET(code); } SRpcMsg rpcMsg = { - .msgType = TDMT_MND_STB_META, + .msgType = TDMT_MND_TABLE_META, .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -664,7 +664,7 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S CTG_ERR_RET(rpcRsp.code); } - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); @@ -675,15 +675,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -709,7 +709,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSe }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -829,7 +829,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { +int32_t ctgStbVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { @@ -1100,7 +1100,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -1258,7 +1258,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); } } @@ -1428,15 +1428,17 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + if (!CTG_IS_INF_DB(flag)) { + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); + } SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; @@ -1445,33 +1447,37 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - - if (CTG_IS_STABLE(isSTable)) { + + if (CTG_IS_INF_DB(flag)) { + ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); + + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); + } else if (CTG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); if (CTG_IS_META_NULL(output->metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); tfree(output->tbMeta); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); @@ -1530,7 +1536,7 @@ _return: CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1538,26 +1544,26 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t exist = 0; int32_t code = 0; - if (!forceUpdate) { + if ((!forceUpdate) || (CTG_IS_INF_DBNAME(pTableName))) { CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); - if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { + if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } tfree(*pTableMeta); - } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { + } else if (CTG_IS_UNKNOWN_STB(flag)) { int32_t tbType = 0; CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); - CTG_SET_STABLE(isSTable, tbType); + CTG_SET_STB(flag, tbType); } STableMetaOutput *output = NULL; while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; @@ -1729,7 +1735,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionCompare)); ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2140,16 +2146,16 @@ _return: } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_STB)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2204,20 +2210,20 @@ _return: } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_GEN_STB_FLAG(isSTable), NULL)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, true, pTableMeta, CTG_GEN_STB_FLAG(isSTable))); } int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { @@ -2236,7 +2242,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -2307,7 +2313,7 @@ _return: } -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); SCtgDBCache* dbCache = NULL; @@ -2316,7 +2322,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpS tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, false, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); @@ -2336,10 +2342,10 @@ _return: } -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2363,7 +2369,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, false, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebe20fbb7f..d387fdf495 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -827,6 +827,7 @@ void *ctgTestSetCtableMetaThread(void *param) { #if 0 + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 753ac75860..d7bbd4660c 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -254,11 +254,11 @@ PROCESS_META_OVER: void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; - queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)] = queryProcessTableMetaRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e6bef5f8b9..f30dc37e4e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -245,6 +245,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +// mnode-infoSchema +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name") + + // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") From bf4efa77cb6d481cd1b77dadcca4a2e3badb1b4c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Feb 2022 14:30:16 +0800 Subject: [PATCH 02/12] feature/qnode --- source/common/src/tname.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 3 +- source/dnode/mnode/impl/src/mndInfoSchema.c | 168 ++++++++++---------- source/libs/catalog/inc/catalogInt.h | 3 +- source/libs/catalog/src/catalog.c | 49 ++++-- source/libs/catalog/test/catalogTests.cpp | 7 +- source/libs/qcom/src/querymsg.c | 4 +- 7 files changed, 137 insertions(+), 101 deletions(-) diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f6892b26bd..480f5150a2 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -111,7 +111,9 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { return -1; } - int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + int32_t len = 0; + + snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 923f838d34..9165fa2264 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -937,7 +937,8 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { goto USE_DB_OVER; } - if (0 == strcmp(usedbReq.db, TSDB_INFORMATION_SCHEMA_DB)) { + char *p = strchr(usedbReq.db, '.'); + if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); } else { pDb = mndAcquireDb(pMnode, usedbReq.db); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index b9ae707c7b..2c391e93e8 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -28,100 +28,100 @@ static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .byt static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, - {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, + {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; -static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, @@ -149,13 +149,11 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S for (int32_t i = 0; i < colNum; ++i) { - strcpy(schema->name, pSrc->name); + strcpy(schema[i].name, pSrc[i].name); - schema->type = pSrc->type; - schema->colId = i + 1; - schema->bytes = pSrc->bytes; - - pSrc++; + schema[i].type = pSrc[i].type; + schema[i].colId = i + 1; + schema[i].bytes = pSrc[i].bytes; } *pDst = schema; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 72e17ba363..975f90366e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -198,11 +198,12 @@ typedef struct SCtgAction { #define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) #define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) #define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) #define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) #define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_IS_INF_DBNAME(_sname) ((_sname)->dbname[0] == 'i' && 0 == strcmp((_sname)->dbname, TSDB_INFORMATION_SCHEMA_DB)) +#define CTG_IS_INF_DBNAME(_dbname) ((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0a3ed0e489..2b6caa2f8f 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -509,7 +509,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); @@ -517,7 +517,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } *pTableMeta = NULL; @@ -590,15 +594,19 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType, int32_t flag) { if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } + SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -1078,6 +1086,10 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + if (CTG_IS_INF_DBNAME(dbFName)) { + return TSDB_CODE_SUCCESS; + } + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); @@ -1544,8 +1556,12 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t exist = 0; int32_t code = 0; - if ((!forceUpdate) || (CTG_IS_INF_DBNAME(pTableName))) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + CTG_SET_INF_DB(flag); + } + + if ((!forceUpdate) || (CTG_IS_INF_DB(flag))) { + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag)); if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; @@ -1555,7 +1571,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons } else if (CTG_IS_UNKNOWN_STB(flag)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType, flag)); CTG_SET_STB(flag, tbType); } @@ -1588,7 +1604,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag)); if (0 == exist) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; @@ -1669,6 +1685,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); @@ -2232,6 +2253,11 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } STableMeta *tbMeta = NULL; int32_t code = 0; @@ -2316,6 +2342,11 @@ _return: int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index d387fdf495..f3aecadd3f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist); + int32_t *exist, int32_t flag); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgDbgEnableDebug(char *option); @@ -243,6 +243,8 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->suid = ctgTestSuid + 1; rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; + + rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -770,7 +772,7 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0); if (code || 0 == exist) { assert(0); } @@ -1290,6 +1292,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + tfree(rsp.pSchemas); while (true) { uint64_t n = 0; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index d7bbd4660c..8bc29656d4 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -198,7 +198,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl } int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { - int32_t code = -1; + int32_t code = 0; STableMetaRsp metaRsp = {0}; if (NULL == output || NULL == msg || msgSize <= 0) { @@ -216,7 +216,7 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_META_OVER; } - if (!tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { + if (0 != strcmp(metaRsp.dbFName, TSDB_INFORMATION_SCHEMA_DB) && !tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { code = TSDB_CODE_TSC_INVALID_VALUE; goto PROCESS_META_OVER; } From 7129ba9cbdfa73b8a18a7f758bb2c0d1e65da791 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Feb 2022 19:15:23 +0800 Subject: [PATCH 03/12] feature/qnode --- include/libs/function/function.h | 12 +- source/libs/scalar/inc/sclvector.h | 1 + source/libs/scalar/src/scalar.c | 60 ++++++---- source/libs/scalar/src/sclvector.c | 183 ++++++++++++++++++----------- 4 files changed, 159 insertions(+), 97 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e970a0d693..2b03fbd933 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -227,11 +227,13 @@ typedef struct SAggFunctionInfo { } SAggFunctionInfo; typedef struct SScalarParam { - void* data; - bool colData; - int32_t num; - int32_t type; - int32_t bytes; + void *data; + SColumnInfoData *columnData; + char *bitmap; + bool dataInBlock; + int32_t num; + int32_t type; + int32_t bytes; } SScalarParam; typedef struct SScalarFunctionInfo { diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 55c4828745..fd0f8b896f 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -22,6 +22,7 @@ extern "C" { #include "sclfunc.h" +typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 742c7fd706..05af5f63b4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -87,6 +87,18 @@ _return: SCL_RET(code); } +bool sclIsNull(SScalarParam* param, int32_t idx) { + if (param->dataInBlock) { + return colDataIsNull(param->columnData, 0, idx, NULL); + } + + return colDataIsNull_f(param->bitmap, idx); +} + +void sclSetNull(SScalarParam* param, int32_t idx) { + colDataSetNull_f(param->bitmap, idx); +} + void sclFreeRes(SHashObj *res) { SScalarParam *p = NULL; @@ -116,7 +128,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; - param->colData = false; + param->dataInBlock = false; break; } @@ -130,7 +142,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); param->num = 1; param->type = SCL_DATA_TYPE_DUMMY_HASH; - param->colData = false; + param->dataInBlock = false; break; } @@ -147,13 +159,9 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); - if (IS_VAR_DATA_TYPE(columnData->info.type)) { - param->data = columnData; - param->colData = true; - } else { - param->data = columnData->pData; - param->colData = false; - } + param->data = NULL; + param->columnData = columnData; + param->dataInBlock = true; param->num = ctx->pSrc->info.rows; param->type = columnData->info.type; @@ -192,20 +200,24 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) { SScalarParam *param = NULL; - for (int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < listNum; ++i) { param = params + i; if (1 == param->num) { continue; } - if (IS_VAR_DATA_TYPE(param->type)) { - param->data = (char *)(param->data) + varDataTLen(param->data); - } else { - param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + if (param->dataInBlock) { + param->data = colDataGet(param->columnData, idx); + } else if (idx) { + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } } } @@ -281,8 +293,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( -"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -298,15 +309,14 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu } for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( -"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } return TSDB_CODE_SUCCESS; @@ -354,6 +364,9 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o bool value = false; for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + for (int32_t m = 0; m < node->pParameterList->length; ++m) { GET_TYPED_DATA(value, bool, params[m].type, params[m].data); @@ -367,9 +380,6 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } *(bool *)output->data = value; - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } output->data = data; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 85af663313..57899f0e82 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -261,6 +261,60 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } +static FORCE_INLINE void convertToSigned(char *buf, SScalarParam* pOut, int32_t outType) { + int64_t value = strtoll(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void convertToUnsigned(char *buf, SScalarParam* pOut, int32_t outType) { + uint64_t value = strtoull(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void convertToFloat(char *buf, SScalarParam* pOut, int32_t outType) { + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); +} + + +int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { + int32_t bufSize = 0; + char *tmp = NULL; + _bufConverteFunc func = NULL; + + if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) { + func = convertToSigned; + } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + func = convertToUnsigned; + } else if (IS_FLOAT_TYPE(outType)) { + func = convertToFloat; + } else { + sclError("unknown outType:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; + } + + if (varDataLen(pIn->data) >= bufSize) { + bufSize = varDataLen(pIn->data) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); + tmp[varDataLen(pIn->data)] = 0; + + (*func)(tmp, pOut, outType); + } + + tfree(tmp); +} int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int16_t inType = pIn->type; @@ -278,65 +332,60 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + int32_t bufSize = 0; + char *tmp = NULL; for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; + } + + if (varDataLen(pIn->data) >= bufSize) { + bufSize = varDataLen(pIn->data) + 1; + tmp = realloc(tmp, bufSize); } - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); + tmp[varDataLen(pIn->data)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); } tfree(tmp); } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + int32_t bufSize = 0; + char *tmp = NULL; for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); } tfree(tmp); @@ -628,8 +677,8 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; - SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { @@ -637,7 +686,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -655,7 +704,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -719,7 +768,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -737,7 +786,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -799,7 +848,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -817,7 +866,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -881,7 +930,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -899,7 +948,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -970,7 +1019,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -988,7 +1037,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -1136,7 +1185,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -1154,7 +1203,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -1218,7 +1267,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ return; } - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } @@ -1236,7 +1285,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ return; } - if (pRight->colData) { + if (pRight->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; pRight->data = colInfo->pData; } @@ -1298,13 +1347,13 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { getVectorValueAddrFnLeft = getVectorValueAddr_default; } else { getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); } - if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { + if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->dataInBlock) { getVectorValueAddrFnRight = getVectorValueAddr_default; } else { getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -1436,7 +1485,7 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool *output=(bool *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { getVectorValueAddrFnLeft = getVectorValueAddr_default; } else { getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); @@ -1462,7 +1511,7 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool *output = (bool *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { getVectorValueAddrFnLeft = getVectorValueAddr_default; } else { getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); @@ -1483,7 +1532,7 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; - if (pLeft->colData) { + if (pLeft->dataInBlock) { SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; pLeft->data = colInfo->pData; } From 2d58e1fed78f2db9726c855de105b77dfde70623 Mon Sep 17 00:00:00 2001 From: dapan Date: Sun, 27 Feb 2022 17:54:47 +0800 Subject: [PATCH 04/12] feature/qnode --- include/common/tep.h | 3 + include/libs/function/function.h | 5 +- include/libs/scalar/scalar.h | 2 +- source/common/src/tep.c | 1 - source/libs/scalar/inc/filterInt.h | 1 + source/libs/scalar/inc/sclInt.h | 9 +- source/libs/scalar/inc/sclvector.h | 2 +- source/libs/scalar/src/filter.c | 7 +- source/libs/scalar/src/scalar.c | 217 ++++--- source/libs/scalar/src/sclvector.c | 952 +++++++++++------------------ 10 files changed, 539 insertions(+), 660 deletions(-) diff --git a/include/common/tep.h b/include/common/tep.h index 956cf6ec36..0cd3f1dbd4 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -67,6 +67,9 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } +#define BitmapLen(_n) (((_n) + ((1<> NBIT) + + #define colDataGet(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 2b03fbd933..3c1c2cb0ea 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -228,7 +228,10 @@ typedef struct SAggFunctionInfo { typedef struct SScalarParam { void *data; - SColumnInfoData *columnData; + union { + SColumnInfoData *columnData; + void *data; + } orig; char *bitmap; bool dataInBlock; int32_t num; diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 1902c11cc6..2e876c8296 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -27,7 +27,7 @@ typedef struct SFilterInfo SFilterInfo; int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f..d7e7aec1e1 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -61,7 +61,6 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } -#define BitmapLen(_n) (((_n) + ((1<> NBIT) int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index a0d4bc680c..646e75e11d 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -253,6 +253,7 @@ typedef struct SFilterInfo { uint32_t *blkUnits; int8_t *blkUnitRes; void *pTable; + SArray *blkList; SFilterPCtx pctx; } SFilterInfo; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 41ee90667e..748c0df797 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -24,7 +24,7 @@ extern "C" { typedef struct SScalarCtx { int32_t code; - SSDataBlock *pSrc; + SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ } SScalarCtx; @@ -44,10 +44,13 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) - +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx); +bool sclIsNull(SScalarParam* param, int32_t idx); +void sclSetNull(SScalarParam* param, int32_t idx); +void sclFreeParam(SScalarParam *param); #ifdef __cplusplus } #endif -#endif // TDENGINE_SCALARINT_H \ No newline at end of file +#endif // TDENGINE_SCALARINT_H diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index fd0f8b896f..51af74208d 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -23,7 +23,7 @@ extern "C" { #include "sclfunc.h" typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); -typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5eb0003662..d6caea8563 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3668,7 +3668,12 @@ _return: bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + SArray *pList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(pList, &pSrc); + + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); + + taosArrayDestroy(pList); *p = output.data; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 05af5f63b4..c2ce09210f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -5,6 +5,7 @@ #include "functionMgt.h" #include "sclvector.h" #include "sclInt.h" +#include "tep.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -87,15 +88,23 @@ _return: SCL_RET(code); } -bool sclIsNull(SScalarParam* param, int32_t idx) { +FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { if (param->dataInBlock) { - return colDataIsNull(param->columnData, 0, idx, NULL); + return colDataIsNull(param->orig.columnData, 0, idx, NULL); } - return colDataIsNull_f(param->bitmap, idx); + return param->bitmap ? colDataIsNull_f(param->bitmap, idx) : false; } -void sclSetNull(SScalarParam* param, int32_t idx) { +FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { + if (NULL == param->bitmap) { + param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + if (NULL == param->bitmap) { + sclError("calloc %d failed", param->num); + return; + } + } + colDataSetNull_f(param->bitmap, idx); } @@ -107,7 +116,7 @@ void sclFreeRes(SHashObj *res) { p = (SScalarParam *)pIter; if (p) { - tfree(p->data); + sclFreeParam(p); } pIter = taosHashIterate(res, pIter); @@ -117,7 +126,15 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - tfree(param->data); + tfree(param->bitmap); + + if (!param->dataInBlock) { + if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { + taosHashCleanup((SHashObj *)param->orig.data); + } else { + tfree(param->orig.data); + } + } } int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { @@ -140,30 +157,45 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->orig.data = param->data; param->num = 1; param->type = SCL_DATA_TYPE_DUMMY_HASH; param->dataInBlock = false; + + if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { + taosHashCleanup(param->orig.data); + param->orig.data = NULL; + sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } break; } case QUERY_NODE_COLUMN: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + if (NULL == ctx->pBlockList) { + sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } SColumnNode *ref = (SColumnNode *)node; - if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { - sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); + if (ref->tupleId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", ref->tupleId, (int32_t)taosArrayGetSize(ctx->pBlockList)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); + SSDataBlock *block = taosArrayGet(ctx->pBlockList, ref->tupleId); + + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId); param->data = NULL; - param->columnData = columnData; + param->orig.columnData = columnData; param->dataInBlock = true; - param->num = ctx->pSrc->info.rows; + param->num = block->info.rows; param->type = columnData->info.type; param->bytes = columnData->info.bytes; @@ -171,11 +203,6 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_OPERATOR: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -211,13 +238,15 @@ int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) } if (param->dataInBlock) { - param->data = colDataGet(param->columnData, idx); + param->data = colDataGet(param->orig.columnData, idx); } else if (idx) { if (IS_VAR_DATA_TYPE(param->type)) { param->data = (char *)(param->data) + varDataTLen(param->data); } else { param->data = (char *)(param->data) + tDataTypes[param->type].bytes; } + } else { + param->data = param->orig.data; } } @@ -307,6 +336,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; for (int32_t i = 0; i < rowNum; ++i) { sclMoveParamListData(output, 1, i); @@ -358,9 +388,8 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; - void *data = output->data; - bool value = false; for (int32_t i = 0; i < rowNum; ++i) { @@ -382,8 +411,6 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o *(bool *)output->data = value; } - output->data = data; - return TSDB_CODE_SUCCESS; _return: @@ -407,6 +434,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); @@ -414,7 +442,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); + OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); return TSDB_CODE_SUCCESS; @@ -425,12 +453,12 @@ _return: } -EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { +EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -438,7 +466,7 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -459,12 +487,12 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { +EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecLogic(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -472,7 +500,7 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -493,12 +521,12 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { +EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecOperator(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -506,7 +534,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -533,28 +561,29 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - return sclRewriteFunction(pNode, pContext); + return sclRewriteFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { - return sclRewriteLogic(pNode, pContext); + return sclRewriteLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, pContext); + return sclRewriteOperator(pNode, ctx); } sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); - - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -EDealRes sclWalkFunction(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)pNode; SScalarParam output = {0}; @@ -572,8 +601,7 @@ EDealRes sclWalkFunction(SNode* pNode, void* pContext) { } -EDealRes sclWalkLogic(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)pNode; SScalarParam output = {0}; @@ -591,8 +619,7 @@ EDealRes sclWalkLogic(SNode* pNode, void* pContext) { } -EDealRes sclWalkOperator(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)pNode; SScalarParam output = {0}; @@ -609,27 +636,69 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { + STargetNode *target = (STargetNode *)pNode; + + if (target->tupleId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->tupleId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = taosArrayGet(ctx->pBlockList, target->tupleId); + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("target slot not exist, slotId:%d, dataBlockNum:%d", target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); + ctx->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + + for (int32_t i = 0; i < res->num; ++i) { + sclMoveParamListData(res, 1, i); + + colDataAppend(col, i, res->data, sclIsNull(res, i)); + } + + sclFreeParam(res); + + return DEAL_RES_CONTINUE; +} + EDealRes sclCalcWalker(SNode* pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } + + SScalarCtx *ctx = (SScalarCtx *)pContext; if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - return sclWalkFunction(pNode, pContext); + return sclWalkFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { - return sclWalkLogic(pNode, pContext); + return sclWalkLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, pContext); + return sclWalkOperator(pNode, ctx); + } + + if (QUERY_NODE_TARGET == nodeType(pNode)) { + return sclWalkTarget(pNode, ctx); } sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); - - SScalarCtx *ctx = (SScalarCtx *)pContext; ctx->code = TSDB_CODE_QRY_INVALID_INPUT; @@ -644,26 +713,33 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { } int32_t code = 0; - - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); - - if (code) { - nodesDestroyNode(pNode); - SCL_ERR_RET(code); + SScalarCtx ctx = {0}; + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); + + SCL_ERR_JRET(ctx.code); *pRes = pNode; - SCL_RET(code); +_return: + + sclFreeRes(ctx.pRes); + + return code; } -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { - if (NULL == pNode || NULL == pSrc || NULL == pDst) { +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { + if (NULL == pNode || NULL == pBlockList || NULL == pDst) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; + SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList}; ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -673,23 +749,24 @@ int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); - if (ctx.code) { - nodesDestroyNode(pNode); - sclFreeRes(ctx.pRes); - SCL_ERR_RET(ctx.code); - } + SCL_ERR_JRET(ctx.code); SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (NULL == res) { - sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); + *pDst = *res; +_return: + nodesDestroyNode(pNode); + sclFreeRes(ctx.pRes); - return TSDB_CODE_SUCCESS; + return code; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 57899f0e82..4003603eb5 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -261,19 +261,19 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } -static FORCE_INLINE void convertToSigned(char *buf, SScalarParam* pOut, int32_t outType) { +static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t outType) { int64_t value = strtoll(buf, NULL, 10); SET_TYPED_DATA(pOut->data, outType, value); } -static FORCE_INLINE void convertToUnsigned(char *buf, SScalarParam* pOut, int32_t outType) { +static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t outType) { uint64_t value = strtoull(buf, NULL, 10); SET_TYPED_DATA(pOut->data, outType, value); } -static FORCE_INLINE void convertToFloat(char *buf, SScalarParam* pOut, int32_t outType) { - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); +static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t outType) { + double value = strtod(buf, NULL); + SET_TYPED_DATA(pOut->data, outType, value); } @@ -282,14 +282,14 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t char *tmp = NULL; _bufConverteFunc func = NULL; - if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) { - func = convertToSigned; + if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType || TSDB_DATA_TYPE_BOOL == outType) { + func = varToSigned; } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { - func = convertToUnsigned; + func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { - func = convertToFloat; + func = varToFloat; } else { - sclError("unknown outType:%d", outType); + sclError("invalid convert outType:%d", outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -301,29 +301,49 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t sclSetNull(pOut, i); continue; } - - if (varDataLen(pIn->data) >= bufSize) { - bufSize = varDataLen(pIn->data) + 1; - tmp = realloc(tmp, bufSize); + + if (TSDB_DATA_TYPE_BINARY == inType) { + if (varDataLen(pIn->data) >= bufSize) { + bufSize = varDataLen(pIn->data) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); + tmp[varDataLen(pIn->data)] = 0; + } else { + if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; } - memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); - tmp[varDataLen(pIn->data)] = 0; - (*func)(tmp, pOut, outType); } tfree(tmp); + + return TSDB_CODE_SUCCESS; } int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int16_t inType = pIn->type; int16_t inBytes = pIn->bytes; - char *input = pIn->data; int16_t outType = pOut->type; int16_t outBytes = pOut->bytes; - char *output = pOut->data; + if (inType == TSDB_DATA_TYPE_BINARY || inType == TSDB_DATA_TYPE_NCHAR) { + return vectorConvertFromVarData(pIn, pOut, inType, outType); + } + switch (outType) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -331,234 +351,55 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = 0; - char *tmp = NULL; + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - for (int32_t i = 0; i < pIn->num; ++i) { - sclMoveParamListData(pIn, 1, i); - sclMoveParamListData(pOut, 1, i); - - if (sclIsNull(pIn, i)) { - sclSetNull(pOut, i); - continue; - } - - if (varDataLen(pIn->data) >= bufSize) { - bufSize = varDataLen(pIn->data) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); - tmp[varDataLen(pIn->data)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(pOut->data, outType, value); + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = 0; - char *tmp = NULL; - - for (int32_t i = 0; i < pIn->num; ++i) { - sclMoveParamListData(pIn, 1, i); - sclMoveParamListData(pOut, 1, i); - - if (sclIsNull(pIn, i)) { - sclSetNull(pOut, i); - continue; - } - if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } + int64_t value = 0; - int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(pOut->data, outType, value); - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + GET_TYPED_DATA(value, int64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } + uint64_t value = 0; - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + GET_TYPED_DATA(value, uint64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } + double value = 0; - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + GET_TYPED_DATA(value, double, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; default: @@ -673,7 +514,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } -void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -686,11 +527,6 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -700,62 +536,65 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -768,11 +607,6 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -782,60 +616,65 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); - } - } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -848,11 +687,6 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -862,62 +696,66 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); - } - } - tfree(leftParam.data); - tfree(rightParam.data); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + } + } + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -930,11 +768,6 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -944,69 +777,66 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } + } - SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) / getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) / getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1019,11 +849,6 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1033,92 +858,92 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double * output = (double *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double left = getVectorDoubleValueFnLeft(pLeft->data, i); double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pLeft->num == 1) { double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pRight->num == 1) { double right = getVectorDoubleValueFnRight(pRight->data, 0); - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double left = getVectorDoubleValueFnLeft(pLeft->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -1168,11 +993,10 @@ void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); } } - } -void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1181,15 +1005,10 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1198,63 +1017,67 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } - if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + SET_BIGINT_VAL(pOut->data, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + + SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + + SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1263,15 +1086,10 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1280,131 +1098,119 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - - if (pRight->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + SET_BIGINT_VAL(pOut->data, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + + SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + + SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); bool res = false; - - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->dataInBlock) { - getVectorValueAddrFnRight = getVectorValueAddr_default; - } else { - getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - } - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pLeft->num == 1) { - void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); - - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { - void *rightData = getVectorValueAddrFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } } -void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { SScalarParam pLeftOut = {0}; SScalarParam pRightOut = {0}; @@ -1426,118 +1232,100 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t param2 = pRight; } - vectorCompareImpl(param1, param2, out, _ord, optr); + vectorCompareImpl(param1, param2, pOut, _ord, optr); } -void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } -void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_EQUAL); } -void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_THAN); } -void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_EQUAL); } -void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_EQUAL); } -void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_EQUAL); } -void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_IN); } -void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_IN); } -void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LIKE); } -void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_LIKE); } -void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_MATCH); } -void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } -void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } -void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - - bool *output = (bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->dataInBlock) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } + } -void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; - - if (pLeft->dataInBlock) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - - vectorConvertImpl(pLeft, &output); +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertImpl(pLeft, pOut); } From 62e5ecd6d496fd3dd87f9454713258d30585e81f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Feb 2022 09:38:44 +0800 Subject: [PATCH 05/12] feature/qnode --- source/libs/scalar/src/scalar.c | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 114 +++++++++++++----- 2 files changed, 83 insertions(+), 33 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index c2ce09210f..1966c47216 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -183,7 +183,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSDataBlock *block = taosArrayGet(ctx->pBlockList, ref->tupleId); + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->tupleId); if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index df746e19cf..5e39521bf4 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -56,6 +56,35 @@ void scltInitLogFile() { } } +void scltAppendReservedSlot(SArray *pBlockList, int16_t *tupleId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { + if (newBlock) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 1; + res->info.rows = rows; + res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SColumnInfoData idata = {{0}}; + idata.info = *colInfo; + + taosArrayPush(res->pDataBlock, &idata); + taosArrayPush(pBlockList, &res); + + blockDataEnsureCapacity(res, rows); + + *tupleId = taosArrayGetSize(pBlockList) - 1; + *slotId = 0; + } else { + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); + res->info.numOfCols++; + SColumnInfoData idata = {{0}}; + idata.info = *colInfo; + + taosArrayPush(res->pDataBlock, &idata); + blockDataEnsureCapacity(res, rows); + + *tupleId = taosArrayGetSize(pBlockList) - 1; + *slotId = taosArrayGetSize(res->pDataBlock) - 1; + } +} void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { SNode *node = nodesMakeNode(QUERY_NODE_VALUE); @@ -74,7 +103,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; @@ -90,7 +119,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -100,7 +129,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; + idata.info.colId = 3; idata.pData = (char *)value; if (IS_VAR_DATA_TYPE(dataType)) { idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); @@ -111,7 +140,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *block = res; } else { @@ -121,12 +150,12 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; + idata.info.colId = 1 + idx; idata.pData = (char *)value; taosArrayPush(res->pDataBlock, &idata); rnode->slotId = idx; - rnode->colId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -171,6 +200,17 @@ void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi *pNode = (SNode *)onode; } +void scltMakeTargetNode(SNode **pNode, int16_t tupleId, int16_t slotId, SNode *snode) { + SNode *node = nodesMakeNode(QUERY_NODE_TARGET); + STargetNode *onode = (STargetNode *)node; + onode->pExpr = snode; + onode->tupleId = tupleId; + onode->slotId = slotId; + + *pNode = (SNode *)onode; +} + + } @@ -802,22 +842,32 @@ TEST(columnTest, smallint_value_add_int_column) { int16_t rightv[5]= {0, -5, -4, 23, 100}; double eRes[5] = {1.0, -4, -3, 24, 101}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t tupleId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &tupleId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, tupleId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGet(column, i)), eRes[i]); } } +#if 0 TEST(columnTest, bigint_column_multi_binary_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int64_t leftv[5]= {1, 2, 3, 4, 5}; @@ -831,8 +881,8 @@ TEST(columnTest, bigint_column_multi_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); int32_t code = scalarCalculate(opNode, src, &res); @@ -858,8 +908,8 @@ TEST(columnTest, smallint_column_and_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); int32_t code = scalarCalculate(opNode, src, &res); @@ -880,8 +930,8 @@ TEST(columnTest, smallint_column_or_float_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); int32_t code = scalarCalculate(opNode, src, &res); @@ -902,7 +952,7 @@ TEST(columnTest, smallint_column_or_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); @@ -924,7 +974,7 @@ TEST(columnTest, smallint_column_greater_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -946,7 +996,7 @@ TEST(columnTest, int_column_in_double_list) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -994,7 +1044,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -1031,7 +1081,7 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); @@ -1063,7 +1113,7 @@ TEST(columnTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -1094,7 +1144,7 @@ TEST(columnTest, binary_column_is_null) { setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -1125,7 +1175,7 @@ TEST(columnTest, binary_column_is_not_null) { setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -1150,11 +1200,11 @@ TEST(columnTest, greater_and_lower) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(v1)/sizeof(v1[0]); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); list[0] = opNode1; list[1] = opNode2; @@ -1169,7 +1219,7 @@ TEST(columnTest, greater_and_lower) { ASSERT_EQ(*((bool *)res.data + i), eRes[i]); } } - +#endif int main(int argc, char** argv) { From 97e7a41a3f8aa87521b6ed5d564370906612e166 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Feb 2022 15:50:22 +0800 Subject: [PATCH 06/12] feature/qnode --- include/common/tep.h | 1 + source/common/src/tep.c | 71 ++-- source/libs/nodes/src/nodesTraverseFuncs.c | 3 + source/libs/scalar/inc/sclvector.h | 3 + source/libs/scalar/src/filter.c | 1 + source/libs/scalar/src/scalar.c | 38 +- source/libs/scalar/src/sclvector.c | 273 +++++++++++++- .../libs/scalar/test/scalar/scalarTests.cpp | 348 ++++++++++++------ 8 files changed, 572 insertions(+), 166 deletions(-) diff --git a/include/common/tep.h b/include/common/tep.h index 0cd3f1dbd4..f85a0f7914 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -101,6 +101,7 @@ size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSi int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); void *blockDataDestroy(SSDataBlock *pBlock); diff --git a/source/common/src/tep.c b/source/common/src/tep.c index d7e7aec1e1..38f602f8db 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -120,14 +120,18 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; switch(type) { + case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} + case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} + case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} default: assert(0); } @@ -1040,36 +1044,47 @@ void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { } } +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { + char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumn->varmeta.offset = (int32_t*)tmp; + memset(pColumn->varmeta.offset, 0, sizeof(int32_t) * numOfRows); + + pColumn->varmeta.length = 0; + pColumn->varmeta.allocLen = 0; + tfree(pColumn->pData); + } else { + char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumn->nullbitmap = tmp; + memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); + + tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumn->pData = tmp; + } + + return TSDB_CODE_SUCCESS; +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(p->info.type)) { - char* tmp = realloc(p->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - p->varmeta.offset = (int32_t*)tmp; - memset(p->varmeta.offset, 0, sizeof(int32_t) * numOfRows); - - p->varmeta.length = 0; - p->varmeta.allocLen = 0; - tfree(p->pData); - } else { - char* tmp = realloc(p->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - p->nullbitmap = tmp; - memset(p->nullbitmap, 0, BitmapLen(numOfRows)); - - tmp = realloc(p->pData, numOfRows * p->info.bytes); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - p->pData = tmp; + code = blockDataEnsureColumnCapacity(p, numOfRows); + if (code) { + return code; } } @@ -1097,4 +1112,4 @@ void* blockDataDestroy(SSDataBlock* pBlock) { tfree(pBlock->pBlockAgg); tfree(pBlock); return NULL; -} \ No newline at end of file +} diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e61375b202..b842986b78 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -105,6 +105,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_RAW_EXPR: res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; + case QUERY_NODE_TARGET: + res = walkNode(((STargetNode*)pNode)->pExpr, order, walker, pContext); + break; default: break; } diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 51af74208d..09b813359a 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -22,6 +22,9 @@ extern "C" { #include "sclfunc.h" +typedef double (*_mathFunc)(double, double, bool *); + + typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 271578d59b..3473b624e3 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3692,3 +3692,4 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData + \ No newline at end of file diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1966c47216..3d5ba9e914 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -142,9 +142,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; param->data = nodesGetValueFromNode(valueNode); + param->orig.data = param->data; param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; + if (TSDB_DATA_TYPE_NULL == param->type) { + sclSetNull(param, 0); + } param->dataInBlock = false; break; @@ -178,12 +182,12 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SColumnNode *ref = (SColumnNode *)node; - if (ref->tupleId >= taosArrayGetSize(ctx->pBlockList)) { - sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", ref->tupleId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + if (ref->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", ref->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->tupleId); + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->dataBlockId); if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); @@ -639,15 +643,15 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { STargetNode *target = (STargetNode *)pNode; - if (target->tupleId >= taosArrayGetSize(ctx->pBlockList)) { - sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->tupleId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } - SSDataBlock *block = taosArrayGet(ctx->pBlockList, target->tupleId); + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { - sclError("target slot not exist, slotId:%d, dataBlockNum:%d", target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } @@ -734,7 +738,7 @@ _return: } int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { - if (NULL == pNode || NULL == pBlockList || NULL == pDst) { + if (NULL == pNode || NULL == pBlockList) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -751,15 +755,17 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(ctx.code); - SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); - if (NULL == res) { - sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + if (pDst) { + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); + + *pDst = *res; } - - taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); - - *pDst = *res; _return: diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4003603eb5..7df9b43284 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -485,6 +485,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut1->orig.data = paramOut1->data; code = vectorConvertImpl(param1, paramOut1); if (code) { @@ -502,6 +503,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p tfree(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { @@ -514,10 +516,11 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } -void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { +void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, _mathFunc func) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - + double leftv = 0, rightv = 0; + bool isNull = false; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { @@ -526,6 +529,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -539,6 +543,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -548,9 +553,6 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pRight = &rightParam; } - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - if (pLeft->num == pRight->num) { for (; i < pRight->num && i >= 0; i += step) { sclMoveParamListData(pLeft, 1, i); @@ -562,9 +564,19 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in continue; } - SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); sclMoveParamListData(pOut, 1, i); @@ -574,9 +586,18 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in continue; } - SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } } } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + for (; i >= 0 && i < pLeft->num; i += step) { sclMoveParamListData(pLeft, 1, i); sclMoveParamListData(pOut, 1, i); @@ -586,7 +607,159 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in continue; } - SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); +} + +double mathAdd(double leftv, double rightv, bool *isNull) { + return leftv + rightv; +} + +double mathSub(double leftv, double rightv, bool *isNull) { + return leftv - rightv; +} + +double mathMultiply(double leftv, double rightv, bool *isNull) { + return leftv * rightv; +} + +double mathDivide(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv / rightv; +} + +double mathRemainder(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv - ((int64_t)(leftv / rightv)) * rightv; +} + + +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathAdd); +} + +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathSub); +} + +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathMultiply); +} + +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathDivide); +} + +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathRemainder); +} + +#if 0 +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + leftParam.orig.data = leftParam.data; + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); + return; + } + rightParam.orig.data = rightParam.data; + + if (vectorConvertImpl(pRight, &rightParam)) { + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); + return; + } + pRight = &rightParam; + } + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); + } + } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); + } + } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } @@ -597,6 +770,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -606,6 +780,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -619,6 +794,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -643,9 +819,15 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in continue; } + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); sclMoveParamListData(pOut, 1, i); @@ -677,6 +859,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -686,6 +869,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -699,6 +883,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -723,9 +908,15 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu continue; } + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); sclMoveParamListData(pOut, 1, i); @@ -758,6 +949,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -767,6 +959,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -780,6 +973,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -804,6 +998,9 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, continue; } + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { @@ -839,6 +1036,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -848,6 +1046,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -861,6 +1060,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -892,8 +1092,9 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO continue; } - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - double right = getVectorDoubleValueFnRight(pRight->data, i); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pLeft->num == 1) { @@ -946,6 +1147,8 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO sclFreeParam(&rightParam); } +#endif + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; @@ -999,6 +1202,7 @@ void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; @@ -1008,6 +1212,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -1021,6 +1226,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -1045,9 +1251,15 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, continue; } - SET_BIGINT_VAL(pOut->data, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); sclMoveParamListData(pOut, 1, i); @@ -1056,10 +1268,15 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclSetNull(pOut, i); continue; } + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); - SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + for (; i >= 0 && i < pLeft->num; i += step) { sclMoveParamListData(pLeft, 1, i); sclMoveParamListData(pOut, 1, i); @@ -1069,7 +1286,9 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, continue; } - SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } @@ -1080,6 +1299,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; @@ -1089,6 +1309,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; if (vectorConvertImpl(pLeft, &leftParam)) { return; @@ -1102,6 +1323,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { sclFreeParam(&leftParam); @@ -1125,9 +1347,15 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, continue; } - SET_BIGINT_VAL(pOut->data, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); sclMoveParamListData(pOut, 1, i); @@ -1136,10 +1364,15 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, sclSetNull(pOut, i); continue; } + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); - SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + for (; i >= 0 && i < pLeft->num; i += step) { sclMoveParamListData(pLeft, 1, i); sclMoveParamListData(pOut, 1, i); @@ -1149,7 +1382,9 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, continue; } - SET_BIGINT_VAL(pOut->data,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } @@ -1181,8 +1416,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + for (; i >= 0 && i < pRight->num; i += step) { sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { sclSetNull(pOut, i); @@ -1195,8 +1433,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + for (; i >= 0 && i < pLeft->num; i += step) { sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { sclSetNull(pOut, i); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8210e9c03f..eaa2ddc247 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -50,19 +50,20 @@ void scltInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } -void scltAppendReservedSlot(SArray *pBlockList, int16_t *tupleId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { +void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 1; res->info.rows = rows; res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - SColumnInfoData idata = {{0}}; + SColumnInfoData idata = {0}; idata.info = *colInfo; taosArrayPush(res->pDataBlock, &idata); @@ -70,18 +71,19 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *tupleId, int16_t *slotI blockDataEnsureCapacity(res, rows); - *tupleId = taosArrayGetSize(pBlockList) - 1; + *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); res->info.numOfCols++; - SColumnInfoData idata = {{0}}; + SColumnInfoData idata = {0}; idata.info = *colInfo; + blockDataEnsureColumnCapacity(&idata, rows); + taosArrayPush(res->pDataBlock, &idata); - blockDataEnsureCapacity(res, rows); - *tupleId = taosArrayGetSize(pBlockList) - 1; + *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1; } } @@ -130,14 +132,21 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.type = dataType; idata.info.bytes = dataBytes; idata.info.colId = 3; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; rnode->colId = 3; @@ -151,8 +160,22 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.type = dataType; idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; - idata.pData = (char *)value; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); + res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; rnode->colId = 1 + idx; @@ -200,11 +223,11 @@ void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi *pNode = (SNode *)onode; } -void scltMakeTargetNode(SNode **pNode, int16_t tupleId, int16_t slotId, SNode *snode) { +void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode *snode) { SNode *node = nodesMakeNode(QUERY_NODE_TARGET); STargetNode *onode = (STargetNode *)node; onode->pExpr = snode; - onode->tupleId = tupleId; + onode->dataBlockId = dataBlockId; onode->slotId = slotId; *pNode = (SNode *)onode; @@ -699,7 +722,7 @@ TEST(constantTest, int_is_null1) { TEST(constantTest, int_is_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -728,8 +751,8 @@ TEST(constantTest, int_is_not_null1) { TEST(constantTest, int_is_not_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -837,6 +860,8 @@ TEST(constantTest, greater_and_lower) { TEST(columnTest, smallint_value_add_int_column) { + scltInitLogFile(); + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int32_t leftv = 1; int16_t rightv[5]= {0, -5, -4, 23, 100}; @@ -850,9 +875,9 @@ TEST(columnTest, smallint_value_add_int_column) { SArray *blockList = taosArrayInit(2, POINTER_BYTES); taosArrayPush(blockList, &src); SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; - int16_t tupleId = 0, slotId = 0; - scltAppendReservedSlot(blockList, &tupleId, &slotId, true, rowNum, &colInfo); - scltMakeTargetNode(&opNode, tupleId, slotId, opNode); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); @@ -867,7 +892,6 @@ TEST(columnTest, smallint_value_add_int_column) { } } -#if 0 TEST(columnTest, bigint_column_multi_binary_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int64_t leftv[5]= {1, 2, 3, 4, 5}; @@ -879,19 +903,29 @@ TEST(columnTest, bigint_column_multi_binary_column) { } double eRes[5] = {0, 2, 6, 12, 20}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGet(column, i)), eRes[i]); } } @@ -906,19 +940,28 @@ TEST(columnTest, smallint_column_and_binary_column) { } int64_t eRes[5] = {0, 0, 2, 0, 4}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } } @@ -928,19 +971,28 @@ TEST(columnTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; int64_t eRes[5] = {3, 3, 7, 5, 7}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } } @@ -950,19 +1002,28 @@ TEST(columnTest, smallint_column_or_double_value) { double rightv= 10.2; int64_t eRes[5] = {11, 10, 11, 14, 15}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } } @@ -972,19 +1033,28 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; bool eRes[5] = {false, false, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -994,7 +1064,6 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); @@ -1006,14 +1075,24 @@ TEST(columnTest, int_column_in_double_list) { nodesListAppend(list, pRight); scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -1021,7 +1100,6 @@ TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -1055,13 +1133,23 @@ TEST(columnTest, binary_column_in_binary_list) { scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -1070,7 +1158,6 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -1088,21 +1175,31 @@ TEST(columnTest, binary_column_like_binary) { scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } + TEST(columnTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -1117,13 +1214,23 @@ TEST(columnTest, binary_column_is_true) { scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -1131,30 +1238,41 @@ TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -1162,30 +1280,40 @@ TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } @@ -1198,28 +1326,36 @@ TEST(columnTest, greater_and_lower) { int32_t v4[5]= {5, 3, 4, 2, 6}; bool eRes[5] = {false, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(v1)/sizeof(v1[0]); scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); - scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v2); scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); - scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); - scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, v3); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v4); scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); list[0] = opNode1; list[1] = opNode2; scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); - int32_t code = scalarCalculate(logicNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&logicNode, dataBlockId, slotId, logicNode); + + int32_t code = scalarCalculate(logicNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } } -#endif int main(int argc, char** argv) { From 454ef8dfdd0474ea26f0f28461621505269f45fe Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Feb 2022 16:52:48 +0800 Subject: [PATCH 07/12] feature/qnode --- source/libs/scalar/src/scalar.c | 1 + .../libs/scalar/test/filter/filterTests.cpp | 150 +++++++++++------- 2 files changed, 90 insertions(+), 61 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 3d5ba9e914..65b413639a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -763,6 +763,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { } taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); + sclMoveParamListData(res, 1, 0); *pDst = *res; } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index a0618cc662..02dcfbd3a1 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -51,6 +51,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -75,16 +76,20 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + static uint64_t dbidx = 0; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; + + sprintf(rnode->dbName, "%" PRIu64, dbidx++); if (NULL == block) { rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *pNode = (SNode *)rnode; return; @@ -99,7 +104,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -109,18 +114,25 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *block = res; } else { @@ -130,14 +142,26 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; - idata.pData = (char *)value; + idata.info.colId = 1 + idx; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; - rnode->colId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -200,7 +224,7 @@ TEST(timerangeTest, greater) { bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); @@ -219,10 +243,10 @@ TEST(timerangeTest, greater_and_lower) { bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); SNode *list[2] = {0}; @@ -250,7 +274,7 @@ TEST(columnTest, smallint_column_greater_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -301,7 +325,7 @@ TEST(columnTest, int_column_greater_smallint_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -353,7 +377,7 @@ TEST(columnTest, int_column_in_double_list) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -416,7 +440,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -465,7 +489,7 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); @@ -499,20 +523,21 @@ TEST(columnTest, binary_column_is_null) { char leftv[5][5]= {0}; SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); SFilterInfo *filter = NULL; @@ -543,17 +568,18 @@ TEST(columnTest, binary_column_is_not_null) { SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -588,8 +614,8 @@ TEST(opTest, smallint_column_greater_int_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); SFilterInfo *filter = NULL; @@ -623,7 +649,7 @@ TEST(opTest, smallint_value_add_int_column) { SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -663,8 +689,8 @@ TEST(opTest, bigint_column_multi_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -702,8 +728,8 @@ TEST(opTest, smallint_column_and_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -736,8 +762,8 @@ TEST(opTest, smallint_column_or_float_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -772,7 +798,7 @@ TEST(opTest, smallint_column_or_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -814,7 +840,7 @@ TEST(opTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -841,6 +867,8 @@ TEST(opTest, binary_column_is_true) { TEST(filterModelogicTest, diff_columns_and_or_and) { + flttInitLogFile(); + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; SNode *logicNode1 = NULL, *logicNode2 = NULL; double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; @@ -851,12 +879,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -866,12 +894,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -916,7 +944,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -979,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -994,12 +1022,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -1044,7 +1072,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -1110,13 +1138,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); - flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); - flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -1125,13 +1153,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); - flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); - flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); From 5dc9d9776cef2665d80ff8d68312a04bef39a55e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Feb 2022 19:35:22 +0800 Subject: [PATCH 08/12] feature/qnode --- source/libs/nodes/src/nodesTraverseFuncs.c | 3 + source/libs/nodes/src/nodesUtilFuncs.c | 37 ++++++++++--- source/libs/scalar/src/scalar.c | 51 +++++++++++++---- source/libs/scalar/src/sclvector.c | 3 + .../libs/scalar/test/scalar/scalarTests.cpp | 55 ++++++++++++++++++- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index b842986b78..5c81cce94a 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -231,6 +231,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_RAW_EXPR: res = rewriteNode(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; + case QUERY_NODE_TARGET: + res = rewriteNode(&(((STargetNode*)pNode)->pExpr), order, rewriter, pContext); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 5d51b2e523..ae4c5675f0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -101,20 +101,39 @@ SNode* nodesMakeNode(ENodeType type) { return NULL; } -static EDealRes destroyNode(SNode* pNode, void* pContext) { - switch (nodeType(pNode)) { - case QUERY_NODE_VALUE: - tfree(((SValueNode*)pNode)->literal); +static EDealRes destroyNode(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_VALUE: { + SValueNode* pValue = (SValueNode*)*pNode; + + tfree(pValue->literal); + if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + tfree(pValue->datum.p); + } + + break; + } + case QUERY_NODE_LOGIC_CONDITION: + nodesDestroyList(((SLogicConditionNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_FUNCTION: + nodesDestroyList(((SFunctionNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_GROUPING_SET: + nodesDestroyList(((SGroupingSetNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_NODE_LIST: + nodesDestroyList(((SNodeListNode*)(*pNode))->pNodeList); break; default: break; } - tfree(pNode); + tfree(*pNode); return DEAL_RES_CONTINUE; } void nodesDestroyNode(SNode* pNode) { - nodesWalkNodePostOrder(pNode, destroyNode, NULL); + nodesRewriteNodePostOrder(&pNode, destroyNode, NULL); } SNodeList* nodesMakeList() { @@ -191,9 +210,9 @@ SNode* nodesListGetNode(SNodeList* pList, int32_t index) { } void nodesDestroyList(SNodeList* pList) { - SNode* node; - FOREACH(node, pList) { - nodesDestroyNode(node); + SListCell* pNext = pList->pHead; + while (NULL != pNext) { + pNext = nodesListErase(pList, pNext); } tfree(pList); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 65b413639a..b6ffbe5beb 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -125,8 +125,13 @@ void sclFreeRes(SHashObj *res) { taosHashCleanup(res); } -void sclFreeParam(SScalarParam *param) { +void sclFreeParamNoData(SScalarParam *param) { tfree(param->bitmap); +} + + +void sclFreeParam(SScalarParam *param) { + sclFreeParamNoData(param); if (!param->dataInBlock) { if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { @@ -137,10 +142,28 @@ void sclFreeParam(SScalarParam *param) { } } + +int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { + if (TSDB_DATA_TYPE_NULL == pNode->node.resType.type) { + return TSDB_CODE_SUCCESS; + } + + *res = malloc(pNode->node.resType.bytes); + if (NULL == (*res)) { + sclError("malloc %d failed", pNode->node.resType.bytes); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); + + return TSDB_CODE_SUCCESS; +} + int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; + //SCL_ERR_RET(sclCopyValueNodeValue(valueNode, ¶m->data)); param->data = nodesGetValueFromNode(valueNode); param->orig.data = param->data; param->num = 1; @@ -172,7 +195,6 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); return TSDB_CODE_QRY_OUT_OF_MEMORY; } - break; } case QUERY_NODE_COLUMN: { @@ -353,11 +375,14 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu } } - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } @@ -415,10 +440,12 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o *(bool *)output->data = value; } - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); SCL_RET(code); } @@ -448,11 +475,15 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); - return TSDB_CODE_SUCCESS; - _return: + + for (int32_t i = 0; i < paramNum; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } @@ -665,8 +696,6 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); - for (int32_t i = 0; i < res->num; ++i) { sclMoveParamListData(res, 1, i); @@ -675,6 +704,8 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { sclFreeParam(res); + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 7df9b43284..55409d2df4 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1474,6 +1474,9 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } vectorCompareImpl(param1, param2, pOut, _ord, optr); + + sclFreeParam(&pLeftOut); + sclFreeParam(&pRightOut); } void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index eaa2ddc247..56b646a898 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -44,6 +44,10 @@ namespace { int64_t scltLeftV = 21, scltRightV = 10; double scltLeftVd = 21.0, scltRightVd = 10.0; +void scltFreeDataBlock(void *block) { + blockDataDestroy(*(SSDataBlock **)block); +} + void scltInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -96,7 +100,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { if (IS_VAR_DATA_TYPE(dataType)) { vnode->datum.p = (char *)malloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); - vnode->node.resType.bytes = varDataLen(value); + vnode->node.resType.bytes = varDataTLen(value); } else { vnode->node.resType.bytes = tDataTypes[dataType].bytes; assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); @@ -250,6 +254,7 @@ TEST(constantTest, bigint_add_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, double_sub_bigint) { @@ -265,6 +270,7 @@ TEST(constantTest, double_sub_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, tinyint_and_smallint) { @@ -280,6 +286,7 @@ TEST(constantTest, tinyint_and_smallint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); + nodesDestroyNode(res); } TEST(constantTest, bigint_or_double) { @@ -295,6 +302,7 @@ TEST(constantTest, bigint_or_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_or_binary) { @@ -313,6 +321,7 @@ TEST(constantTest, int_or_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); + nodesDestroyNode(res); } @@ -329,6 +338,7 @@ TEST(constantTest, int_greater_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_greater_equal_binary) { @@ -347,6 +357,7 @@ TEST(constantTest, int_greater_equal_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, tinyint_lower_ubigint) { @@ -362,6 +373,7 @@ TEST(constantTest, tinyint_lower_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); + nodesDestroyNode(res); } TEST(constantTest, usmallint_lower_equal_ubigint) { @@ -378,6 +390,7 @@ TEST(constantTest, usmallint_lower_equal_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv <= rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint1) { @@ -394,6 +407,7 @@ TEST(constantTest, int_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint2) { @@ -410,6 +424,7 @@ TEST(constantTest, int_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint1) { @@ -426,6 +441,7 @@ TEST(constantTest, int_not_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint2) { @@ -442,6 +458,7 @@ TEST(constantTest, int_not_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } @@ -469,6 +486,7 @@ TEST(constantTest, int_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_in_smallint2) { @@ -494,6 +512,7 @@ TEST(constantTest, int_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint1) { @@ -517,6 +536,7 @@ TEST(constantTest, int_not_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint2) { @@ -542,6 +562,7 @@ TEST(constantTest, int_not_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary1) { @@ -562,6 +583,7 @@ TEST(constantTest, binary_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary2) { @@ -582,6 +604,7 @@ TEST(constantTest, binary_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary1) { @@ -602,6 +625,7 @@ TEST(constantTest, binary_not_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary2) { @@ -622,6 +646,7 @@ TEST(constantTest, binary_not_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary1) { @@ -642,6 +667,7 @@ TEST(constantTest, binary_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary2) { @@ -662,6 +688,7 @@ TEST(constantTest, binary_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary1) { @@ -682,6 +709,7 @@ TEST(constantTest, binary_not_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary2) { @@ -702,6 +730,7 @@ TEST(constantTest, binary_not_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null1) { @@ -717,6 +746,7 @@ TEST(constantTest, int_is_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null2) { @@ -732,6 +762,7 @@ TEST(constantTest, int_is_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null1) { @@ -747,6 +778,7 @@ TEST(constantTest, int_is_not_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null2) { @@ -762,6 +794,7 @@ TEST(constantTest, int_is_not_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true1) { @@ -779,6 +812,7 @@ TEST(constantTest, int_add_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true2) { @@ -796,6 +830,7 @@ TEST(constantTest, int_add_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } @@ -814,6 +849,7 @@ TEST(constantTest, int_greater_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_greater_int_is_true2) { @@ -831,6 +867,7 @@ TEST(constantTest, int_greater_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, greater_and_lower) { @@ -855,6 +892,7 @@ TEST(constantTest, greater_and_lower) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } @@ -890,6 +928,8 @@ TEST(columnTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((double *)colDataGet(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, bigint_column_multi_binary_column) { @@ -927,6 +967,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((double *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, smallint_column_and_binary_column) { @@ -963,6 +1004,7 @@ TEST(columnTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, smallint_column_or_float_column) { @@ -994,6 +1036,7 @@ TEST(columnTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, smallint_column_or_double_value) { @@ -1025,6 +1068,7 @@ TEST(columnTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, smallint_column_greater_double_value) { @@ -1056,6 +1100,7 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, int_column_in_double_list) { @@ -1094,6 +1139,7 @@ TEST(columnTest, int_column_in_double_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, binary_column_in_binary_list) { @@ -1151,6 +1197,7 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, binary_column_like_binary) { @@ -1193,6 +1240,8 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); } @@ -1232,6 +1281,7 @@ TEST(columnTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, binary_column_is_null) { @@ -1274,6 +1324,7 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, binary_column_is_not_null) { @@ -1315,6 +1366,7 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } TEST(columnTest, greater_and_lower) { @@ -1355,6 +1407,7 @@ TEST(columnTest, greater_and_lower) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); } From c0c4b5c8c83f27b8ab6bf509cb954d4f03b78a79 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Mar 2022 09:30:23 +0800 Subject: [PATCH 09/12] feature/qnode --- source/libs/nodes/src/nodesUtilFuncs.c | 8 ++ source/libs/scalar/src/filter.c | 25 +++--- source/libs/scalar/src/scalar.c | 5 +- .../libs/scalar/test/filter/filterTests.cpp | 83 ++++++++++++++++++- .../libs/scalar/test/scalar/scalarTests.cpp | 19 ++++- 5 files changed, 123 insertions(+), 17 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index ae4c5675f0..eac5606bd6 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -102,6 +102,10 @@ SNode* nodesMakeNode(ENodeType type) { } static EDealRes destroyNode(SNode** pNode, void* pContext) { + if (NULL == pNode || NULL == *pNode) { + return DEAL_RES_IGNORE_CHILD; + } + switch (nodeType(*pNode)) { case QUERY_NODE_VALUE: { SValueNode* pValue = (SValueNode*)*pNode; @@ -133,6 +137,10 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { } void nodesDestroyNode(SNode* pNode) { + if (NULL == pNode) { + return; + } + nodesRewriteNodePostOrder(&pNode, destroyNode, NULL); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3473b624e3..0c7e7ba4f6 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -18,6 +18,7 @@ //#include "queryLog.h" #include "tcompare.h" #include "filterInt.h" +#include "sclInt.h" #include "filter.h" OptrStr gOptrStr[] = { @@ -1677,7 +1678,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); - + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); @@ -2783,7 +2784,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { + if (colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -2880,12 +2881,12 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, (*p)[i] = 1; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ (*p)[i] = 0; } }else{ - (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { all = false; @@ -2915,12 +2916,12 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows (*p)[i] = 0; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ // for json->'key' is not null (*p)[i] = 1; } }else { - (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { @@ -2951,7 +2952,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD for (int32_t i = 0; i < numOfRows; ++i) { void *colData = colDataGet((SColumnInfoData *)info->cunits[0].colData, i); - if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[0].colData, 0, i, NULL)) { all = false; continue; } @@ -2981,7 +2982,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); - if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)) { (*p)[i] = 0; all = false; continue; @@ -3038,7 +3039,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg //} else { uint8_t optr = cunit->optr; - if (colData == NULL || isNull(colData, cunit->dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -3675,7 +3676,10 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData taosArrayDestroy(pList); - *p = output.data; + *p = output.orig.data; + output.orig.data = NULL; + + sclFreeParam(&output); int8_t *r = output.data; for (int32_t i = 0; i < output.num; ++i) { @@ -3692,4 +3696,3 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData - \ No newline at end of file diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b6ffbe5beb..1abb684cf6 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -793,15 +793,16 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); sclMoveParamListData(res, 1, 0); *pDst = *res; + + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } _return: - nodesDestroyNode(pNode); + //nodesDestroyNode(pNode); sclFreeRes(ctx.pRes); return code; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 02dcfbd3a1..0366386ae1 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -220,7 +220,7 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { } TEST(timerangeTest, greater) { - SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; @@ -236,6 +236,8 @@ TEST(timerangeTest, greater) { ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, INT64_MAX); + filterFreeInfo(filter); + nodesDestroyNode(opNode1); } TEST(timerangeTest, greater_and_lower) { @@ -263,6 +265,8 @@ TEST(timerangeTest, greater_and_lower) { ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, tbig); + filterFreeInfo(filter); + nodesDestroyNode(logicNode); } @@ -315,6 +319,10 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + blockDataDestroy(src); + nodesDestroyNode(opNode); } TEST(columnTest, int_column_greater_smallint_value) { @@ -366,6 +374,10 @@ TEST(columnTest, int_column_greater_smallint_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -408,7 +420,10 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -470,6 +485,10 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -515,6 +534,10 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -559,6 +582,10 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(columnTest, binary_column_is_not_null) { @@ -602,6 +629,10 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -637,6 +668,10 @@ TEST(opTest, smallint_column_greater_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -672,6 +707,10 @@ TEST(opTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -713,6 +752,10 @@ TEST(opTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_and_binary_column) { @@ -752,6 +795,10 @@ TEST(opTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_or_float_column) { @@ -786,6 +833,10 @@ TEST(opTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -822,6 +873,10 @@ TEST(opTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -863,6 +918,10 @@ TEST(opTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -931,6 +990,10 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } TEST(filterModelogicTest, same_column_and_or_and) { @@ -993,6 +1056,10 @@ TEST(filterModelogicTest, same_column_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } @@ -1059,6 +1126,10 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } TEST(filterModelogicTest, same_column_or_and_or) { @@ -1121,6 +1192,10 @@ TEST(filterModelogicTest, same_column_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } @@ -1190,6 +1265,10 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 56b646a898..7e63ad4ed2 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -378,7 +378,8 @@ TEST(constantTest, tinyint_lower_ubigint) { TEST(constantTest, usmallint_lower_equal_ubigint) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int64_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -395,7 +396,8 @@ TEST(constantTest, usmallint_lower_equal_ubigint) { TEST(constantTest, int_equal_smallint1) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int16_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -930,6 +932,7 @@ TEST(columnTest, smallint_value_add_int_column) { } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, bigint_column_multi_binary_column) { @@ -968,6 +971,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { ASSERT_EQ(*((double *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_and_binary_column) { @@ -1005,6 +1009,7 @@ TEST(columnTest, smallint_column_and_binary_column) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_float_column) { @@ -1037,6 +1042,7 @@ TEST(columnTest, smallint_column_or_float_column) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_double_value) { @@ -1069,6 +1075,7 @@ TEST(columnTest, smallint_column_or_double_value) { ASSERT_EQ(*((int64_t *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_greater_double_value) { @@ -1101,6 +1108,7 @@ TEST(columnTest, smallint_column_greater_double_value) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, int_column_in_double_list) { @@ -1140,6 +1148,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_in_binary_list) { @@ -1198,6 +1207,7 @@ TEST(columnTest, binary_column_in_binary_list) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_like_binary) { @@ -1242,6 +1252,7 @@ TEST(columnTest, binary_column_like_binary) { } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } @@ -1282,6 +1293,7 @@ TEST(columnTest, binary_column_is_true) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_null) { @@ -1325,6 +1337,7 @@ TEST(columnTest, binary_column_is_null) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_not_null) { @@ -1367,6 +1380,7 @@ TEST(columnTest, binary_column_is_not_null) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, greater_and_lower) { @@ -1408,6 +1422,7 @@ TEST(columnTest, greater_and_lower) { ASSERT_EQ(*((bool *)colDataGet(column, i)), eRes[i]); } taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(logicNode); } From 5f3e439ddc997c504c18d10d719a5546a7cf2c82 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Mar 2022 09:42:28 +0800 Subject: [PATCH 10/12] feature/qnode --- source/libs/scalar/test/filter/filterTests.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 0366386ae1..fafc1ea42e 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -1012,6 +1012,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -1020,11 +1021,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { list = nodesMakeList(); - + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -1148,6 +1151,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -1157,10 +1161,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { list = nodesMakeList(); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); From 712137ba1823db254e6ab092bed023d35fb6401a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Mar 2022 10:32:36 +0800 Subject: [PATCH 11/12] feature/qnode --- include/libs/scalar/scalar.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 2e876c8296..e55c0b9e76 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -25,9 +25,17 @@ extern "C" { typedef struct SFilterInfo SFilterInfo; - +/* +pNode will be freed in API; +*pRes need to freed in caller +*/ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); + +/* +pDst need to freed in caller +*/ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); + int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); From 125091ad05ad1eab13ed5a61a8e5807d9b2ed756 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Mar 2022 11:14:28 +0800 Subject: [PATCH 12/12] feature/qnode --- source/common/src/tname.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 5e5816229f..e061862856 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -122,9 +122,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { return -1; } - int32_t len = 0; - - snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); + int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); size_t tnameLen = strlen(name->tname); if (tnameLen > 0) {